5101: Fix index settings opt out r=Kerollmops a=ManyTheFish

# Pull Request

## Related issue
Fixes #5099 

## What does this PR do?
- Refactor the settings implementation ensuring the routes are configured
- Add a test checking if all the routes are tested
- Refactor the tests to ease the modifications


Co-authored-by: ManyTheFish <many@meilisearch.com>
This commit is contained in:
meili-bors[bot] 2024-11-28 14:23:33 +00:00 committed by GitHub
commit d49d127863
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 426 additions and 398 deletions

View File

@ -17,6 +17,32 @@ use crate::extractors::authentication::GuardedData;
use crate::routes::{get_task_id, is_dry_run, SummarizedTaskView}; use crate::routes::{get_task_id, is_dry_run, SummarizedTaskView};
use crate::Opt; use crate::Opt;
/// This macro generates the routes for the settings.
///
/// It takes a list of settings and generates a module for each setting.
/// Each module contains the `get`, `update` and `delete` routes for the setting.
///
/// It also generates a `configure` function that configures the routes for the settings.
macro_rules! make_setting_routes {
($({route: $route:literal, update_verb: $update_verb:ident, value_type: $type:ty, err_type: $err_ty:ty, attr: $attr:ident, camelcase_attr: $camelcase_attr:literal, analytics: $analytics:ident},)*) => {
$(
make_setting_route!($route, $update_verb, $type, $err_ty, $attr, $camelcase_attr, $analytics);
)*
pub fn configure(cfg: &mut web::ServiceConfig) {
use crate::extractors::sequential_extractor::SeqHandler;
cfg.service(
web::resource("")
.route(web::patch().to(SeqHandler(update_all)))
.route(web::get().to(SeqHandler(get_all)))
.route(web::delete().to(SeqHandler(delete_all))))
$(.service($attr::resources()))*;
}
pub const ALL_SETTINGS_NAMES: &[&str] = &[$(stringify!($attr)),*];
};
}
#[macro_export] #[macro_export]
macro_rules! make_setting_route { macro_rules! make_setting_route {
($route:literal, $update_verb:ident, $type:ty, $err_ty:ty, $attr:ident, $camelcase_attr:literal, $analytics:ident) => { ($route:literal, $update_verb:ident, $type:ty, $err_ty:ty, $attr:ident, $camelcase_attr:literal, $analytics:ident) => {
@ -153,279 +179,227 @@ macro_rules! make_setting_route {
}; };
} }
make_setting_route!( make_setting_routes!(
"/filterable-attributes", {
put, route: "/filterable-attributes",
std::collections::BTreeSet<String>, update_verb: put,
meilisearch_types::deserr::DeserrJsonError< value_type: std::collections::BTreeSet<String>,
meilisearch_types::error::deserr_codes::InvalidSettingsFilterableAttributes, err_type: meilisearch_types::deserr::DeserrJsonError<
>, meilisearch_types::error::deserr_codes::InvalidSettingsFilterableAttributes,
filterable_attributes, >,
"filterableAttributes", attr: filterable_attributes,
FilterableAttributesAnalytics camelcase_attr: "filterableAttributes",
); analytics: FilterableAttributesAnalytics
},
make_setting_route!( {
"/sortable-attributes", route: "/sortable-attributes",
put, update_verb: put,
std::collections::BTreeSet<String>, value_type: std::collections::BTreeSet<String>,
meilisearch_types::deserr::DeserrJsonError< err_type: meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsSortableAttributes, meilisearch_types::error::deserr_codes::InvalidSettingsSortableAttributes,
>, >,
sortable_attributes, attr: sortable_attributes,
"sortableAttributes", camelcase_attr: "sortableAttributes",
SortableAttributesAnalytics analytics: SortableAttributesAnalytics
); },
{
make_setting_route!( route: "/displayed-attributes",
"/displayed-attributes", update_verb: put,
put, value_type: Vec<String>,
Vec<String>, err_type: meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::deserr::DeserrJsonError< meilisearch_types::error::deserr_codes::InvalidSettingsDisplayedAttributes,
meilisearch_types::error::deserr_codes::InvalidSettingsDisplayedAttributes, >,
>, attr: displayed_attributes,
displayed_attributes, camelcase_attr: "displayedAttributes",
"displayedAttributes", analytics: DisplayedAttributesAnalytics
DisplayedAttributesAnalytics },
); {
route: "/typo-tolerance",
make_setting_route!( update_verb: patch,
"/typo-tolerance", value_type: meilisearch_types::settings::TypoSettings,
patch, err_type: meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::settings::TypoSettings, meilisearch_types::error::deserr_codes::InvalidSettingsTypoTolerance,
meilisearch_types::deserr::DeserrJsonError< >,
meilisearch_types::error::deserr_codes::InvalidSettingsTypoTolerance, attr: typo_tolerance,
>, camelcase_attr: "typoTolerance",
typo_tolerance, analytics: TypoToleranceAnalytics
"typoTolerance", },
TypoToleranceAnalytics {
); route: "/searchable-attributes",
update_verb: put,
make_setting_route!( value_type: Vec<String>,
"/searchable-attributes", err_type: meilisearch_types::deserr::DeserrJsonError<
put, meilisearch_types::error::deserr_codes::InvalidSettingsSearchableAttributes,
Vec<String>, >,
meilisearch_types::deserr::DeserrJsonError< attr: searchable_attributes,
meilisearch_types::error::deserr_codes::InvalidSettingsSearchableAttributes, camelcase_attr: "searchableAttributes",
>, analytics: SearchableAttributesAnalytics
searchable_attributes, },
"searchableAttributes", {
SearchableAttributesAnalytics route: "/stop-words",
); update_verb: put,
value_type: std::collections::BTreeSet<String>,
make_setting_route!( err_type: meilisearch_types::deserr::DeserrJsonError<
"/stop-words", meilisearch_types::error::deserr_codes::InvalidSettingsStopWords,
put, >,
std::collections::BTreeSet<String>, attr: stop_words,
meilisearch_types::deserr::DeserrJsonError< camelcase_attr: "stopWords",
meilisearch_types::error::deserr_codes::InvalidSettingsStopWords, analytics: StopWordsAnalytics
>, },
stop_words, {
"stopWords", route: "/non-separator-tokens",
StopWordsAnalytics update_verb: put,
); value_type: std::collections::BTreeSet<String>,
err_type: meilisearch_types::deserr::DeserrJsonError<
make_setting_route!( meilisearch_types::error::deserr_codes::InvalidSettingsNonSeparatorTokens,
"/non-separator-tokens", >,
put, attr: non_separator_tokens,
std::collections::BTreeSet<String>, camelcase_attr: "nonSeparatorTokens",
meilisearch_types::deserr::DeserrJsonError< analytics: NonSeparatorTokensAnalytics
meilisearch_types::error::deserr_codes::InvalidSettingsNonSeparatorTokens, },
>, {
non_separator_tokens, route: "/separator-tokens",
"nonSeparatorTokens", update_verb: put,
NonSeparatorTokensAnalytics value_type: std::collections::BTreeSet<String>,
); err_type: meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsSeparatorTokens,
make_setting_route!( >,
"/separator-tokens", attr: separator_tokens,
put, camelcase_attr: "separatorTokens",
std::collections::BTreeSet<String>, analytics: SeparatorTokensAnalytics
meilisearch_types::deserr::DeserrJsonError< },
meilisearch_types::error::deserr_codes::InvalidSettingsSeparatorTokens, {
>, route: "/dictionary",
separator_tokens, update_verb: put,
"separatorTokens", value_type: std::collections::BTreeSet<String>,
SeparatorTokensAnalytics err_type: meilisearch_types::deserr::DeserrJsonError<
); meilisearch_types::error::deserr_codes::InvalidSettingsDictionary,
>,
make_setting_route!( attr: dictionary,
"/dictionary", camelcase_attr: "dictionary",
put, analytics: DictionaryAnalytics
std::collections::BTreeSet<String>, },
meilisearch_types::deserr::DeserrJsonError< {
meilisearch_types::error::deserr_codes::InvalidSettingsDictionary, route: "/synonyms",
>, update_verb: put,
dictionary, value_type: std::collections::BTreeMap<String, Vec<String>>,
"dictionary", err_type: meilisearch_types::deserr::DeserrJsonError<
DictionaryAnalytics meilisearch_types::error::deserr_codes::InvalidSettingsSynonyms,
); >,
attr: synonyms,
make_setting_route!( camelcase_attr: "synonyms",
"/synonyms", analytics: SynonymsAnalytics
put, },
std::collections::BTreeMap<String, Vec<String>>, {
meilisearch_types::deserr::DeserrJsonError< route: "/distinct-attribute",
meilisearch_types::error::deserr_codes::InvalidSettingsSynonyms, update_verb: put,
>, value_type: String,
synonyms, err_type: meilisearch_types::deserr::DeserrJsonError<
"synonyms", meilisearch_types::error::deserr_codes::InvalidSettingsDistinctAttribute,
SynonymsAnalytics >,
); attr: distinct_attribute,
camelcase_attr: "distinctAttribute",
make_setting_route!( analytics: DistinctAttributeAnalytics
"/distinct-attribute", },
put, {
String, route: "/proximity-precision",
meilisearch_types::deserr::DeserrJsonError< update_verb: put,
meilisearch_types::error::deserr_codes::InvalidSettingsDistinctAttribute, value_type: meilisearch_types::settings::ProximityPrecisionView,
>, err_type: meilisearch_types::deserr::DeserrJsonError<
distinct_attribute, meilisearch_types::error::deserr_codes::InvalidSettingsProximityPrecision,
"distinctAttribute", >,
DistinctAttributeAnalytics attr: proximity_precision,
); camelcase_attr: "proximityPrecision",
analytics: ProximityPrecisionAnalytics
make_setting_route!( },
"/proximity-precision", {
put, route: "/localized-attributes",
meilisearch_types::settings::ProximityPrecisionView, update_verb: put,
meilisearch_types::deserr::DeserrJsonError< value_type: Vec<meilisearch_types::locales::LocalizedAttributesRuleView>,
meilisearch_types::error::deserr_codes::InvalidSettingsProximityPrecision, err_type: meilisearch_types::deserr::DeserrJsonError<
>, meilisearch_types::error::deserr_codes::InvalidSettingsLocalizedAttributes,
proximity_precision, >,
"proximityPrecision", attr: localized_attributes,
ProximityPrecisionAnalytics camelcase_attr: "localizedAttributes",
); analytics: LocalesAnalytics
},
make_setting_route!( {
"/localized-attributes", route: "/ranking-rules",
put, update_verb: put,
Vec<meilisearch_types::locales::LocalizedAttributesRuleView>, value_type: Vec<meilisearch_types::settings::RankingRuleView>,
meilisearch_types::deserr::DeserrJsonError< err_type: meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsLocalizedAttributes, meilisearch_types::error::deserr_codes::InvalidSettingsRankingRules,
>, >,
localized_attributes, attr: ranking_rules,
"localizedAttributes", camelcase_attr: "rankingRules",
LocalesAnalytics analytics: RankingRulesAnalytics
); },
{
make_setting_route!( route: "/faceting",
"/ranking-rules", update_verb: patch,
put, value_type: meilisearch_types::settings::FacetingSettings,
Vec<meilisearch_types::settings::RankingRuleView>, err_type: meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::deserr::DeserrJsonError< meilisearch_types::error::deserr_codes::InvalidSettingsFaceting,
meilisearch_types::error::deserr_codes::InvalidSettingsRankingRules, >,
>, attr: faceting,
ranking_rules, camelcase_attr: "faceting",
"rankingRules", analytics: FacetingAnalytics
RankingRulesAnalytics },
); {
route: "/pagination",
make_setting_route!( update_verb: patch,
"/faceting", value_type: meilisearch_types::settings::PaginationSettings,
patch, err_type: meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::settings::FacetingSettings, meilisearch_types::error::deserr_codes::InvalidSettingsPagination,
meilisearch_types::deserr::DeserrJsonError< >,
meilisearch_types::error::deserr_codes::InvalidSettingsFaceting, attr: pagination,
>, camelcase_attr: "pagination",
faceting, analytics: PaginationAnalytics
"faceting", },
FacetingAnalytics {
); route: "/embedders",
update_verb: patch,
make_setting_route!( value_type: std::collections::BTreeMap<String, Setting<meilisearch_types::milli::vector::settings::EmbeddingSettings>>,
"/pagination", err_type: meilisearch_types::deserr::DeserrJsonError<
patch, meilisearch_types::error::deserr_codes::InvalidSettingsEmbedders,
meilisearch_types::settings::PaginationSettings, >,
meilisearch_types::deserr::DeserrJsonError< attr: embedders,
meilisearch_types::error::deserr_codes::InvalidSettingsPagination, camelcase_attr: "embedders",
>, analytics: EmbeddersAnalytics
pagination, },
"pagination", {
PaginationAnalytics route: "/search-cutoff-ms",
); update_verb: put,
value_type: u64,
make_setting_route!( err_type: meilisearch_types::deserr::DeserrJsonError<
"/embedders", meilisearch_types::error::deserr_codes::InvalidSettingsSearchCutoffMs,
patch, >,
std::collections::BTreeMap<String, Setting<meilisearch_types::milli::vector::settings::EmbeddingSettings>>, attr: search_cutoff_ms,
meilisearch_types::deserr::DeserrJsonError< camelcase_attr: "searchCutoffMs",
meilisearch_types::error::deserr_codes::InvalidSettingsEmbedders, analytics: SearchCutoffMsAnalytics
>, },
embedders, {
"embedders", route: "/facet-search",
EmbeddersAnalytics update_verb: put,
); value_type: bool,
err_type: meilisearch_types::deserr::DeserrJsonError<
make_setting_route!( meilisearch_types::error::deserr_codes::InvalidSettingsFacetSearch,
"/search-cutoff-ms", >,
put, attr: facet_search,
u64, camelcase_attr: "facetSearch",
meilisearch_types::deserr::DeserrJsonError< analytics: FacetSearchAnalytics
meilisearch_types::error::deserr_codes::InvalidSettingsSearchCutoffMs, },
>, {
search_cutoff_ms, route: "/prefix-search",
"searchCutoffMs", update_verb: put,
SearchCutoffMsAnalytics value_type: meilisearch_types::settings::PrefixSearchSettings,
); err_type: meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsPrefixSearch,
make_setting_route!( >,
"/facet-search", attr: prefix_search,
put, camelcase_attr: "prefixSearch",
bool, analytics: PrefixSearchAnalytics
meilisearch_types::deserr::DeserrJsonError< },
meilisearch_types::error::deserr_codes::InvalidSettingsFacetSearch,
>,
facet_search,
"facetSearch",
FacetSearchAnalytics
);
make_setting_route!(
"/prefix-search",
put,
meilisearch_types::settings::PrefixSearchSettings,
meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsPrefixSearch,
>,
prefix_search,
"prefixSearch",
PrefixSearchAnalytics
);
macro_rules! generate_configure {
($($mod:ident),*) => {
pub fn configure(cfg: &mut web::ServiceConfig) {
use crate::extractors::sequential_extractor::SeqHandler;
cfg.service(
web::resource("")
.route(web::patch().to(SeqHandler(update_all)))
.route(web::get().to(SeqHandler(get_all)))
.route(web::delete().to(SeqHandler(delete_all))))
$(.service($mod::resources()))*;
}
};
}
generate_configure!(
filterable_attributes,
sortable_attributes,
displayed_attributes,
localized_attributes,
searchable_attributes,
distinct_attribute,
proximity_precision,
stop_words,
separator_tokens,
non_separator_tokens,
dictionary,
synonyms,
ranking_rules,
typo_tolerance,
pagination,
faceting,
embedders,
search_cutoff_ms
); );
pub async fn update_all( pub async fn update_all(

View File

@ -1,44 +1,185 @@
use std::collections::HashMap; use crate::common::Server;
use once_cell::sync::Lazy;
use crate::common::{Server, Value};
use crate::json; use crate::json;
static DEFAULT_SETTINGS_VALUES: Lazy<HashMap<&'static str, Value>> = Lazy::new(|| { macro_rules! test_setting_routes {
let mut map = HashMap::new(); ($({setting: $setting:ident, update_verb: $update_verb:ident, default_value: $default_value:tt},) *) => {
map.insert("displayed_attributes", json!(["*"])); $(
map.insert("searchable_attributes", json!(["*"])); mod $setting {
map.insert("localized_attributes", json!(null)); use crate::common::Server;
map.insert("filterable_attributes", json!([]));
map.insert("distinct_attribute", json!(null)); #[actix_rt::test]
map.insert( async fn get_unexisting_index() {
"ranking_rules", let server = Server::new().await;
json!(["words", "typo", "proximity", "attribute", "sort", "exactness"]), let url = format!("/indexes/test/settings/{}",
); stringify!($setting)
map.insert("stop_words", json!([])); .chars()
map.insert("non_separator_tokens", json!([])); .map(|c| if c == '_' { '-' } else { c })
map.insert("separator_tokens", json!([])); .collect::<String>());
map.insert("dictionary", json!([])); let (_response, code) = server.service.get(url).await;
map.insert("synonyms", json!({})); assert_eq!(code, 404);
map.insert( }
"faceting",
json!({ #[actix_rt::test]
"maxValuesPerFacet": json!(100), async fn update_unexisting_index() {
"sortFacetValuesBy": { let server = Server::new().await;
"*": "alpha" let url = format!("/indexes/test/settings/{}",
stringify!($setting)
.chars()
.map(|c| if c == '_' { '-' } else { c })
.collect::<String>());
let (response, code) = server.service.$update_verb(url, serde_json::Value::Null.into()).await;
assert_eq!(code, 202, "{}", response);
server.index("").wait_task(0).await;
let (response, code) = server.index("test").get().await;
assert_eq!(code, 200, "{}", response);
}
#[actix_rt::test]
async fn delete_unexisting_index() {
let server = Server::new().await;
let url = format!("/indexes/test/settings/{}",
stringify!($setting)
.chars()
.map(|c| if c == '_' { '-' } else { c })
.collect::<String>());
let (_, code) = server.service.delete(url).await;
assert_eq!(code, 202);
let response = server.index("").wait_task(0).await;
assert_eq!(response["status"], "failed");
}
#[actix_rt::test]
async fn get_default() {
let server = Server::new().await;
let index = server.index("test");
let (response, code) = index.create(None).await;
assert_eq!(code, 202, "{}", response);
index.wait_task(0).await;
let url = format!("/indexes/test/settings/{}",
stringify!($setting)
.chars()
.map(|c| if c == '_' { '-' } else { c })
.collect::<String>());
let (response, code) = server.service.get(url).await;
assert_eq!(code, 200, "{}", response);
let expected = crate::json!($default_value);
assert_eq!(expected, response);
}
} }
}), )*
);
map.insert( #[actix_rt::test]
"pagination", async fn all_setting_tested() {
json!({ let expected = std::collections::BTreeSet::from_iter(meilisearch::routes::indexes::settings::ALL_SETTINGS_NAMES.iter());
"maxTotalHits": json!(1000), let tested = std::collections::BTreeSet::from_iter([$(stringify!($setting)),*].iter());
}), let diff: Vec<_> = expected.difference(&tested).collect();
); assert!(diff.is_empty(), "Not all settings were tested, please add the following settings to the `test_setting_routes!` macro: {:?}", diff);
map.insert("search_cutoff_ms", json!(null)); }
map };
}); }
test_setting_routes!(
{
setting: filterable_attributes,
update_verb: put,
default_value: []
},
{
setting: displayed_attributes,
update_verb: put,
default_value: ["*"]
},
{
setting: localized_attributes,
update_verb: put,
default_value: null
},
{
setting: searchable_attributes,
update_verb: put,
default_value: ["*"]
},
{
setting: distinct_attribute,
update_verb: put,
default_value: null
},
{
setting: stop_words,
update_verb: put,
default_value: []
},
{
setting: separator_tokens,
update_verb: put,
default_value: []
},
{
setting: non_separator_tokens,
update_verb: put,
default_value: []
},
{
setting: dictionary,
update_verb: put,
default_value: []
},
{
setting: ranking_rules,
update_verb: put,
default_value: ["words", "typo", "proximity", "attribute", "sort", "exactness"]
},
{
setting: synonyms,
update_verb: put,
default_value: {}
},
{
setting: pagination,
update_verb: patch,
default_value: {"maxTotalHits": 1000}
},
{
setting: faceting,
update_verb: patch,
default_value: {"maxValuesPerFacet": 100, "sortFacetValuesBy": {"*": "alpha"}}
},
{
setting: search_cutoff_ms,
update_verb: put,
default_value: null
},
{
setting: embedders,
update_verb: patch,
default_value: null
},
{
setting: facet_search,
update_verb: put,
default_value: true
},
{
setting: prefix_search,
update_verb: put,
default_value: "indexingTime"
},
{
setting: proximity_precision,
update_verb: put,
default_value: "byWord"
},
{
setting: sortable_attributes,
update_verb: put,
default_value: []
},
{
setting: typo_tolerance,
update_verb: patch,
default_value: {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": []}
},
);
#[actix_rt::test] #[actix_rt::test]
async fn get_settings_unexisting_index() { async fn get_settings_unexisting_index() {
@ -342,93 +483,6 @@ async fn error_update_setting_unexisting_index_invalid_uid() {
"###); "###);
} }
macro_rules! test_setting_routes {
($($setting:ident $write_method:ident), *) => {
$(
mod $setting {
use crate::common::Server;
use super::DEFAULT_SETTINGS_VALUES;
#[actix_rt::test]
async fn get_unexisting_index() {
let server = Server::new().await;
let url = format!("/indexes/test/settings/{}",
stringify!($setting)
.chars()
.map(|c| if c == '_' { '-' } else { c })
.collect::<String>());
let (_response, code) = server.service.get(url).await;
assert_eq!(code, 404);
}
#[actix_rt::test]
async fn update_unexisting_index() {
let server = Server::new().await;
let url = format!("/indexes/test/settings/{}",
stringify!($setting)
.chars()
.map(|c| if c == '_' { '-' } else { c })
.collect::<String>());
let (response, code) = server.service.$write_method(url, serde_json::Value::Null.into()).await;
assert_eq!(code, 202, "{}", response);
server.index("").wait_task(0).await;
let (response, code) = server.index("test").get().await;
assert_eq!(code, 200, "{}", response);
}
#[actix_rt::test]
async fn delete_unexisting_index() {
let server = Server::new().await;
let url = format!("/indexes/test/settings/{}",
stringify!($setting)
.chars()
.map(|c| if c == '_' { '-' } else { c })
.collect::<String>());
let (_, code) = server.service.delete(url).await;
assert_eq!(code, 202);
let response = server.index("").wait_task(0).await;
assert_eq!(response["status"], "failed");
}
#[actix_rt::test]
async fn get_default() {
let server = Server::new().await;
let index = server.index("test");
let (response, code) = index.create(None).await;
assert_eq!(code, 202, "{}", response);
index.wait_task(0).await;
let url = format!("/indexes/test/settings/{}",
stringify!($setting)
.chars()
.map(|c| if c == '_' { '-' } else { c })
.collect::<String>());
let (response, code) = server.service.get(url).await;
assert_eq!(code, 200, "{}", response);
let expected = DEFAULT_SETTINGS_VALUES.get(stringify!($setting)).unwrap();
assert_eq!(expected, &response);
}
}
)*
};
}
test_setting_routes!(
filterable_attributes put,
displayed_attributes put,
localized_attributes put,
searchable_attributes put,
distinct_attribute put,
stop_words put,
separator_tokens put,
non_separator_tokens put,
dictionary put,
ranking_rules put,
synonyms put,
pagination patch,
faceting patch,
search_cutoff_ms put
);
#[actix_rt::test] #[actix_rt::test]
async fn error_set_invalid_ranking_rules() { async fn error_set_invalid_ranking_rules() {
let server = Server::new().await; let server = Server::new().await;