snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-01-18 17:11:15 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge #2546

Browse Source 
2546: Bump milli to 0.31.1 r=irevoire a=Kerollmops



Co-authored-by: Kerollmops <clement@meilisearch.com>
This commit is contained in:
bors[bot] 2022-06-23 11:07:20 +00:00 committed by GitHub
commit 90b0a4e99f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 61 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Cargo.lock generated
View File

@ -1123,8 +1123,8 @@ dependencies = [
[[package]] [[package]]
name = "filter-parser" name = "filter-parser"
version = "0.29.3" version = "0.31.1"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.29.3#f1d848bb9add86b9414d110a083dfa0462d5d636" source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8"
dependencies = [ dependencies = [
"nom", "nom",
"nom_locate", "nom_locate",
@ -1148,8 +1148,8 @@ dependencies = [
[[package]] [[package]]
name = "flatten-serde-json" name = "flatten-serde-json"
version = "0.29.3" version = "0.31.1"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.29.3#f1d848bb9add86b9414d110a083dfa0462d5d636" source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8"
dependencies = [ dependencies = [
"serde_json", "serde_json",
] ]
@ -1661,8 +1661,8 @@ dependencies = [
[[package]] [[package]]
name = "json-depth-checker" name = "json-depth-checker"
version = "0.29.3" version = "0.31.1"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.29.3#f1d848bb9add86b9414d110a083dfa0462d5d636" source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8"
dependencies = [ dependencies = [
"serde_json", "serde_json",
] ]
@ -2189,8 +2189,8 @@ dependencies = [
[[package]] [[package]]
name = "milli" name = "milli"
version = "0.29.3" version = "0.31.1"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.29.3#f1d848bb9add86b9414d110a083dfa0462d5d636" source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8"
dependencies = [ dependencies = [
"bimap", "bimap",
"bincode", "bincode",

2
meilisearch-auth/Cargo.toml
View File

@ -8,7 +8,7 @@ base64 = "0.13.0"
enum-iterator = "0.7.0" enum-iterator = "0.7.0"
hmac = "0.12.1" hmac = "0.12.1"
meilisearch-types = { path = "../meilisearch-types" } meilisearch-types = { path = "../meilisearch-types" }
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.29.3" } milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.31.1" }
rand = "0.8.4" rand = "0.8.4"
serde = { version = "1.0.136", features = ["derive"] } serde = { version = "1.0.136", features = ["derive"] }
serde_json = { version = "1.0.79", features = ["preserve_order"] } serde_json = { version = "1.0.79", features = ["preserve_order"] }

6
meilisearch-http/src/routes/indexes/settings.rs
View File

@ -318,7 +318,7 @@ make_setting_route!(
"Pagination Updated".to_string(), "Pagination Updated".to_string(),
json!({ json!({
"pagination": { "pagination": {
"limited_to": setting.as_ref().and_then(|s| s.limited_to.set()), "max_total_hits": setting.as_ref().and_then(|s| s.max_total_hits.set()),
}, },
}), }),
Some(req), Some(req),
@ -418,10 +418,10 @@ pub async fn update_all(
.and_then(|s| s.max_values_per_facet.as_ref().set()), .and_then(|s| s.max_values_per_facet.as_ref().set()),
}, },
"pagination": { "pagination": {
"limited_to": settings.pagination "max_total_hits": settings.pagination
.as_ref() .as_ref()
.set() .set()
.and_then(|s| s.limited_to.as_ref().set()), .and_then(|s| s.max_total_hits.as_ref().set()),
}, },
}), }),
Some(&req), Some(&req),

18
meilisearch-http/tests/dumps/mod.rs
View File

@ -61,7 +61,7 @@ async fn import_dump_v2_movie_raw() {
assert_eq!(code, 200); assert_eq!(code, 200);
assert_eq!( assert_eq!(
settings, settings,
json!({"displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) json!({"displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } })
); );
let (tasks, code) = index.list_tasks().await; let (tasks, code) = index.list_tasks().await;
@ -125,7 +125,7 @@ async fn import_dump_v2_movie_with_settings() {
assert_eq!(code, 200); assert_eq!(code, 200);
assert_eq!( assert_eq!(
settings, settings,
json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } })
); );
let (tasks, code) = index.list_tasks().await; let (tasks, code) = index.list_tasks().await;
@ -189,7 +189,7 @@ async fn import_dump_v2_rubygems_with_settings() {
assert_eq!(code, 200); assert_eq!(code, 200);
assert_eq!( assert_eq!(
settings, settings,
json!({"displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 }}) json!({"displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 }})
); );
let (tasks, code) = index.list_tasks().await; let (tasks, code) = index.list_tasks().await;
@ -253,7 +253,7 @@ async fn import_dump_v3_movie_raw() {
assert_eq!(code, 200); assert_eq!(code, 200);
assert_eq!( assert_eq!(
settings, settings,
json!({"displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) json!({"displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } })
); );
let (tasks, code) = index.list_tasks().await; let (tasks, code) = index.list_tasks().await;
@ -317,7 +317,7 @@ async fn import_dump_v3_movie_with_settings() {
assert_eq!(code, 200); assert_eq!(code, 200);
assert_eq!( assert_eq!(
settings, settings,
json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } })
); );
let (tasks, code) = index.list_tasks().await; let (tasks, code) = index.list_tasks().await;
@ -381,7 +381,7 @@ async fn import_dump_v3_rubygems_with_settings() {
assert_eq!(code, 200); assert_eq!(code, 200);
assert_eq!( assert_eq!(
settings, settings,
json!({"displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) json!({"displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } })
); );
let (tasks, code) = index.list_tasks().await; let (tasks, code) = index.list_tasks().await;
@ -445,7 +445,7 @@ async fn import_dump_v4_movie_raw() {
assert_eq!(code, 200); assert_eq!(code, 200);
assert_eq!( assert_eq!(
settings, settings,
json!({ "displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) json!({ "displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } })
); );
let (tasks, code) = index.list_tasks().await; let (tasks, code) = index.list_tasks().await;
@ -509,7 +509,7 @@ async fn import_dump_v4_movie_with_settings() {
assert_eq!(code, 200); assert_eq!(code, 200);
assert_eq!( assert_eq!(
settings, settings,
json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } })
); );
let (tasks, code) = index.list_tasks().await; let (tasks, code) = index.list_tasks().await;
@ -573,7 +573,7 @@ async fn import_dump_v4_rubygems_with_settings() {
assert_eq!(code, 200); assert_eq!(code, 200);
assert_eq!( assert_eq!(
settings, settings,
json!({ "displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) json!({ "displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } })
); );
let (tasks, code) = index.list_tasks().await; let (tasks, code) = index.list_tasks().await;

4
meilisearch-http/tests/search/mod.rs
View File

@ -567,7 +567,7 @@ async fn placeholder_search_is_hard_limited() {
.await; .await;
index index
.update_settings(json!({ "pagination": { "limitedTo": 10_000 } })) .update_settings(json!({ "pagination": { "maxTotalHits": 10_000 } }))
.await; .await;
index.wait_task(1).await; index.wait_task(1).await;
@ -636,7 +636,7 @@ async fn search_is_hard_limited() {
.await; .await;
index index
.update_settings(json!({ "pagination": { "limitedTo": 10_000 } })) .update_settings(json!({ "pagination": { "maxTotalHits": 10_000 } }))
.await; .await;
index.wait_task(1).await; index.wait_task(1).await;

4
meilisearch-http/tests/settings/get_settings.rs
View File

@ -33,7 +33,7 @@ static DEFAULT_SETTINGS_VALUES: Lazy<HashMap<&'static str, Value>> = Lazy::new(|
map.insert( map.insert(
"pagination", "pagination",
json!({ json!({
"limitedTo": json!(1000), "maxTotalHits": json!(1000),
}), }),
); );
map map
@ -82,7 +82,7 @@ async fn get_settings() {
assert_eq!( assert_eq!(
settings["pagination"], settings["pagination"],
json!({ json!({
"limitedTo": 1000, "maxTotalHits": 1000,
}) })
); );
} }

4
meilisearch-lib/Cargo.toml
View File

@ -3,8 +3,6 @@ name = "meilisearch-lib"
version = "0.28.0" version = "0.28.0"
edition = "2021" edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
actix-web = { version = "4.0.1", default-features = false } actix-web = { version = "4.0.1", default-features = false }
anyhow = { version = "1.0.56", features = ["backtrace"] } anyhow = { version = "1.0.56", features = ["backtrace"] }
@ -30,7 +28,7 @@ lazy_static = "1.4.0"
log = "0.4.14" log = "0.4.14"
meilisearch-auth = { path = "../meilisearch-auth" } meilisearch-auth = { path = "../meilisearch-auth" }
meilisearch-types = { path = "../meilisearch-types" } meilisearch-types = { path = "../meilisearch-types" }
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.29.3" } milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.31.1" }
mime = "0.3.16" mime = "0.3.16"
num_cpus = "1.13.1" num_cpus = "1.13.1"
obkv = "0.2.0" obkv = "0.2.0"

2
meilisearch-lib/src/index/dump.rs
View File

@ -27,7 +27,7 @@ const DATA_FILE_NAME: &str = "documents.jsonl";
impl Index { impl Index {
pub fn dump(&self, path: impl AsRef<Path>) -> Result<()> { pub fn dump(&self, path: impl AsRef<Path>) -> Result<()> {
// acquire write txn make sure any ongoing write is finished before we start. // acquire write txn make sure any ongoing write is finished before we start.
let txn = self.env.write_txn()?; let txn = self.write_txn()?;
let path = path.as_ref().join(format!("indexes/{}", self.uuid)); let path = path.as_ref().join(format!("indexes/{}", self.uuid));
create_dir_all(&path)?; create_dir_all(&path)?;

25
meilisearch-lib/src/index/index.rs
View File

@ -4,9 +4,10 @@ use std::marker::PhantomData;
use std::ops::Deref; use std::ops::Deref;
use std::path::Path; use std::path::Path;
use std::sync::Arc; use std::sync::Arc;
use walkdir::WalkDir;
use fst::IntoStreamer; use fst::IntoStreamer;
use milli::heed::{EnvOpenOptions, RoTxn}; use milli::heed::{CompactionOption, EnvOpenOptions, RoTxn};
use milli::update::{IndexerConfig, Setting}; use milli::update::{IndexerConfig, Setting};
use milli::{obkv_to_json, FieldDistribution, DEFAULT_VALUES_PER_FACET}; use milli::{obkv_to_json, FieldDistribution, DEFAULT_VALUES_PER_FACET};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -14,8 +15,7 @@ use serde_json::{Map, Value};
use time::OffsetDateTime; use time::OffsetDateTime;
use uuid::Uuid; use uuid::Uuid;
use crate::index::search::DEFAULT_PAGINATION_LIMITED_TO; use crate::index::search::DEFAULT_PAGINATION_MAX_TOTAL_HITS;
use crate::EnvSizer;
use super::error::IndexError; use super::error::IndexError;
use super::error::Result; use super::error::Result;
@ -202,9 +202,9 @@ impl Index {
}; };
let pagination = PaginationSettings { let pagination = PaginationSettings {
limited_to: Setting::Set( max_total_hits: Setting::Set(
self.pagination_limited_to(txn)? self.pagination_max_total_hits(txn)?
.unwrap_or(DEFAULT_PAGINATION_LIMITED_TO), .unwrap_or(DEFAULT_PAGINATION_MAX_TOTAL_HITS),
), ),
}; };
@ -245,7 +245,7 @@ impl Index {
let fields_ids_map = self.fields_ids_map(&txn)?; let fields_ids_map = self.fields_ids_map(&txn)?;
let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect(); let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); let iter = self.all_documents(&txn)?.skip(offset).take(limit);
let mut documents = Vec::new(); let mut documents = Vec::new();
@ -302,7 +302,12 @@ impl Index {
} }
pub fn size(&self) -> u64 { pub fn size(&self) -> u64 {
self.env.size() WalkDir::new(self.inner.path())
.into_iter()
.filter_map(|entry| entry.ok())
.filter_map(|entry| entry.metadata().ok())
.filter(|metadata| metadata.is_file())
.fold(0, |acc, m| acc + m.len())
} }
pub fn snapshot(&self, path: impl AsRef<Path>) -> Result<()> { pub fn snapshot(&self, path: impl AsRef<Path>) -> Result<()> {
@ -310,9 +315,7 @@ impl Index {
create_dir_all(&dst)?; create_dir_all(&dst)?;
dst.push("data.mdb"); dst.push("data.mdb");
let _txn = self.write_txn()?; let _txn = self.write_txn()?;
self.inner self.inner.copy_to_path(dst, CompactionOption::Enabled)?;
.env
.copy_to_path(dst, milli::heed::CompactionOption::Enabled)?;
Ok(()) Ok(())
} }
} }

12
meilisearch-lib/src/index/search.rs
View File

@ -29,7 +29,7 @@ pub const DEFAULT_HIGHLIGHT_POST_TAG: fn() -> String = || "</em>".to_string();
/// The maximimum number of results that the engine /// The maximimum number of results that the engine
/// will be able to return in one search call. /// will be able to return in one search call.
pub const DEFAULT_PAGINATION_LIMITED_TO: usize = 1000; pub const DEFAULT_PAGINATION_MAX_TOTAL_HITS: usize = 1000;
#[derive(Deserialize, Debug, Clone, PartialEq)] #[derive(Deserialize, Debug, Clone, PartialEq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)] #[serde(rename_all = "camelCase", deny_unknown_fields)]
@ -91,14 +91,14 @@ impl Index {
search.query(query); search.query(query);
} }
let pagination_limited_to = self let max_total_hits = self
.pagination_limited_to(&rtxn)? .pagination_max_total_hits(&rtxn)?
.unwrap_or(DEFAULT_PAGINATION_LIMITED_TO); .unwrap_or(DEFAULT_PAGINATION_MAX_TOTAL_HITS);
// Make sure that a user can't get more documents than the hard limit, // Make sure that a user can't get more documents than the hard limit,
// we align that on the offset too. // we align that on the offset too.
let offset = min(query.offset.unwrap_or(0), pagination_limited_to); let offset = min(query.offset.unwrap_or(0), max_total_hits);
let limit = min(query.limit, pagination_limited_to.saturating_sub(offset)); let limit = min(query.limit, max_total_hits.saturating_sub(offset));
search.offset(offset); search.offset(offset);
search.limit(limit); search.limit(limit);

10
meilisearch-lib/src/index/updates.rs
View File

@ -86,7 +86,7 @@ pub struct FacetingSettings {
pub struct PaginationSettings { pub struct PaginationSettings {
#[cfg_attr(test, proptest(strategy = "test::setting_strategy()"))] #[cfg_attr(test, proptest(strategy = "test::setting_strategy()"))]
#[serde(default, skip_serializing_if = "Setting::is_not_set")] #[serde(default, skip_serializing_if = "Setting::is_not_set")]
pub limited_to: Setting<usize>, pub max_total_hits: Setting<usize>,
} }
/// Holds all the settings for an index. `T` can either be `Checked` if they represents settings /// Holds all the settings for an index. `T` can either be `Checked` if they represents settings
@ -474,12 +474,12 @@ pub fn apply_settings_to_builder(
} }
match settings.pagination { match settings.pagination {
Setting::Set(ref value) => match value.limited_to { Setting::Set(ref value) => match value.max_total_hits {
Setting::Set(val) => builder.set_pagination_limited_to(val), Setting::Set(val) => builder.set_pagination_max_total_hits(val),
Setting::Reset => builder.reset_pagination_limited_to(), Setting::Reset => builder.reset_pagination_max_total_hits(),
Setting::NotSet => (), Setting::NotSet => (),
}, },
Setting::Reset => builder.reset_pagination_limited_to(), Setting::Reset => builder.reset_pagination_max_total_hits(),
Setting::NotSet => (), Setting::NotSet => (),
} }
} }

9
meilisearch-lib/src/index_resolver/meta_store.rs
View File

@ -3,6 +3,7 @@ use std::fs::{create_dir_all, File};
use std::io::{BufRead, BufReader, Write}; use std::io::{BufRead, BufReader, Write};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::Arc; use std::sync::Arc;
use walkdir::WalkDir;
use milli::heed::types::{SerdeBincode, Str}; use milli::heed::types::{SerdeBincode, Str};
use milli::heed::{CompactionOption, Database, Env}; use milli::heed::{CompactionOption, Database, Env};
@ -11,7 +12,6 @@ use uuid::Uuid;
use super::error::{IndexResolverError, Result}; use super::error::{IndexResolverError, Result};
use crate::tasks::task::TaskId; use crate::tasks::task::TaskId;
use crate::EnvSizer;
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize)]
pub struct DumpEntry { pub struct DumpEntry {
@ -131,7 +131,12 @@ impl HeedMetaStore {
} }
fn get_size(&self) -> Result<u64> { fn get_size(&self) -> Result<u64> {
Ok(self.env.size()) Ok(WalkDir::new(self.env.path())
.into_iter()
.filter_map(|entry| entry.ok())
.filter_map(|entry| entry.metadata().ok())
.filter(|metadata| metadata.is_file())
.fold(0, |acc, m| acc + m.len()))
} }
pub fn dump(&self, path: PathBuf) -> Result<()> { pub fn dump(&self, path: PathBuf) -> Result<()> {

17
meilisearch-lib/src/lib.rs
View File

@ -20,23 +20,6 @@ pub use milli::heed;
mod compression; mod compression;
pub mod document_formats; pub mod document_formats;
use walkdir::WalkDir;
pub trait EnvSizer {
fn size(&self) -> u64;
}
impl EnvSizer for milli::heed::Env {
fn size(&self) -> u64 {
WalkDir::new(self.path())
.into_iter()
.filter_map(|entry| entry.ok())
.filter_map(|entry| entry.metadata().ok())
.filter(|metadata| metadata.is_file())
.fold(0, |acc, m| acc + m.len())
}
}
/// Check if a db is empty. It does not provide any information on the /// Check if a db is empty. It does not provide any information on the
/// validity of the data in it. /// validity of the data in it.
/// We consider a database as non empty when it's a non empty directory. /// We consider a database as non empty when it's a non empty directory.

5
meilisearch-lib/src/snapshot.rs
View File

@ -7,6 +7,7 @@ use anyhow::bail;
use fs_extra::dir::{self, CopyOptions}; use fs_extra::dir::{self, CopyOptions};
use log::{info, trace}; use log::{info, trace};
use meilisearch_auth::open_auth_store_env; use meilisearch_auth::open_auth_store_env;
use milli::heed::CompactionOption;
use tokio::sync::RwLock; use tokio::sync::RwLock;
use tokio::time::sleep; use tokio::time::sleep;
use walkdir::WalkDir; use walkdir::WalkDir;
@ -181,9 +182,7 @@ impl SnapshotJob {
let mut options = milli::heed::EnvOpenOptions::new(); let mut options = milli::heed::EnvOpenOptions::new();
options.map_size(self.index_size); options.map_size(self.index_size);
let index = milli::Index::new(options, entry.path())?; let index = milli::Index::new(options, entry.path())?;
index index.copy_to_path(dst, CompactionOption::Enabled)?;
.env
.copy_to_path(dst, milli::heed::CompactionOption::Enabled)?;
} }
Ok(()) Ok(())