snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-26 12:05:05 +08:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Merge #3204

Browse Source 
3204: Bring back v0.30.1 changes to `main` r=curquiza a=curquiza

I was not able to just import `release-v0.30.1` to `main`, see:
<img width="1371" alt="Capture d’écran 2022-12-06 à 20 03 50" src="https://user-images.githubusercontent.com/20380692/206000844-b39b3063-7da2-475f-b3e4-1791c39a7c2f.png">

So I cherry-picked the commits.

⚠️ ⚠️ ⚠️ I had a git conflict here

<img width="730" alt="Capture d’écran 2022-12-06 à 20 09 04" src="https://user-images.githubusercontent.com/20380692/206001007-f56bc28f-c0b1-46a0-bb60-cce4e73b9584.png">


⚠️ ⚠️ ⚠️ Check out carefully how I fixed it


Co-authored-by: curquiza <curquiza@users.noreply.github.com>
Co-authored-by: Kerollmops <clement@meilisearch.com>
This commit is contained in:
bors[bot] 2022-12-07 11:08:37 +00:00 committed by GitHub
commit 2ed6017603
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 50 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
Cargo.lock generated
View File

@ -1101,7 +1101,7 @@ dependencies = [
[[package]] [[package]]
name = "dump" name = "dump"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"big_s", "big_s",
@ -1310,7 +1310,7 @@ dependencies = [
[[package]] [[package]]
name = "file-store" name = "file-store"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"faux", "faux",
"tempfile", "tempfile",
@ -1332,8 +1332,8 @@ dependencies = [
[[package]] [[package]]
name = "filter-parser" name = "filter-parser"
version = "0.37.0" version = "0.37.1"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e"
dependencies = [ dependencies = [
"nom", "nom",
"nom_locate", "nom_locate",
@ -1351,8 +1351,8 @@ dependencies = [
[[package]] [[package]]
name = "flatten-serde-json" name = "flatten-serde-json"
version = "0.37.0" version = "0.37.1"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e"
dependencies = [ dependencies = [
"serde_json", "serde_json",
] ]
@ -1625,7 +1625,7 @@ dependencies = [
"libc", "libc",
"lmdb-rkv-sys", "lmdb-rkv-sys",
"once_cell", "once_cell",
"page_size", "page_size 0.4.2",
"synchronoise", "synchronoise",
"url", "url",
"zerocopy", "zerocopy",
@ -1767,7 +1767,7 @@ dependencies = [
[[package]] [[package]]
name = "index-scheduler" name = "index-scheduler"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"big_s", "big_s",
@ -1783,6 +1783,7 @@ dependencies = [
"meili-snap", "meili-snap",
"meilisearch-types", "meilisearch-types",
"nelson", "nelson",
"page_size 0.5.0",
"roaring", "roaring",
"serde", "serde",
"serde_json", "serde_json",
@ -1897,8 +1898,8 @@ dependencies = [
[[package]] [[package]]
name = "json-depth-checker" name = "json-depth-checker"
version = "0.37.0" version = "0.37.1"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e"
dependencies = [ dependencies = [
"serde_json", "serde_json",
] ]
@ -2257,7 +2258,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]] [[package]]
name = "meili-snap" name = "meili-snap"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"insta", "insta",
"md5", "md5",
@ -2266,7 +2267,7 @@ dependencies = [
[[package]] [[package]]
name = "meilisearch-auth" name = "meilisearch-auth"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"enum-iterator", "enum-iterator",
"hmac", "hmac",
@ -2283,7 +2284,7 @@ dependencies = [
[[package]] [[package]]
name = "meilisearch-http" name = "meilisearch-http"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"actix-cors", "actix-cors",
"actix-http", "actix-http",
@ -2366,7 +2367,7 @@ dependencies = [
[[package]] [[package]]
name = "meilisearch-types" name = "meilisearch-types"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"actix-web", "actix-web",
"anyhow", "anyhow",
@ -2416,8 +2417,8 @@ dependencies = [
[[package]] [[package]]
name = "milli" name = "milli"
version = "0.37.0" version = "0.37.1"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e"
dependencies = [ dependencies = [
"bimap", "bimap",
"bincode", "bincode",
@ -2663,6 +2664,16 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "page_size"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b7663cbd190cfd818d08efa8497f6cd383076688c49a391ef7c0d03cd12b561"
dependencies = [
"libc",
"winapi",
]
[[package]] [[package]]
name = "parking_lot" name = "parking_lot"
version = "0.12.1" version = "0.12.1"
@ -2747,7 +2758,7 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
[[package]] [[package]]
name = "permissive-json-pointer" name = "permissive-json-pointer"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"big_s", "big_s",
"serde_json", "serde_json",

2
dump/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "dump" name = "dump"
version = "0.30.0" version = "0.30.1"
edition = "2021" edition = "2021"
[dependencies] [dependencies]

2
dump/src/writer.rs
View File

@ -295,7 +295,7 @@ pub(crate) mod test {
insta::assert_json_snapshot!(metadata, { ".dumpDate" => "[date]" }, @r###" insta::assert_json_snapshot!(metadata, { ".dumpDate" => "[date]" }, @r###"
{ {
"dumpVersion": "V6", "dumpVersion": "V6",
"dbVersion": "0.30.0", "dbVersion": "0.30.1",
"dumpDate": "[date]" "dumpDate": "[date]"
} }
"###); "###);

2
file-store/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "file-store" name = "file-store"
version = "0.30.0" version = "0.30.1"
edition = "2021" edition = "2021"
[dependencies] [dependencies]

3
index-scheduler/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "index-scheduler" name = "index-scheduler"
version = "0.30.0" version = "0.30.1"
edition = "2021" edition = "2021"
[dependencies] [dependencies]
@ -13,6 +13,7 @@ enum-iterator = "1.1.3"
file-store = { path = "../file-store" } file-store = { path = "../file-store" }
log = "0.4.14" log = "0.4.14"
meilisearch-types = { path = "../meilisearch-types" } meilisearch-types = { path = "../meilisearch-types" }
page_size = "0.5.0"
roaring = { version = "0.10.0", features = ["serde"] } roaring = { version = "0.10.0", features = ["serde"] }
serde = { version = "1.0.136", features = ["derive"] } serde = { version = "1.0.136", features = ["derive"] }
serde_json = { version = "1.0.85", features = ["preserve_order"] } serde_json = { version = "1.0.85", features = ["preserve_order"] }

4
index-scheduler/src/index_mapper.rs
View File

@ -13,7 +13,7 @@ use uuid::Uuid;
use self::IndexStatus::{Available, BeingDeleted}; use self::IndexStatus::{Available, BeingDeleted};
use crate::uuid_codec::UuidCodec; use crate::uuid_codec::UuidCodec;
use crate::{Error, Result}; use crate::{clamp_to_page_size, Error, Result};
const INDEX_MAPPING: &str = "index-mapping"; const INDEX_MAPPING: &str = "index-mapping";
@ -68,7 +68,7 @@ impl IndexMapper {
/// The path *must* exists or an error will be thrown. /// The path *must* exists or an error will be thrown.
fn create_or_open_index(&self, path: &Path) -> Result<Index> { fn create_or_open_index(&self, path: &Path) -> Result<Index> {
let mut options = EnvOpenOptions::new(); let mut options = EnvOpenOptions::new();
options.map_size(self.index_size); options.map_size(clamp_to_page_size(self.index_size));
options.max_readers(1024); options.max_readers(1024);
Ok(Index::new(options, path)?) Ok(Index::new(options, path)?)
} }

8
index-scheduler/src/lib.rs
View File

@ -55,7 +55,7 @@ use utils::{filter_out_references_to_newer_tasks, keep_tasks_within_datetimes, m
use uuid::Uuid; use uuid::Uuid;
use crate::index_mapper::IndexMapper; use crate::index_mapper::IndexMapper;
use crate::utils::check_index_swap_validity; use crate::utils::{check_index_swap_validity, clamp_to_page_size};
pub(crate) type BEI128 = pub(crate) type BEI128 =
meilisearch_types::heed::zerocopy::I128<meilisearch_types::heed::byteorder::BE>; meilisearch_types::heed::zerocopy::I128<meilisearch_types::heed::byteorder::BE>;
@ -362,7 +362,7 @@ impl IndexScheduler {
let env = heed::EnvOpenOptions::new() let env = heed::EnvOpenOptions::new()
.max_dbs(10) .max_dbs(10)
.map_size(options.task_db_size) .map_size(clamp_to_page_size(options.task_db_size))
.open(options.tasks_path)?; .open(options.tasks_path)?;
let file_store = FileStore::new(&options.update_file_path)?; let file_store = FileStore::new(&options.update_file_path)?;
@ -1112,8 +1112,8 @@ mod tests {
indexes_path: tempdir.path().join("indexes"), indexes_path: tempdir.path().join("indexes"),
snapshots_path: tempdir.path().join("snapshots"), snapshots_path: tempdir.path().join("snapshots"),
dumps_path: tempdir.path().join("dumps"), dumps_path: tempdir.path().join("dumps"),
task_db_size: 1024 * 1024, // 1 MiB task_db_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose.
index_size: 1024 * 1024, // 1 MiB index_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose.
indexer_config: IndexerConfig::default(), indexer_config: IndexerConfig::default(),
autobatching_enabled, autobatching_enabled,
}; };

5
index-scheduler/src/utils.rs
View File

@ -324,6 +324,11 @@ pub(crate) fn check_index_swap_validity(task: &Task) -> Result<()> {
Ok(()) Ok(())
} }
/// Clamp the provided value to be a multiple of system page size.
pub fn clamp_to_page_size(size: usize) -> usize {
size / page_size::get() * page_size::get()
}
#[cfg(test)] #[cfg(test)]
impl IndexScheduler { impl IndexScheduler {
/// Asserts that the index scheduler's content is internally consistent. /// Asserts that the index scheduler's content is internally consistent.

2
meili-snap/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "meili-snap" name = "meili-snap"
version = "0.30.0" version = "0.30.1"
edition = "2021" edition = "2021"
[dependencies] [dependencies]

2
meilisearch-auth/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "meilisearch-auth" name = "meilisearch-auth"
version = "0.30.0" version = "0.30.1"
edition = "2021" edition = "2021"
[dependencies] [dependencies]

2
meilisearch-http/Cargo.toml
View File

@ -4,7 +4,7 @@ description = "Meilisearch HTTP server"
edition = "2021" edition = "2021"
license = "MIT" license = "MIT"
name = "meilisearch-http" name = "meilisearch-http"
version = "0.30.0" version = "0.30.1"
[[bin]] [[bin]]
name = "meilisearch" name = "meilisearch"

4
meilisearch-types/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "meilisearch-types" name = "meilisearch-types"
version = "0.30.0" version = "0.30.1"
authors = ["marin <postma.marin@protonmail.com>"] authors = ["marin <postma.marin@protonmail.com>"]
edition = "2021" edition = "2021"
@ -12,7 +12,7 @@ either = { version = "1.6.1", features = ["serde"] }
enum-iterator = "1.1.3" enum-iterator = "1.1.3"
flate2 = "1.0.24" flate2 = "1.0.24"
fst = "0.4.7" fst = "0.4.7"
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.37.0", default-features = false } milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.37.1", default-features = false }
proptest = { version = "1.0.0", optional = true } proptest = { version = "1.0.0", optional = true }
proptest-derive = { version = "0.3.0", optional = true } proptest-derive = { version = "0.3.0", optional = true }
roaring = { version = "0.10.0", features = ["serde"] } roaring = { version = "0.10.0", features = ["serde"] }

2
permissive-json-pointer/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "permissive-json-pointer" name = "permissive-json-pointer"
version = "0.30.0" version = "0.30.1"
edition = "2021" edition = "2021"
description = "A permissive json pointer" description = "A permissive json pointer"
readme = "README.md" readme = "README.md"