snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-12-02 10:05:05 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Clamp the databases size to the page size

Browse Source
This commit is contained in:
Kerollmops 2022-11-29 10:38:27 +01:00
parent b948bb5191
commit e6f4a8a992
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
5 changed files with 22 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Cargo.lock generated
View File

@ -1625,7 +1625,7 @@ dependencies = [
"libc", "libc",
"lmdb-rkv-sys", "lmdb-rkv-sys",
"once_cell", "once_cell",
"page_size", "page_size 0.4.2",
"synchronoise", "synchronoise",
"url", "url",
"zerocopy", "zerocopy",
@ -1783,6 +1783,7 @@ dependencies = [
"meili-snap", "meili-snap",
"meilisearch-types", "meilisearch-types",
"nelson", "nelson",
"page_size 0.5.0",
"roaring", "roaring",
"serde", "serde",
"serde_json", "serde_json",
@ -2663,6 +2664,16 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "page_size"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b7663cbd190cfd818d08efa8497f6cd383076688c49a391ef7c0d03cd12b561"
dependencies = [
"libc",
"winapi",
]
[[package]] [[package]]
name = "parking_lot" name = "parking_lot"
version = "0.12.1" version = "0.12.1"

1
index-scheduler/Cargo.toml
View File

@ -13,6 +13,7 @@ enum-iterator = "1.1.3"
file-store = { path = "../file-store" } file-store = { path = "../file-store" }
log = "0.4.14" log = "0.4.14"
meilisearch-types = { path = "../meilisearch-types" } meilisearch-types = { path = "../meilisearch-types" }
page_size = "0.5.0"
roaring = { version = "0.10.0", features = ["serde"] } roaring = { version = "0.10.0", features = ["serde"] }
serde = { version = "1.0.136", features = ["derive"] } serde = { version = "1.0.136", features = ["derive"] }
serde_json = { version = "1.0.85", features = ["preserve_order"] } serde_json = { version = "1.0.85", features = ["preserve_order"] }

4
index-scheduler/src/index_mapper.rs
View File

@ -12,7 +12,7 @@ use meilisearch_types::milli::Index;
use uuid::Uuid; use uuid::Uuid;
use self::IndexStatus::{Available, BeingDeleted}; use self::IndexStatus::{Available, BeingDeleted};
use crate::{Error, Result}; use crate::{clamp_to_page_size, Error, Result};
const INDEX_MAPPING: &str = "index-mapping"; const INDEX_MAPPING: &str = "index-mapping";
@ -68,7 +68,7 @@ impl IndexMapper {
/// The path *must* exists or an error will be thrown. /// The path *must* exists or an error will be thrown.
fn create_or_open_index(&self, path: &Path) -> Result<Index> { fn create_or_open_index(&self, path: &Path) -> Result<Index> {
let mut options = EnvOpenOptions::new(); let mut options = EnvOpenOptions::new();
options.map_size(self.index_size); options.map_size(clamp_to_page_size(self.index_size));
options.max_readers(1024); options.max_readers(1024);
Ok(Index::new(options, path)?) Ok(Index::new(options, path)?)
} }

4
index-scheduler/src/lib.rs
View File

@ -54,7 +54,7 @@ use utils::{filter_out_references_to_newer_tasks, keep_tasks_within_datetimes, m
use uuid::Uuid; use uuid::Uuid;
use crate::index_mapper::IndexMapper; use crate::index_mapper::IndexMapper;
use crate::utils::check_index_swap_validity; use crate::utils::{check_index_swap_validity, clamp_to_page_size};
pub(crate) type BEI128 = pub(crate) type BEI128 =
meilisearch_types::heed::zerocopy::I128<meilisearch_types::heed::byteorder::BE>; meilisearch_types::heed::zerocopy::I128<meilisearch_types::heed::byteorder::BE>;
@ -361,7 +361,7 @@ impl IndexScheduler {
let env = heed::EnvOpenOptions::new() let env = heed::EnvOpenOptions::new()
.max_dbs(10) .max_dbs(10)
.map_size(options.task_db_size) .map_size(clamp_to_page_size(options.task_db_size))
.open(options.tasks_path)?; .open(options.tasks_path)?;
let file_store = FileStore::new(&options.update_file_path)?; let file_store = FileStore::new(&options.update_file_path)?;

5
index-scheduler/src/utils.rs
View File

@ -324,6 +324,11 @@ pub(crate) fn check_index_swap_validity(task: &Task) -> Result<()> {
Ok(()) Ok(())
} }
/// Clamp the provided value to be a multiple of system page size.
pub fn clamp_to_page_size(size: usize) -> usize {
size / page_size::get() * page_size::get()
}
#[cfg(test)] #[cfg(test)]
impl IndexScheduler { impl IndexScheduler {
/// Asserts that the index scheduler's content is internally consistent. /// Asserts that the index scheduler's content is internally consistent.