Experiments with Quentin

This commit is contained in:
Clément Renault 2024-07-15 16:20:02 +02:00
parent b64b4ab6ca
commit f82ab3cc50
No known key found for this signature in database
GPG Key ID: F250A4C4E3AE5F5F
3 changed files with 2 additions and 4 deletions

1
Cargo.lock generated
View File

@ -2191,7 +2191,6 @@ dependencies = [
"bytemuck", "bytemuck",
"byteorder", "byteorder",
"rayon", "rayon",
"tempfile",
] ]
[[package]] [[package]]

View File

@ -27,8 +27,7 @@ fst = "0.4.7"
fxhash = "0.2.1" fxhash = "0.2.1"
geoutils = "0.5.1" geoutils = "0.5.1"
grenad = { version = "0.4.6", default-features = false, features = [ grenad = { version = "0.4.6", default-features = false, features = [
"rayon", "rayon"
"tempfile",
] } ] }
heed = { version = "0.20.1", default-features = false, features = [ heed = { version = "0.20.1", default-features = false, features = [
"serde-json", "serde-json",

View File

@ -325,7 +325,7 @@ where
let documents_chunk_size = match self.indexer_config.documents_chunk_size { let documents_chunk_size = match self.indexer_config.documents_chunk_size {
Some(chunk_size) => chunk_size, Some(chunk_size) => chunk_size,
None => { None => {
let default_chunk_size = 1024 * 1024 * 4; // 4MiB let default_chunk_size = 1024 * 1024 * 1024 * 2; // 2 GiB
let min_chunk_size = 1024 * 512; // 512KiB let min_chunk_size = 1024 * 512; // 512KiB
// compute the chunk size from the number of available threads and the inputed data size. // compute the chunk size from the number of available threads and the inputed data size.