snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-22 01:57:41 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Use the new rayon-par-bridge library

Browse Source
This commit is contained in:
Clément Renault 2024-05-12 14:45:25 +02:00
parent b67d385cf0
commit 81ec0abad1
No known key found for this signature in database
GPG Key ID: F250A4C4E3AE5F5F
3 changed files with 14 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Cargo.lock generated
View File

@ -3510,6 +3510,7 @@ dependencies = [
"rand",
"rand_pcg",
"rayon",
"rayon-par-bridge",
"rhai",
"roaring",
"rstar",
@ -4289,6 +4290,15 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "rayon-par-bridge"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb6a14d8f65834aca6b0fe4cbbd7a27e639cd3efb1f2a32de9942368f1991de8"
dependencies = [
"rayon",
]
[[package]]
name = "reborrow"
version = "0.5.5"

1
milli/Cargo.toml
View File

@ -84,6 +84,7 @@ rand = "0.8.5"
tracing = "0.1.40"
ureq = { version = "2.10.0", features = ["json"] }
url = "2.5.2"
rayon-par-bridge = "0.1.0"
[dev-dependencies]
mimalloc = { version = "0.1.43", default-features = false }

11
milli/src/update/index_documents/mod.rs
View File

@ -44,7 +44,7 @@ use crate::update::{
IndexerConfig, UpdateIndexingStep, WordPrefixDocids, WordPrefixIntegerDocids, WordsPrefixesFst,
};
use crate::vector::EmbeddingConfigs;
use crate::{CboRoaringBitmapCodec, FieldsIdsMap, Index, Object, Result};
use crate::{CboRoaringBitmapCodec, Index, Object, Result};
static MERGED_DATABASE_COUNT: usize = 7;
static PREFIX_DATABASE_COUNT: usize = 4;
@ -262,11 +262,8 @@ where
Ok(DocumentEdition::Nothing) as Result<_>
});
std::thread::scope(|s| {
let (send, recv) = std::sync::mpsc::sync_channel(100);
s.spawn(move || processing.for_each(|el| drop(send.send(el))));
for result in recv {
rayon_par_bridge::par_bridge(100, processing, |iterator| {
for result in iterator {
if (self.should_abort)() {
return Err(Error::InternalError(InternalError::AbortedIndexation));
}
@ -285,8 +282,6 @@ where
Ok(())
})?;
drop(immutable_obkvs);
let file = documents_batch_builder.into_inner()?;
let reader = DocumentsBatchReader::from_reader(file)?;