snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-22 18:17:39 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Rewrite the parallel merge indexing part

Browse Source
This commit is contained in:
Clément Renault 2020-10-05 20:54:06 +02:00
parent e9e03259c1
commit a2182e68a6
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
1 changed files with 14 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/bin/indexer.rs
View File

@ -794,26 +794,23 @@ fn main() -> anyhow::Result<()> {
// the readers merges potentially done on another thread.
enum DatabaseType { Main, WordDocids, WordsPairsProximitiesDocids };
let (sender, receiver) = sync_channel(3);
let main_sender = sender.clone();
let word_docids_sender = sender.clone();
let words_pairs_proximities_docids_sender = sender;
debug!("Merging the main, word docids and words pairs proximity docids in parallel...");
rayon::spawn(move || {
let result = merge_readers(main_readers, main_merge);
main_sender.send((DatabaseType::Main, result)).unwrap();
});
rayon::spawn(move || {
let result = merge_readers(word_docids_readers, word_docids_merge);
word_docids_sender.send((DatabaseType::WordDocids, result)).unwrap();
});
rayon::spawn(move || {
let result = merge_readers(
words_pairs_proximities_docids_readers,
words_pairs_proximities_docids_merge,
);
let message = (DatabaseType::WordsPairsProximitiesDocids, result);
words_pairs_proximities_docids_sender.send(message).unwrap();
vec![
(DatabaseType::Main, main_readers, main_merge as MergeFn),
(DatabaseType::WordDocids, word_docids_readers, word_docids_merge),
(
DatabaseType::WordsPairsProximitiesDocids,
words_pairs_proximities_docids_readers,
words_pairs_proximities_docids_merge,
),
]
.into_par_iter()
.for_each(|(dbtype, readers, merge)| {
let result = merge_readers(readers, merge);
sender.send((dbtype, result)).unwrap();
});
});
let mut wtxn = env.write_txn()?;