snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-25 19:45:05 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Introduce the try_for_each_try_init method to be used with Arced Errors

Browse Source
This commit is contained in:
Clément Renault 2024-09-29 17:21:11 +02:00
parent f3356ddaa4
commit d83c9a4074
No known key found for this signature in database
GPG Key ID: F250A4C4E3AE5F5F
5 changed files with 89 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
milli/src/update/new/extract/faceted/extract_facets.rs
View File

@ -13,6 +13,7 @@ use super::facet_document::extract_document_facets;
use super::FacetKind; use super::FacetKind;
use crate::facet::value_encoding::f64_into_bytes; use crate::facet::value_encoding::f64_into_bytes;
use crate::update::new::extract::DocidsExtractor; use crate::update::new::extract::DocidsExtractor;
use crate::update::new::items_pool::ParallelIteratorExt;
use crate::update::new::{DocumentChange, ItemsPool}; use crate::update::new::{DocumentChange, ItemsPool};
use crate::update::{create_sorter, GrenadParameters, MergeDeladdCboRoaringBitmaps}; use crate::update::{create_sorter, GrenadParameters, MergeDeladdCboRoaringBitmaps};
use crate::{ use crate::{
@ -211,7 +212,7 @@ impl DocidsExtractor for FacetedDocidsExtractor {
let context_pool = ItemsPool::new(|| { let context_pool = ItemsPool::new(|| {
Ok(( Ok((
index.read_txn()?, index.read_txn().map_err(Error::from).map_err(Arc::new)?,
fields_ids_map.clone(), fields_ids_map.clone(),
Vec::new(), Vec::new(),
CboCachedSorter::new( CboCachedSorter::new(
@ -233,19 +234,23 @@ impl DocidsExtractor for FacetedDocidsExtractor {
let span = let span =
tracing::trace_span!(target: "indexing::documents::extract", "docids_extraction"); tracing::trace_span!(target: "indexing::documents::extract", "docids_extraction");
let _entered = span.enter(); let _entered = span.enter();
document_changes.into_par_iter().try_for_each(|document_change| { document_changes.into_par_iter().try_for_each_try_init(
context_pool.with(|(rtxn, fields_ids_map, buffer, cached_sorter)| { || Ok(()),
Self::extract_document_change( |_, document_change| {
&*rtxn, context_pool.with(|(rtxn, fields_ids_map, buffer, cached_sorter)| {
index, Self::extract_document_change(
buffer, &*rtxn,
fields_ids_map, index,
&attributes_to_extract, buffer,
cached_sorter, fields_ids_map,
document_change?, &attributes_to_extract,
) cached_sorter,
}) document_change?,
})?; )
.map_err(Arc::new)
})
},
)?;
} }
{ {
let mut builder = grenad::MergerBuilder::new(MergeDeladdCboRoaringBitmaps); let mut builder = grenad::MergerBuilder::new(MergeDeladdCboRoaringBitmaps);

31
milli/src/update/new/extract/searchable/extract_word_docids.rs
View File

@ -10,6 +10,7 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator};
use super::tokenize_document::{tokenizer_builder, DocumentTokenizer}; use super::tokenize_document::{tokenizer_builder, DocumentTokenizer};
use crate::update::new::extract::cache::CboCachedSorter; use crate::update::new::extract::cache::CboCachedSorter;
use crate::update::new::extract::perm_json_p::contained_in; use crate::update::new::extract::perm_json_p::contained_in;
use crate::update::new::items_pool::ParallelIteratorExt;
use crate::update::new::{DocumentChange, ItemsPool}; use crate::update::new::{DocumentChange, ItemsPool};
use crate::update::{create_sorter, GrenadParameters, MergeDeladdCboRoaringBitmaps}; use crate::update::{create_sorter, GrenadParameters, MergeDeladdCboRoaringBitmaps};
use crate::{ use crate::{
@ -340,7 +341,7 @@ impl WordDocidsExtractors {
let context_pool = ItemsPool::new(|| { let context_pool = ItemsPool::new(|| {
Ok(( Ok((
index.read_txn()?, index.read_txn().map_err(Error::from).map_err(Arc::new)?,
&document_tokenizer, &document_tokenizer,
fields_ids_map.clone(), fields_ids_map.clone(),
WordDocidsCachedSorters::new( WordDocidsCachedSorters::new(
@ -356,18 +357,24 @@ impl WordDocidsExtractors {
let span = let span =
tracing::trace_span!(target: "indexing::documents::extract", "docids_extraction"); tracing::trace_span!(target: "indexing::documents::extract", "docids_extraction");
let _entered = span.enter(); let _entered = span.enter();
document_changes.into_par_iter().try_for_each(|document_change| { document_changes.into_par_iter().try_for_each_try_init(
context_pool.with(|(rtxn, document_tokenizer, fields_ids_map, cached_sorter)| { || Ok(()),
Self::extract_document_change( |_, document_change| {
&*rtxn, context_pool.with(
index, |(rtxn, document_tokenizer, fields_ids_map, cached_sorter)| {
document_tokenizer, Self::extract_document_change(
fields_ids_map, &*rtxn,
cached_sorter, index,
document_change?, document_tokenizer,
fields_ids_map,
cached_sorter,
document_change?,
)
.map_err(Arc::new)
},
) )
}) },
})?; )?;
} }
{ {

31
milli/src/update/new/extract/searchable/mod.rs
View File

@ -14,6 +14,7 @@ use tokenize_document::{tokenizer_builder, DocumentTokenizer};
use super::cache::CboCachedSorter; use super::cache::CboCachedSorter;
use super::DocidsExtractor; use super::DocidsExtractor;
use crate::update::new::items_pool::ParallelIteratorExt;
use crate::update::new::{DocumentChange, ItemsPool}; use crate::update::new::{DocumentChange, ItemsPool};
use crate::update::{create_sorter, GrenadParameters, MergeDeladdCboRoaringBitmaps}; use crate::update::{create_sorter, GrenadParameters, MergeDeladdCboRoaringBitmaps};
use crate::{Error, GlobalFieldsIdsMap, Index, Result, MAX_POSITION_PER_ATTRIBUTE}; use crate::{Error, GlobalFieldsIdsMap, Index, Result, MAX_POSITION_PER_ATTRIBUTE};
@ -59,7 +60,7 @@ pub trait SearchableExtractor {
let context_pool = ItemsPool::new(|| { let context_pool = ItemsPool::new(|| {
Ok(( Ok((
index.read_txn()?, index.read_txn().map_err(Error::from).map_err(Arc::new)?,
&document_tokenizer, &document_tokenizer,
fields_ids_map.clone(), fields_ids_map.clone(),
CboCachedSorter::new( CboCachedSorter::new(
@ -81,18 +82,24 @@ pub trait SearchableExtractor {
let span = let span =
tracing::trace_span!(target: "indexing::documents::extract", "docids_extraction"); tracing::trace_span!(target: "indexing::documents::extract", "docids_extraction");
let _entered = span.enter(); let _entered = span.enter();
document_changes.into_par_iter().try_for_each(|document_change| { document_changes.into_par_iter().try_for_each_try_init(
context_pool.with(|(rtxn, document_tokenizer, fields_ids_map, cached_sorter)| { || Ok(()),
Self::extract_document_change( |_, document_change| {
&*rtxn, context_pool.with(
index, |(rtxn, document_tokenizer, fields_ids_map, cached_sorter)| {
document_tokenizer, Self::extract_document_change(
fields_ids_map, &*rtxn,
cached_sorter, index,
document_change?, document_tokenizer,
fields_ids_map,
cached_sorter,
document_change?,
)
.map_err(Arc::new)
},
) )
}) },
})?; )?;
} }
{ {
let mut builder = grenad::MergerBuilder::new(MergeDeladdCboRoaringBitmaps); let mut builder = grenad::MergerBuilder::new(MergeDeladdCboRoaringBitmaps);

5
milli/src/update/new/indexer/mod.rs
View File

@ -22,6 +22,7 @@ use super::{StdResult, TopLevelMap};
use crate::documents::{PrimaryKey, DEFAULT_PRIMARY_KEY}; use crate::documents::{PrimaryKey, DEFAULT_PRIMARY_KEY};
use crate::update::new::channel::ExtractorSender; use crate::update::new::channel::ExtractorSender;
use crate::update::settings::InnerIndexSettings; use crate::update::settings::InnerIndexSettings;
use crate::update::new::items_pool::ParallelIteratorExt;
use crate::update::GrenadParameters; use crate::update::GrenadParameters;
use crate::{Error, FieldsIdsMap, GlobalFieldsIdsMap, Index, Result, UserError}; use crate::{Error, FieldsIdsMap, GlobalFieldsIdsMap, Index, Result, UserError};
@ -80,7 +81,7 @@ where
// document but we need to create a function that collects and compresses documents. // document but we need to create a function that collects and compresses documents.
let document_sender = extractor_sender.document_sender(); let document_sender = extractor_sender.document_sender();
document_changes.clone().into_par_iter().try_for_each(|result| { document_changes.clone().into_par_iter().try_for_each_try_init(|| Ok(()) as Result<_>, |_, result| {
match result? { match result? {
DocumentChange::Deletion(deletion) => { DocumentChange::Deletion(deletion) => {
let docid = deletion.docid(); let docid = deletion.docid();
@ -98,7 +99,7 @@ where
// extracted_dictionary_sender.send(self, dictionary: &[u8]); // extracted_dictionary_sender.send(self, dictionary: &[u8]);
} }
} }
Ok(()) as Result<_> Ok(()) as std::result::Result<_, Arc<_>>
})?; })?;
document_sender.finish().unwrap(); document_sender.finish().unwrap();

31
milli/src/update/new/items_pool.rs
View File

@ -1,3 +1,4 @@
use std::convert::identity;
use std::sync::Arc; use std::sync::Arc;
use crossbeam_channel::{Receiver, Sender, TryRecvError}; use crossbeam_channel::{Receiver, Sender, TryRecvError};
@ -27,12 +28,38 @@ pub trait ParallelIteratorExt: ParallelIterator {
Ok(t) => Ok(t), Ok(t) => Ok(t),
Err(err) => Err(Arc::new(err)), Err(err) => Err(Arc::new(err)),
}, },
move |maybe_t, item| match maybe_t { move |result, item| match result {
Ok(t) => map_op(t, item).map_err(Arc::new), Ok(t) => map_op(t, item).map_err(Arc::new),
Err(maybe_err) => Err(maybe_err.clone()), Err(err) => Err(err.clone()),
}, },
) )
} }
/// A method to run a closure of all the items and return an owned error.
///
/// The init function is ran only as necessary which is basically once by thread.
fn try_for_each_try_init<F, INIT, T, E>(self, init: INIT, op: F) -> Result<(), E>
where
E: Send + Sync,
F: Fn(&mut T, Self::Item) -> Result<(), Arc<E>> + Sync + Send + Clone,
INIT: Fn() -> Result<T, E> + Sync + Send + Clone,
{
let result = self.try_for_each_init(
move || match init() {
Ok(t) => Ok(t),
Err(err) => Err(Arc::new(err)),
},
move |result, item| match result {
Ok(t) => op(t, item),
Err(err) => Err(err.clone()),
},
);
match result {
Ok(()) => Ok(()),
Err(err) => Err(Arc::into_inner(err).expect("the error must be only owned by us")),
}
}
} }
impl<T: ParallelIterator> ParallelIteratorExt for T {} impl<T: ParallelIterator> ParallelIteratorExt for T {}