This commit is contained in:
Louis Dureuil 2024-11-18 16:23:11 +01:00
parent 0a21d9bfb3
commit 5f93651cef
No known key found for this signature in database
6 changed files with 20 additions and 33 deletions

View File

@ -1395,12 +1395,14 @@ impl IndexScheduler {
Ok(tasks) Ok(tasks)
} }
IndexOperation::DocumentEdition { mut task, .. } => { IndexOperation::DocumentEdition { mut task, .. } => {
let (filter, context, code) = let (filter, code) = if let KindWithContent::DocumentEdition {
if let KindWithContent::DocumentEdition { filter_expr,
filter_expr, context, function, .. context: _,
function,
..
} = &task.kind } = &task.kind
{ {
(filter_expr, context, function) (filter_expr, function)
} else { } else {
unreachable!() unreachable!()
}; };

View File

@ -971,6 +971,8 @@ impl IndexScheduler {
let ProcessingTasks { started_at, processing, progress, .. } = let ProcessingTasks { started_at, processing, progress, .. } =
self.processing_tasks.read().map_err(|_| Error::CorruptedTaskQueue)?.clone(); self.processing_tasks.read().map_err(|_| Error::CorruptedTaskQueue)?.clone();
let _ = progress;
let ret = tasks.into_iter(); let ret = tasks.into_iter();
if processing.is_empty() { if processing.is_empty() {
Ok((ret.collect(), total)) Ok((ret.collect(), total))

View File

@ -220,11 +220,12 @@ pub fn read_json(input: &File, output: impl io::Write) -> Result<u64> {
let mut out = BufWriter::new(output); let mut out = BufWriter::new(output);
let mut deserializer = serde_json::Deserializer::from_slice(&input); let mut deserializer = serde_json::Deserializer::from_slice(&input);
let count = match array_each(&mut deserializer, |obj: &RawValue| { let res = array_each(&mut deserializer, |obj: &RawValue| {
doc_alloc.reset(); doc_alloc.reset();
let map = RawMap::from_raw_value(obj, &doc_alloc)?; let map = RawMap::from_raw_value(obj, &doc_alloc)?;
to_writer(&mut out, &map) to_writer(&mut out, &map)
}) { });
let count = match res {
// The json data has been deserialized and does not need to be processed again. // The json data has been deserialized and does not need to be processed again.
// The data has been transferred to the writer during the deserialization process. // The data has been transferred to the writer during the deserialization process.
Ok(Ok(count)) => count, Ok(Ok(count)) => count,

View File

@ -156,6 +156,7 @@ impl FacetedDocidsExtractor {
res res
} }
#[allow(clippy::too_many_arguments)]
fn facet_fn_with_options<'extractor, 'doc>( fn facet_fn_with_options<'extractor, 'doc>(
doc_alloc: &'doc Bump, doc_alloc: &'doc Bump,
cached_sorter: &mut BalancedCaches<'extractor>, cached_sorter: &mut BalancedCaches<'extractor>,
@ -336,6 +337,7 @@ fn truncate_str(s: &str) -> &str {
} }
impl FacetedDocidsExtractor { impl FacetedDocidsExtractor {
#[allow(clippy::too_many_arguments)]
#[tracing::instrument(level = "trace", skip_all, target = "indexing::extract::faceted")] #[tracing::instrument(level = "trace", skip_all, target = "indexing::extract::faceted")]
pub fn run_extraction< pub fn run_extraction<
'pl, 'pl,

View File

@ -106,6 +106,7 @@ impl<'pl> DocumentOperation<'pl> {
} }
} }
#[allow(clippy::too_many_arguments)]
fn extract_addition_payload_changes<'r, 'pl: 'r>( fn extract_addition_payload_changes<'r, 'pl: 'r>(
indexer: &'pl Bump, indexer: &'pl Bump,
index: &Index, index: &Index,

View File

@ -1,37 +1,16 @@
use std::cell::{Ref, RefCell, RefMut}; use std::cell::{RefCell, RefMut};
pub trait RefCellExt<T: ?Sized> { pub trait RefCellExt<T: ?Sized> {
fn try_borrow_or_yield(&self) -> std::result::Result<Ref<'_, T>, std::cell::BorrowError>;
fn try_borrow_mut_or_yield( fn try_borrow_mut_or_yield(
&self, &self,
) -> std::result::Result<RefMut<'_, T>, std::cell::BorrowMutError>; ) -> std::result::Result<RefMut<'_, T>, std::cell::BorrowMutError>;
fn borrow_or_yield(&self) -> Ref<'_, T> {
self.try_borrow_or_yield().unwrap()
}
fn borrow_mut_or_yield(&self) -> RefMut<'_, T> { fn borrow_mut_or_yield(&self) -> RefMut<'_, T> {
self.try_borrow_mut_or_yield().unwrap() self.try_borrow_mut_or_yield().unwrap()
} }
} }
impl<T: ?Sized> RefCellExt<T> for RefCell<T> { impl<T: ?Sized> RefCellExt<T> for RefCell<T> {
fn try_borrow_or_yield(&self) -> std::result::Result<Ref<'_, T>, std::cell::BorrowError> {
/// TODO: move this trait and impl elsewhere
loop {
match self.try_borrow() {
Ok(borrow) => break Ok(borrow),
Err(error) => {
tracing::warn!("dynamic borrow failed, yielding to local tasks");
match rayon::yield_local() {
Some(rayon::Yield::Executed) => continue,
_ => return Err(error),
}
}
}
}
}
fn try_borrow_mut_or_yield( fn try_borrow_mut_or_yield(
&self, &self,
) -> std::result::Result<RefMut<'_, T>, std::cell::BorrowMutError> { ) -> std::result::Result<RefMut<'_, T>, std::cell::BorrowMutError> {