snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-25 19:45:05 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

improve the way we access the mutex

Browse Source
This commit is contained in:
Tamo 2024-11-19 19:40:31 +01:00
parent 4abcd9c04e
commit b906e3ed70
No known key found for this signature in database
GPG Key ID: 20CD8020AFA88D69
34 changed files with 155 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crates/index-scheduler/src/error.rs
View File

@ -230,7 +230,7 @@ impl ErrorCode for Error {
Error::InvalidTaskCanceledBy { .. } => Code::InvalidTaskCanceledBy, Error::InvalidTaskCanceledBy { .. } => Code::InvalidTaskCanceledBy,
Error::InvalidIndexUid { .. } => Code::InvalidIndexUid, Error::InvalidIndexUid { .. } => Code::InvalidIndexUid,
Error::TaskNotFound(_) => Code::TaskNotFound, Error::TaskNotFound(_) => Code::TaskNotFound,
Error::BatchNotFound(_) => Code::TaskNotFound, Error::BatchNotFound(_) => Code::BatchNotFound,
Error::TaskDeletionWithEmptyQuery => Code::MissingTaskFilters, Error::TaskDeletionWithEmptyQuery => Code::MissingTaskFilters,
Error::TaskCancelationWithEmptyQuery => Code::MissingTaskFilters, Error::TaskCancelationWithEmptyQuery => Code::MissingTaskFilters,
// TODO: not sure of the Code to use // TODO: not sure of the Code to use

81
crates/index-scheduler/src/lib.rs
View File

@ -163,7 +163,7 @@ impl Query {
} }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
struct ProcessingTasks { pub struct ProcessingTasks {
batch: Option<ProcessingBatch>, batch: Option<ProcessingBatch>,
/// The list of tasks ids that are currently running. /// The list of tasks ids that are currently running.
processing: RoaringBitmap, processing: RoaringBitmap,
@ -948,6 +948,7 @@ impl IndexScheduler {
processing: &ProcessingTasks, processing: &ProcessingTasks,
query: &Query, query: &Query,
) -> Result<RoaringBitmap> { ) -> Result<RoaringBitmap> {
dbg!();
let mut batches = self.all_batch_ids(rtxn)?; let mut batches = self.all_batch_ids(rtxn)?;
if let Some(batch_id) = processing.batch.as_ref().map(|batch| batch.uid) { if let Some(batch_id) = processing.batch.as_ref().map(|batch| batch.uid) {
batches.insert(batch_id); batches.insert(batch_id);
@ -1235,22 +1236,21 @@ impl IndexScheduler {
/// 1. IndexSwap tasks are not publicly associated with any index, but they are associated /// 1. IndexSwap tasks are not publicly associated with any index, but they are associated
/// with many indexes internally. /// with many indexes internally.
/// 2. The user may not have the rights to access the tasks (internally) associated with all indexes. /// 2. The user may not have the rights to access the tasks (internally) associated with all indexes.
pub fn get_batch_ids_from_authorized_indexes( fn get_batch_ids_from_authorized_indexes(
&self, &self,
rtxn: &RoTxn, rtxn: &RoTxn,
processing: &ProcessingTasks,
query: &Query, query: &Query,
filters: &meilisearch_auth::AuthFilter, filters: &meilisearch_auth::AuthFilter,
) -> Result<(RoaringBitmap, u64)> { ) -> Result<(RoaringBitmap, u64)> {
let processing = self.processing_tasks.read().unwrap().clone();
// compute all batches matching the filter by ignoring the limits, to find the number of batches matching // compute all batches matching the filter by ignoring the limits, to find the number of batches matching
// the filter. // the filter.
// As this causes us to compute the filter twice it is slightly inefficient, but doing it this way spares // As this causes us to compute the filter twice it is slightly inefficient, but doing it this way spares
// us from modifying the underlying implementation, and the performance remains sufficient. // us from modifying the underlying implementation, and the performance remains sufficient.
// Should this change, we would modify `get_batch_ids` to directly return the number of matching batches. // Should this change, we would modify `get_batch_ids` to directly return the number of matching batches.
let total_batches = let total_batches =
self.get_batch_ids(rtxn, &processing, &query.clone().without_limits())?; self.get_batch_ids(rtxn, processing, &query.clone().without_limits())?;
let mut batches = self.get_batch_ids(rtxn, &processing, query)?; let mut batches = self.get_batch_ids(rtxn, processing, query)?;
// If the query contains a list of index uid or there is a finite list of authorized indexes, // If the query contains a list of index uid or there is a finite list of authorized indexes,
// then we must exclude all the batches that only contains tasks associated to multiple indexes. // then we must exclude all the batches that only contains tasks associated to multiple indexes.
@ -1369,11 +1369,14 @@ impl IndexScheduler {
filters: &meilisearch_auth::AuthFilter, filters: &meilisearch_auth::AuthFilter,
) -> Result<(Vec<Batch>, u64)> { ) -> Result<(Vec<Batch>, u64)> {
let rtxn = self.env.read_txn()?; let rtxn = self.env.read_txn()?;
let processing = self.processing_tasks.read().unwrap().clone();
let (batches, total) = let (batches, total) =
self.get_batch_ids_from_authorized_indexes(&rtxn, &query, filters)?; self.get_batch_ids_from_authorized_indexes(&rtxn, &processing, &query, filters)?;
let batches = self.get_existing_batches( let batches = self.get_existing_batches(
&rtxn, &rtxn,
&processing,
batches.into_iter().rev().take(query.limit.unwrap_or(u32::MAX) as usize), batches.into_iter().rev().take(query.limit.unwrap_or(u32::MAX) as usize),
)?; )?;
@ -4202,46 +4205,47 @@ mod tests {
handle.advance_n_successful_batches(3); handle.advance_n_successful_batches(3);
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "processed_all_tasks"); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "processed_all_tasks");
let proc = index_scheduler.processing_tasks.read().unwrap().clone();
let rtxn = index_scheduler.env.read_txn().unwrap(); let rtxn = index_scheduler.env.read_txn().unwrap();
let query = Query { limit: Some(0), ..Default::default() }; let query = Query { limit: Some(0), ..Default::default() };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
snapshot!(snapshot_bitmap(&batches), @"[]"); snapshot!(snapshot_bitmap(&batches), @"[]");
let query = Query { limit: Some(1), ..Default::default() }; let query = Query { limit: Some(1), ..Default::default() };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
snapshot!(snapshot_bitmap(&batches), @"[2,]"); snapshot!(snapshot_bitmap(&batches), @"[2,]");
let query = Query { limit: Some(2), ..Default::default() }; let query = Query { limit: Some(2), ..Default::default() };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
snapshot!(snapshot_bitmap(&batches), @"[1,2,]"); snapshot!(snapshot_bitmap(&batches), @"[1,2,]");
let query = Query { from: Some(1), ..Default::default() }; let query = Query { from: Some(1), ..Default::default() };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
snapshot!(snapshot_bitmap(&batches), @"[0,1,]"); snapshot!(snapshot_bitmap(&batches), @"[0,1,]");
let query = Query { from: Some(2), ..Default::default() }; let query = Query { from: Some(2), ..Default::default() };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
snapshot!(snapshot_bitmap(&batches), @"[0,1,2,]"); snapshot!(snapshot_bitmap(&batches), @"[0,1,2,]");
let query = Query { from: Some(1), limit: Some(1), ..Default::default() }; let query = Query { from: Some(1), limit: Some(1), ..Default::default() };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
snapshot!(snapshot_bitmap(&batches), @"[1,]"); snapshot!(snapshot_bitmap(&batches), @"[1,]");
let query = Query { from: Some(1), limit: Some(2), ..Default::default() }; let query = Query { from: Some(1), limit: Some(2), ..Default::default() };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
snapshot!(snapshot_bitmap(&batches), @"[0,1,]"); snapshot!(snapshot_bitmap(&batches), @"[0,1,]");
} }
@ -4265,16 +4269,17 @@ mod tests {
handle.advance_till([Start, BatchCreated]); handle.advance_till([Start, BatchCreated]);
let rtxn = index_scheduler.env.read_txn().unwrap(); let rtxn = index_scheduler.env.read_txn().unwrap();
let proc = index_scheduler.processing_tasks.read().unwrap().clone();
let query = Query { statuses: Some(vec![Status::Processing]), ..Default::default() }; let query = Query { statuses: Some(vec![Status::Processing]), ..Default::default() };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
snapshot!(snapshot_bitmap(&batches), @"[0,]"); // only the processing batch in the first tick snapshot!(snapshot_bitmap(&batches), @"[0,]"); // only the processing batch in the first tick
let query = Query { statuses: Some(vec![Status::Enqueued]), ..Default::default() }; let query = Query { statuses: Some(vec![Status::Enqueued]), ..Default::default() };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
snapshot!(snapshot_bitmap(&batches), @"[]"); // The batches don't contains any enqueued tasks snapshot!(snapshot_bitmap(&batches), @"[]"); // The batches don't contains any enqueued tasks
@ -4283,7 +4288,7 @@ mod tests {
..Default::default() ..Default::default()
}; };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
snapshot!(snapshot_bitmap(&batches), @"[0,]"); // both enqueued and processing tasks in the first tick snapshot!(snapshot_bitmap(&batches), @"[0,]"); // both enqueued and processing tasks in the first tick
@ -4293,7 +4298,7 @@ mod tests {
..Default::default() ..Default::default()
}; };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// both enqueued and processing tasks in the first tick, but limited to those with a started_at // both enqueued and processing tasks in the first tick, but limited to those with a started_at
// that comes after the start of the test, which should excludes the enqueued tasks // that comes after the start of the test, which should excludes the enqueued tasks
@ -4305,7 +4310,7 @@ mod tests {
..Default::default() ..Default::default()
}; };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// both enqueued and processing tasks in the first tick, but limited to those with a started_at // both enqueued and processing tasks in the first tick, but limited to those with a started_at
// that comes before the start of the test, which should excludes all of them // that comes before the start of the test, which should excludes all of them
@ -4318,7 +4323,7 @@ mod tests {
..Default::default() ..Default::default()
}; };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// both enqueued and processing tasks in the first tick, but limited to those with a started_at // both enqueued and processing tasks in the first tick, but limited to those with a started_at
// that comes after the start of the test and before one minute after the start of the test, // that comes after the start of the test and before one minute after the start of the test,
@ -4333,8 +4338,10 @@ mod tests {
Start, Start,
BatchCreated, BatchCreated,
]); ]);
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after-advancing-a-bit");
let rtxn = index_scheduler.env.read_txn().unwrap(); let rtxn = index_scheduler.env.read_txn().unwrap();
let proc = index_scheduler.processing_tasks.read().unwrap().clone();
let second_start_time = OffsetDateTime::now_utc(); let second_start_time = OffsetDateTime::now_utc();
@ -4345,7 +4352,7 @@ mod tests {
..Default::default() ..Default::default()
}; };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// both succeeded and processing tasks in the first tick, but limited to those with a started_at // both succeeded and processing tasks in the first tick, but limited to those with a started_at
// that comes after the start of the test and before one minute after the start of the test, // that comes after the start of the test and before one minute after the start of the test,
@ -4358,7 +4365,7 @@ mod tests {
..Default::default() ..Default::default()
}; };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// both succeeded and processing tasks in the first tick, but limited to those with a started_at // both succeeded and processing tasks in the first tick, but limited to those with a started_at
// that comes before the start of the test, which should exclude all tasks // that comes before the start of the test, which should exclude all tasks
@ -4371,7 +4378,7 @@ mod tests {
..Default::default() ..Default::default()
}; };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// both succeeded and processing tasks in the first tick, but limited to those with a started_at // both succeeded and processing tasks in the first tick, but limited to those with a started_at
// that comes after the start of the second part of the test and before one minute after the // that comes after the start of the second part of the test and before one minute after the
@ -4389,9 +4396,10 @@ mod tests {
]); ]);
let rtxn = index_scheduler.env.read_txn().unwrap(); let rtxn = index_scheduler.env.read_txn().unwrap();
let proc = index_scheduler.processing_tasks.read().unwrap().clone();
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// we run the same query to verify that, and indeed find that the last task is matched // we run the same query to verify that, and indeed find that the last task is matched
snapshot!(snapshot_bitmap(&batches), @"[2,]"); snapshot!(snapshot_bitmap(&batches), @"[2,]");
@ -4403,7 +4411,7 @@ mod tests {
..Default::default() ..Default::default()
}; };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// enqueued, succeeded, or processing tasks started after the second part of the test, should // enqueued, succeeded, or processing tasks started after the second part of the test, should
// again only return the last task // again only return the last task
@ -4411,11 +4419,12 @@ mod tests {
handle.advance_till([ProcessBatchFailed, AfterProcessing]); handle.advance_till([ProcessBatchFailed, AfterProcessing]);
let rtxn = index_scheduler.read_txn().unwrap(); let rtxn = index_scheduler.read_txn().unwrap();
let proc = index_scheduler.processing_tasks.read().unwrap().clone();
// now the last task should have failed // now the last task should have failed
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "end"); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "end");
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// so running the last query should return nothing // so running the last query should return nothing
snapshot!(snapshot_bitmap(&batches), @"[]"); snapshot!(snapshot_bitmap(&batches), @"[]");
@ -4427,7 +4436,7 @@ mod tests {
..Default::default() ..Default::default()
}; };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// but the same query on failed tasks should return the last task // but the same query on failed tasks should return the last task
snapshot!(snapshot_bitmap(&batches), @"[2,]"); snapshot!(snapshot_bitmap(&batches), @"[2,]");
@ -4439,7 +4448,7 @@ mod tests {
..Default::default() ..Default::default()
}; };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// but the same query on failed tasks should return the last task // but the same query on failed tasks should return the last task
snapshot!(snapshot_bitmap(&batches), @"[2,]"); snapshot!(snapshot_bitmap(&batches), @"[2,]");
@ -4452,7 +4461,7 @@ mod tests {
..Default::default() ..Default::default()
}; };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// same query but with an invalid uid // same query but with an invalid uid
snapshot!(snapshot_bitmap(&batches), @"[]"); snapshot!(snapshot_bitmap(&batches), @"[]");
@ -4465,7 +4474,7 @@ mod tests {
..Default::default() ..Default::default()
}; };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// same query but with a valid uid // same query but with a valid uid
snapshot!(snapshot_bitmap(&batches), @"[2,]"); snapshot!(snapshot_bitmap(&batches), @"[2,]");
@ -4494,10 +4503,11 @@ mod tests {
handle.advance_till([Start, BatchCreated]); handle.advance_till([Start, BatchCreated]);
let rtxn = index_scheduler.env.read_txn().unwrap(); let rtxn = index_scheduler.env.read_txn().unwrap();
let proc = index_scheduler.processing_tasks.read().unwrap().clone();
let query = Query { index_uids: Some(vec!["catto".to_owned()]), ..Default::default() }; let query = Query { index_uids: Some(vec!["catto".to_owned()]), ..Default::default() };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// only the first task associated with catto is returned, the indexSwap tasks are excluded! // only the first task associated with catto is returned, the indexSwap tasks are excluded!
snapshot!(snapshot_bitmap(&batches), @"[0,]"); snapshot!(snapshot_bitmap(&batches), @"[0,]");
@ -4506,6 +4516,7 @@ mod tests {
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes( .get_batch_ids_from_authorized_indexes(
&rtxn, &rtxn,
&proc,
&query, &query,
&AuthFilter::with_allowed_indexes( &AuthFilter::with_allowed_indexes(
vec![IndexUidPattern::new_unchecked("doggo")].into_iter().collect(), vec![IndexUidPattern::new_unchecked("doggo")].into_iter().collect(),
@ -4533,6 +4544,7 @@ mod tests {
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes( .get_batch_ids_from_authorized_indexes(
&rtxn, &rtxn,
&proc,
&query, &query,
&AuthFilter::with_allowed_indexes( &AuthFilter::with_allowed_indexes(
vec![IndexUidPattern::new_unchecked("doggo")].into_iter().collect(), vec![IndexUidPattern::new_unchecked("doggo")].into_iter().collect(),
@ -4547,6 +4559,7 @@ mod tests {
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes( .get_batch_ids_from_authorized_indexes(
&rtxn, &rtxn,
&proc,
&query, &query,
&AuthFilter::with_allowed_indexes( &AuthFilter::with_allowed_indexes(
vec![ vec![
@ -4564,7 +4577,7 @@ mod tests {
let query = Query::default(); let query = Query::default();
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// we asked for all the tasks with all index authorized -> all tasks returned // we asked for all the tasks with all index authorized -> all tasks returned
snapshot!(snapshot_bitmap(&batches), @"[0,1,2,3,]"); snapshot!(snapshot_bitmap(&batches), @"[0,1,2,3,]");
@ -4595,9 +4608,10 @@ mod tests {
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "start"); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "start");
let rtxn = index_scheduler.read_txn().unwrap(); let rtxn = index_scheduler.read_txn().unwrap();
let proc = index_scheduler.processing_tasks.read().unwrap().clone();
let query = Query { canceled_by: Some(vec![task_cancelation.uid]), ..Query::default() }; let query = Query { canceled_by: Some(vec![task_cancelation.uid]), ..Query::default() };
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap(); .unwrap();
// The batch zero was the index creation task, the 1 is the task cancellation // The batch zero was the index creation task, the 1 is the task cancellation
snapshot!(snapshot_bitmap(&batches), @"[1,]"); snapshot!(snapshot_bitmap(&batches), @"[1,]");
@ -4606,6 +4620,7 @@ mod tests {
let (batches, _) = index_scheduler let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes( .get_batch_ids_from_authorized_indexes(
&rtxn, &rtxn,
&proc,
&query, &query,
&AuthFilter::with_allowed_indexes( &AuthFilter::with_allowed_indexes(
vec![IndexUidPattern::new_unchecked("doggo")].into_iter().collect(), vec![IndexUidPattern::new_unchecked("doggo")].into_iter().collect(),

2
crates/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_failing_the_deletion.snap
View File

@ -38,7 +38,7 @@ doggos [0,1,]
[timestamp] [0,] [timestamp] [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentDeletion":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"providedIds":2,"deletedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentDeletion":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_last_successful_addition.snap
View File

@ -42,7 +42,7 @@ doggos: { number_of_documents: 3, field_distribution: {"catto": 1, "doggo": 2, "
[timestamp] [1,] [timestamp] [1,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentDeletion":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"providedIds":2,"deletedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentDeletion":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":3,"indexedDocuments":3}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":3,"indexedDocuments":3}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:

2
crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/document_addition_failed.snap
View File

@ -35,7 +35,7 @@ doggos [0,]
[timestamp] [0,] [timestamp] [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_removing_the_documents.snap
View File

@ -55,7 +55,7 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
### All Batches: ### All Batches:
0 {uid: 0, details: {"filterableAttributes":["catto"]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"settingsUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"filterableAttributes":["catto"]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"settingsUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":3,"indexedDocuments":3}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":3,"indexedDocuments":3}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {"providedIds":1,"deletedDocuments":2,"originalFilter":"\"catto EXISTS\""}, stats: {"totalNbTasks":4,"status":{"succeeded":2,"failed":2},"types":{"documentDeletion":4},"indexUids":{"doggos":4}}, } 2 {uid: 2, details: {"providedIds":1,"deletedDocuments":2,"originalFilter":"true&\"id = 2\"&\"catto EXISTS\""}, stats: {"totalNbTasks":4,"status":{"succeeded":2,"failed":2},"types":{"documentDeletion":4},"indexUids":{"doggos":4}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap
View File

@ -35,7 +35,7 @@ catto [0,]
[timestamp] [0,] [timestamp] [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, } 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap
View File

@ -35,7 +35,7 @@ catto [0,]
[timestamp] [0,] [timestamp] [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, } 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/query_batches_simple/end.snap
View File

@ -50,7 +50,7 @@ doggo: { number_of_documents: 0, field_distribution: {} }
### All Batches: ### All Batches:
0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, } 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, }
1 {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, } 1 {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"whalo":1}}, } 2 {uid: 2, details: {"primaryKey":"fish"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"whalo":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

4
crates/index-scheduler/src/snapshots/lib.rs/query_batches_special_rules/after-processing-everything.snap
View File

@ -55,8 +55,8 @@ doggo: { number_of_documents: 0, field_distribution: {} }
### All Batches: ### All Batches:
0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, } 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, }
1 {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, } 1 {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, } 2 {uid: 2, details: {"swaps":[{"indexes":["catto","doggo"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, }
3 {uid: 3, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, } 3 {uid: 3, details: {"swaps":[{"indexes":["catto","whalo"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/query_tasks_simple/end.snap
View File

@ -50,7 +50,7 @@ doggo: { number_of_documents: 0, field_distribution: {} }
### All Batches: ### All Batches:
0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, } 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, }
1 {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, } 1 {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"whalo":1}}, } 2 {uid: 2, details: {"primaryKey":"fish"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"whalo":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/swap_indexes_errors/first_swap_failed.snap
View File

@ -66,7 +66,7 @@ d: { number_of_documents: 0, field_distribution: {} }
1 {uid: 1, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"b":1}}, } 1 {uid: 1, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"b":1}}, }
2 {uid: 2, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"c":1}}, } 2 {uid: 2, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"c":1}}, }
3 {uid: 3, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"d":1}}, } 3 {uid: 3, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"d":1}}, }
4 {uid: 4, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, } 4 {uid: 4, details: {"swaps":[{"indexes":["a","b"]},{"indexes":["c","e"]},{"indexes":["d","f"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index/after_processing_the_10_tasks.snap
View File

@ -53,7 +53,7 @@ doggos [0,1,2,3,4,5,6,7,8,9,]
[timestamp] [0,1,2,3,4,5,6,7,8,9,] [timestamp] [0,1,2,3,4,5,6,7,8,9,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":10,"status":{"failed":10},"types":{"documentAdditionOrUpdate":10},"indexUids":{"doggos":10}}, } 0 {uid: 0, details: {"receivedDocuments":10,"indexedDocuments":0}, stats: {"totalNbTasks":10,"status":{"failed":10},"types":{"documentAdditionOrUpdate":10},"indexUids":{"doggos":10}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,1,2,3,4,5,6,7,8,9,] 0 [0,1,2,3,4,5,6,7,8,9,]

20
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/all_tasks_processed.snap
View File

@ -71,16 +71,16 @@ doggos [0,1,2,3,4,5,6,7,8,9,]
[timestamp] [9,] [timestamp] [9,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
3 {uid: 3, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 3 {uid: 3, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
4 {uid: 4, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 4 {uid: 4, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
5 {uid: 5, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 5 {uid: 5, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
6 {uid: 6, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 6 {uid: 6, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
7 {uid: 7, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 7 {uid: 7, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
8 {uid: 8, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 8 {uid: 8, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
9 {uid: 9, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 9 {uid: 9, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

10
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/five_tasks_processed.snap
View File

@ -61,11 +61,11 @@ doggos [0,1,2,3,4,5,6,7,8,9,]
[timestamp] [4,] [timestamp] [4,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
3 {uid: 3, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 3 {uid: 3, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
4 {uid: 4, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 4 {uid: 4, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/all_tasks_processed.snap
View File

@ -57,7 +57,7 @@ doggos: { number_of_documents: 9, field_distribution: {"doggo": 9, "id": 9} }
[timestamp] [1,2,3,4,5,6,7,8,9,] [timestamp] [1,2,3,4,5,6,7,8,9,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":9,"indexedDocuments":9}, stats: {"totalNbTasks":9,"status":{"succeeded":9},"types":{"documentAdditionOrUpdate":9},"indexUids":{"doggos":9}}, } 1 {uid: 1, details: {"receivedDocuments":9,"indexedDocuments":9}, stats: {"totalNbTasks":9,"status":{"succeeded":9},"types":{"documentAdditionOrUpdate":9},"indexUids":{"doggos":9}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/only_first_task_failed.snap
View File

@ -53,7 +53,7 @@ doggos [0,1,2,3,4,5,6,7,8,9,]
[timestamp] [0,] [timestamp] [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

4
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/fifth_task_succeeds.snap
View File

@ -51,9 +51,9 @@ doggos: { number_of_documents: 2, field_distribution: {"doggo": 2, "id": 2} }
[timestamp] [4,] [timestamp] [4,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, } 0 {uid: 0, details: {"receivedDocuments":2,"indexedDocuments":0}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
3 {uid: 3, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 3 {uid: 3, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/first_and_second_task_fails.snap
View File

@ -44,7 +44,7 @@ doggos: { number_of_documents: 0, field_distribution: {} }
[timestamp] [0,1,] [timestamp] [0,1,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, } 0 {uid: 0, details: {"receivedDocuments":2,"indexedDocuments":0}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,1,] 0 [0,1,]

4
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/fourth_task_fails.snap
View File

@ -49,9 +49,9 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
[timestamp] [3,] [timestamp] [3,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, } 0 {uid: 0, details: {"receivedDocuments":2,"indexedDocuments":0}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,1,] 0 [0,1,]

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/third_task_succeeds.snap
View File

@ -47,7 +47,7 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
[timestamp] [2,] [timestamp] [2,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, } 0 {uid: 0, details: {"receivedDocuments":2,"indexedDocuments":0}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/second_task_fails.snap
View File

@ -44,7 +44,7 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

4
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/third_task_fails.snap
View File

@ -46,8 +46,8 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/second_and_third_tasks_fails.snap
View File

@ -44,7 +44,7 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

4
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/all_other_tasks_succeeds.snap
View File

@ -53,8 +53,8 @@ doggos: { number_of_documents: 4, field_distribution: {"doggo": 4, "paw": 4} }
[timestamp] [3,4,5,] [timestamp] [3,4,5,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
3 {uid: 3, details: {"receivedDocuments":3,"indexedDocuments":3}, stats: {"totalNbTasks":3,"status":{"succeeded":3},"types":{"documentAdditionOrUpdate":3},"indexUids":{"doggos":3}}, } 3 {uid: 3, details: {"receivedDocuments":3,"indexedDocuments":3}, stats: {"totalNbTasks":3,"status":{"succeeded":3},"types":{"documentAdditionOrUpdate":3},"indexUids":{"doggos":3}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/first_task_fails.snap
View File

@ -46,7 +46,7 @@ doggos: { number_of_documents: 0, field_distribution: {} }
[timestamp] [0,] [timestamp] [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

4
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/second_task_fails.snap
View File

@ -48,8 +48,8 @@ doggos: { number_of_documents: 0, field_distribution: {} }
[timestamp] [1,] [timestamp] [1,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

4
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/third_task_succeeds.snap
View File

@ -51,8 +51,8 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "paw": 1} }
[timestamp] [2,] [timestamp] [2,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/all_other_tasks_succeeds.snap
View File

@ -54,7 +54,7 @@ doggos: { number_of_documents: 5, field_distribution: {"doggo": 5, "doggoid": 5}
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
3 {uid: 3, details: {"receivedDocuments":3,"indexedDocuments":3}, stats: {"totalNbTasks":3,"status":{"succeeded":3},"types":{"documentAdditionOrUpdate":3},"indexUids":{"doggos":3}}, } 3 {uid: 3, details: {"receivedDocuments":3,"indexedDocuments":3}, stats: {"totalNbTasks":3,"status":{"succeeded":3},"types":{"documentAdditionOrUpdate":3},"indexUids":{"doggos":3}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/second_task_fails.snap
View File

@ -50,7 +50,7 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "doggoid": 1}
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/third_task_succeeds.snap
View File

@ -52,7 +52,7 @@ doggos: { number_of_documents: 2, field_distribution: {"doggo": 2, "doggoid": 2}
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, } 2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:

19
crates/index-scheduler/src/utils.rs
View File

@ -12,7 +12,7 @@ use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status
use roaring::{MultiOps, RoaringBitmap}; use roaring::{MultiOps, RoaringBitmap};
use time::OffsetDateTime; use time::OffsetDateTime;
use crate::{Error, IndexScheduler, Result, Task, TaskId, BEI128}; use crate::{Error, IndexScheduler, ProcessingTasks, Result, Task, TaskId, BEI128};
/// This structure contains all the information required to write a batch in the database without reading the tasks. /// This structure contains all the information required to write a batch in the database without reading the tasks.
/// It'll stay in RAM so it must be small. /// It'll stay in RAM so it must be small.
@ -106,11 +106,9 @@ impl ProcessingBatch {
self.statuses.insert(task.status); self.statuses.insert(task.status);
// Craft an aggregation of the details of all the tasks encountered in this batch. // Craft an aggregation of the details of all the tasks encountered in this batch.
if task.status != Status::Failed {
if let Some(ref details) = task.details { if let Some(ref details) = task.details {
self.details.accumulate(&DetailsView::from(details.clone())); self.details.accumulate(&DetailsView::from(details.clone()));
} }
}
self.stats.total_nb_tasks += 1; self.stats.total_nb_tasks += 1;
*self.stats.status.entry(task.status).or_default() += 1; *self.stats.status.entry(task.status).or_default() += 1;
*self.stats.types.entry(task.kind.as_kind()).or_default() += 1; *self.stats.types.entry(task.kind.as_kind()).or_default() += 1;
@ -118,6 +116,16 @@ impl ProcessingBatch {
*self.stats.index_uids.entry(index_uid.to_string()).or_default() += 1; *self.stats.index_uids.entry(index_uid.to_string()).or_default() += 1;
} }
} }
pub fn to_batch(&self) -> Batch {
Batch {
uid: self.uid,
details: self.details.clone(),
stats: self.stats.clone(),
started_at: self.started_at,
finished_at: self.finished_at,
}
}
} }
impl IndexScheduler { impl IndexScheduler {
@ -243,13 +251,18 @@ impl IndexScheduler {
pub(crate) fn get_existing_batches( pub(crate) fn get_existing_batches(
&self, &self,
rtxn: &RoTxn, rtxn: &RoTxn,
processing: &ProcessingTasks,
tasks: impl IntoIterator<Item = BatchId>, tasks: impl IntoIterator<Item = BatchId>,
) -> Result<Vec<Batch>> { ) -> Result<Vec<Batch>> {
tasks tasks
.into_iter() .into_iter()
.map(|batch_id| { .map(|batch_id| {
if Some(batch_id) == processing.batch.as_ref().map(|batch| batch.uid) {
Ok(processing.batch.as_ref().unwrap().to_batch())
} else {
self.get_batch(rtxn, batch_id) self.get_batch(rtxn, batch_id)
.and_then(|task| task.ok_or(Error::CorruptedTaskQueue)) .and_then(|task| task.ok_or(Error::CorruptedTaskQueue))
}
}) })
.collect::<Result<_>>() .collect::<Result<_>>()
} }

2
crates/meilisearch/src/routes/batches.rs
View File

@ -42,7 +42,7 @@ async fn get_batch(
let task_view = BatchView::from_batch(batch); let task_view = BatchView::from_batch(batch);
Ok(HttpResponse::Ok().json(task_view)) Ok(HttpResponse::Ok().json(task_view))
} else { } else {
Err(index_scheduler::Error::TaskNotFound(batch_uid).into()) Err(index_scheduler::Error::BatchNotFound(batch_uid).into())
} }
} }

100
crates/meilisearch/tests/batches/mod.rs
View File

@ -2,8 +2,6 @@ mod errors;
use meili_snap::insta::assert_json_snapshot; use meili_snap::insta::assert_json_snapshot;
use meili_snap::snapshot; use meili_snap::snapshot;
use time::format_description::well_known::Rfc3339;
use time::OffsetDateTime;
use crate::common::Server; use crate::common::Server;
use crate::json; use crate::json;
@ -17,7 +15,7 @@ async fn error_get_unexisting_batch_status() {
let (response, code) = index.get_batch(1).await; let (response, code) = index.get_batch(1).await;
let expected_response = json!({ let expected_response = json!({
"message": "batch `1` not found.", "message": "Batch `1` not found.",
"code": "batch_not_found", "code": "batch_not_found",
"type": "invalid_request", "type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#batch_not_found" "link": "https://docs.meilisearch.com/errors#batch_not_found"
@ -32,19 +30,9 @@ async fn get_batch_status() {
let server = Server::new().await; let server = Server::new().await;
let index = server.index("test"); let index = server.index("test");
index.create(None).await; index.create(None).await;
index
.add_documents(
json!([{
"id": 1,
"content": "foobar",
}]),
None,
)
.await;
index.wait_task(0).await; index.wait_task(0).await;
let (_response, code) = index.get_batch(1).await; let (_response, code) = index.get_batch(0).await;
assert_eq!(code, 200); assert_eq!(code, 200);
// TODO check response format, as per #48
} }
#[actix_rt::test] #[actix_rt::test]
@ -241,49 +229,6 @@ async fn get_batch_filter_error() {
"#); "#);
} }
macro_rules! assert_valid_summarized_batch {
($response:expr, $batch_type:literal, $index:literal) => {{
assert_eq!($response.as_object().unwrap().len(), 5);
assert!($response["batchUid"].as_u64().is_some());
assert_eq!($response["indexUid"], $index);
assert_eq!($response["status"], "enqueued");
assert_eq!($response["type"], $batch_type);
let date = $response["enqueuedAt"].as_str().expect("missing date");
OffsetDateTime::parse(date, &Rfc3339).unwrap();
}};
}
#[actix_web::test]
async fn test_summarized_batch_view() {
let server = Server::new().await;
let index = server.index("test");
let (response, _) = index.create(None).await;
assert_valid_summarized_batch!(response, "indexCreation", "test");
let (response, _) = index.update(None).await;
assert_valid_summarized_batch!(response, "indexUpdate", "test");
let (response, _) = index.update_settings(json!({})).await;
assert_valid_summarized_batch!(response, "settingsUpdate", "test");
let (response, _) = index.update_documents(json!([{"id": 1}]), None).await;
assert_valid_summarized_batch!(response, "documentAdditionOrUpdate", "test");
let (response, _) = index.add_documents(json!([{"id": 1}]), None).await;
assert_valid_summarized_batch!(response, "documentAdditionOrUpdate", "test");
let (response, _) = index.delete_document(1).await;
assert_valid_summarized_batch!(response, "documentDeletion", "test");
let (response, _) = index.clear_all_documents().await;
assert_valid_summarized_batch!(response, "documentDeletion", "test");
let (response, _) = index.delete().await;
assert_valid_summarized_batch!(response, "indexDeletion", "test");
}
#[actix_web::test] #[actix_web::test]
async fn test_summarized_document_addition_or_update() { async fn test_summarized_document_addition_or_update() {
let server = Server::new().await; let server = Server::new().await;
@ -361,7 +306,10 @@ async fn test_summarized_delete_documents_by_batch() {
@r#" @r#"
{ {
"uid": 0, "uid": 0,
"details": {}, "details": {
"providedIds": 3,
"deletedDocuments": 0
},
"stats": { "stats": {
"totalNbTasks": 1, "totalNbTasks": 1,
"status": { "status": {
@ -425,7 +373,11 @@ async fn test_summarized_delete_documents_by_filter() {
@r#" @r#"
{ {
"uid": 0, "uid": 0,
"details": {}, "details": {
"providedIds": 0,
"deletedDocuments": 0,
"originalFilter": "\"doggo = bernese\""
},
"stats": { "stats": {
"totalNbTasks": 1, "totalNbTasks": 1,
"status": { "status": {
@ -453,7 +405,11 @@ async fn test_summarized_delete_documents_by_filter() {
@r#" @r#"
{ {
"uid": 2, "uid": 2,
"details": {}, "details": {
"providedIds": 0,
"deletedDocuments": 0,
"originalFilter": "\"doggo = bernese\""
},
"stats": { "stats": {
"totalNbTasks": 1, "totalNbTasks": 1,
"status": { "status": {
@ -517,7 +473,10 @@ async fn test_summarized_delete_document_by_id() {
@r#" @r#"
{ {
"uid": 0, "uid": 0,
"details": {}, "details": {
"providedIds": 1,
"deletedDocuments": 0
},
"stats": { "stats": {
"totalNbTasks": 1, "totalNbTasks": 1,
"status": { "status": {
@ -663,7 +622,9 @@ async fn test_summarized_index_creation() {
@r#" @r#"
{ {
"uid": 1, "uid": 1,
"details": {}, "details": {
"primaryKey": "doggos"
},
"stats": { "stats": {
"totalNbTasks": 1, "totalNbTasks": 1,
"status": { "status": {
@ -832,7 +793,9 @@ async fn test_summarized_index_update() {
@r#" @r#"
{ {
"uid": 1, "uid": 1,
"details": {}, "details": {
"primaryKey": "bones"
},
"stats": { "stats": {
"totalNbTasks": 1, "totalNbTasks": 1,
"status": { "status": {
@ -926,7 +889,16 @@ async fn test_summarized_index_swap() {
@r#" @r#"
{ {
"uid": 0, "uid": 0,
"details": {}, "details": {
"swaps": [
{
"indexes": [
"doggos",
"cattos"
]
}
]
},
"stats": { "stats": {
"totalNbTasks": 1, "totalNbTasks": 1,
"status": { "status": {