snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-22 01:57:41 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

improve the way we access the mutex

Browse Source
This commit is contained in:
Tamo 2024-11-19 19:40:31 +01:00
parent b817251a83
commit 7fd98d74be
No known key found for this signature in database
GPG Key ID: 20CD8020AFA88D69
34 changed files with 155 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crates/index-scheduler/src/error.rs
View File

@ -230,7 +230,7 @@ impl ErrorCode for Error {
Error::InvalidTaskCanceledBy { .. } => Code::InvalidTaskCanceledBy,
Error::InvalidIndexUid { .. } => Code::InvalidIndexUid,
Error::TaskNotFound(_) => Code::TaskNotFound,
Error::BatchNotFound(_) => Code::TaskNotFound,
Error::BatchNotFound(_) => Code::BatchNotFound,
Error::TaskDeletionWithEmptyQuery => Code::MissingTaskFilters,
Error::TaskCancelationWithEmptyQuery => Code::MissingTaskFilters,
// TODO: not sure of the Code to use

81
crates/index-scheduler/src/lib.rs
View File

@ -160,7 +160,7 @@ impl Query {
}
#[derive(Debug, Clone)]
struct ProcessingTasks {
pub struct ProcessingTasks {
batch: Option<ProcessingBatch>,
/// The list of tasks ids that are currently running.
processing: RoaringBitmap,
@ -935,6 +935,7 @@ impl IndexScheduler {
processing: &ProcessingTasks,
query: &Query,
) -> Result<RoaringBitmap> {
dbg!();
let mut batches = self.all_batch_ids(rtxn)?;
if let Some(batch_id) = processing.batch.as_ref().map(|batch| batch.uid) {
batches.insert(batch_id);
@ -1222,22 +1223,21 @@ impl IndexScheduler {
/// 1. IndexSwap tasks are not publicly associated with any index, but they are associated
/// with many indexes internally.
/// 2. The user may not have the rights to access the tasks (internally) associated with all indexes.
pub fn get_batch_ids_from_authorized_indexes(
fn get_batch_ids_from_authorized_indexes(
&self,
rtxn: &RoTxn,
processing: &ProcessingTasks,
query: &Query,
filters: &meilisearch_auth::AuthFilter,
) -> Result<(RoaringBitmap, u64)> {
let processing = self.processing_tasks.read().unwrap().clone();
// compute all batches matching the filter by ignoring the limits, to find the number of batches matching
// the filter.
// As this causes us to compute the filter twice it is slightly inefficient, but doing it this way spares
// us from modifying the underlying implementation, and the performance remains sufficient.
// Should this change, we would modify `get_batch_ids` to directly return the number of matching batches.
let total_batches =
self.get_batch_ids(rtxn, &processing, &query.clone().without_limits())?;
let mut batches = self.get_batch_ids(rtxn, &processing, query)?;
self.get_batch_ids(rtxn, processing, &query.clone().without_limits())?;
let mut batches = self.get_batch_ids(rtxn, processing, query)?;
// If the query contains a list of index uid or there is a finite list of authorized indexes,
// then we must exclude all the batches that only contains tasks associated to multiple indexes.
@ -1353,11 +1353,14 @@ impl IndexScheduler {
filters: &meilisearch_auth::AuthFilter,
) -> Result<(Vec<Batch>, u64)> {
let rtxn = self.env.read_txn()?;
let processing = self.processing_tasks.read().unwrap().clone();
let (batches, total) =
self.get_batch_ids_from_authorized_indexes(&rtxn, &query, filters)?;
self.get_batch_ids_from_authorized_indexes(&rtxn, &processing, &query, filters)?;
let batches = self.get_existing_batches(
&rtxn,
&processing,
batches.into_iter().rev().take(query.limit.unwrap_or(u32::MAX) as usize),
)?;
@ -4186,46 +4189,47 @@ mod tests {
handle.advance_n_successful_batches(3);
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "processed_all_tasks");
let proc = index_scheduler.processing_tasks.read().unwrap().clone();
let rtxn = index_scheduler.env.read_txn().unwrap();
let query = Query { limit: Some(0), ..Default::default() };
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
snapshot!(snapshot_bitmap(&batches), @"[]");
let query = Query { limit: Some(1), ..Default::default() };
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
snapshot!(snapshot_bitmap(&batches), @"[2,]");
let query = Query { limit: Some(2), ..Default::default() };
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
snapshot!(snapshot_bitmap(&batches), @"[1,2,]");
let query = Query { from: Some(1), ..Default::default() };
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
snapshot!(snapshot_bitmap(&batches), @"[0,1,]");
let query = Query { from: Some(2), ..Default::default() };
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
snapshot!(snapshot_bitmap(&batches), @"[0,1,2,]");
let query = Query { from: Some(1), limit: Some(1), ..Default::default() };
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
snapshot!(snapshot_bitmap(&batches), @"[1,]");
let query = Query { from: Some(1), limit: Some(2), ..Default::default() };
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
snapshot!(snapshot_bitmap(&batches), @"[0,1,]");
}
@ -4249,16 +4253,17 @@ mod tests {
handle.advance_till([Start, BatchCreated]);
let rtxn = index_scheduler.env.read_txn().unwrap();
let proc = index_scheduler.processing_tasks.read().unwrap().clone();
let query = Query { statuses: Some(vec![Status::Processing]), ..Default::default() };
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
snapshot!(snapshot_bitmap(&batches), @"[0,]"); // only the processing batch in the first tick
let query = Query { statuses: Some(vec![Status::Enqueued]), ..Default::default() };
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
snapshot!(snapshot_bitmap(&batches), @"[]"); // The batches don't contains any enqueued tasks
@ -4267,7 +4272,7 @@ mod tests {
..Default::default()
};
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
snapshot!(snapshot_bitmap(&batches), @"[0,]"); // both enqueued and processing tasks in the first tick
@ -4277,7 +4282,7 @@ mod tests {
..Default::default()
};
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// both enqueued and processing tasks in the first tick, but limited to those with a started_at
// that comes after the start of the test, which should excludes the enqueued tasks
@ -4289,7 +4294,7 @@ mod tests {
..Default::default()
};
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// both enqueued and processing tasks in the first tick, but limited to those with a started_at
// that comes before the start of the test, which should excludes all of them
@ -4302,7 +4307,7 @@ mod tests {
..Default::default()
};
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// both enqueued and processing tasks in the first tick, but limited to those with a started_at
// that comes after the start of the test and before one minute after the start of the test,
@ -4317,8 +4322,10 @@ mod tests {
Start,
BatchCreated,
]);
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after-advancing-a-bit");
let rtxn = index_scheduler.env.read_txn().unwrap();
let proc = index_scheduler.processing_tasks.read().unwrap().clone();
let second_start_time = OffsetDateTime::now_utc();
@ -4329,7 +4336,7 @@ mod tests {
..Default::default()
};
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// both succeeded and processing tasks in the first tick, but limited to those with a started_at
// that comes after the start of the test and before one minute after the start of the test,
@ -4342,7 +4349,7 @@ mod tests {
..Default::default()
};
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// both succeeded and processing tasks in the first tick, but limited to those with a started_at
// that comes before the start of the test, which should exclude all tasks
@ -4355,7 +4362,7 @@ mod tests {
..Default::default()
};
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// both succeeded and processing tasks in the first tick, but limited to those with a started_at
// that comes after the start of the second part of the test and before one minute after the
@ -4373,9 +4380,10 @@ mod tests {
]);
let rtxn = index_scheduler.env.read_txn().unwrap();
let proc = index_scheduler.processing_tasks.read().unwrap().clone();
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// we run the same query to verify that, and indeed find that the last task is matched
snapshot!(snapshot_bitmap(&batches), @"[2,]");
@ -4387,7 +4395,7 @@ mod tests {
..Default::default()
};
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// enqueued, succeeded, or processing tasks started after the second part of the test, should
// again only return the last task
@ -4395,11 +4403,12 @@ mod tests {
handle.advance_till([ProcessBatchFailed, AfterProcessing]);
let rtxn = index_scheduler.read_txn().unwrap();
let proc = index_scheduler.processing_tasks.read().unwrap().clone();
// now the last task should have failed
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "end");
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// so running the last query should return nothing
snapshot!(snapshot_bitmap(&batches), @"[]");
@ -4411,7 +4420,7 @@ mod tests {
..Default::default()
};
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// but the same query on failed tasks should return the last task
snapshot!(snapshot_bitmap(&batches), @"[2,]");
@ -4423,7 +4432,7 @@ mod tests {
..Default::default()
};
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// but the same query on failed tasks should return the last task
snapshot!(snapshot_bitmap(&batches), @"[2,]");
@ -4436,7 +4445,7 @@ mod tests {
..Default::default()
};
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// same query but with an invalid uid
snapshot!(snapshot_bitmap(&batches), @"[]");
@ -4449,7 +4458,7 @@ mod tests {
..Default::default()
};
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// same query but with a valid uid
snapshot!(snapshot_bitmap(&batches), @"[2,]");
@ -4478,10 +4487,11 @@ mod tests {
handle.advance_till([Start, BatchCreated]);
let rtxn = index_scheduler.env.read_txn().unwrap();
let proc = index_scheduler.processing_tasks.read().unwrap().clone();
let query = Query { index_uids: Some(vec!["catto".to_owned()]), ..Default::default() };
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// only the first task associated with catto is returned, the indexSwap tasks are excluded!
snapshot!(snapshot_bitmap(&batches), @"[0,]");
@ -4490,6 +4500,7 @@ mod tests {
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(
&rtxn,
&proc,
&query,
&AuthFilter::with_allowed_indexes(
vec![IndexUidPattern::new_unchecked("doggo")].into_iter().collect(),
@ -4517,6 +4528,7 @@ mod tests {
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(
&rtxn,
&proc,
&query,
&AuthFilter::with_allowed_indexes(
vec![IndexUidPattern::new_unchecked("doggo")].into_iter().collect(),
@ -4531,6 +4543,7 @@ mod tests {
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(
&rtxn,
&proc,
&query,
&AuthFilter::with_allowed_indexes(
vec![
@ -4548,7 +4561,7 @@ mod tests {
let query = Query::default();
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// we asked for all the tasks with all index authorized -> all tasks returned
snapshot!(snapshot_bitmap(&batches), @"[0,1,2,3,]");
@ -4579,9 +4592,10 @@ mod tests {
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "start");
let rtxn = index_scheduler.read_txn().unwrap();
let proc = index_scheduler.processing_tasks.read().unwrap().clone();
let query = Query { canceled_by: Some(vec![task_cancelation.uid]), ..Query::default() };
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default())
.get_batch_ids_from_authorized_indexes(&rtxn, &proc, &query, &AuthFilter::default())
.unwrap();
// The batch zero was the index creation task, the 1 is the task cancellation
snapshot!(snapshot_bitmap(&batches), @"[1,]");
@ -4590,6 +4604,7 @@ mod tests {
let (batches, _) = index_scheduler
.get_batch_ids_from_authorized_indexes(
&rtxn,
&proc,
&query,
&AuthFilter::with_allowed_indexes(
vec![IndexUidPattern::new_unchecked("doggo")].into_iter().collect(),

2
crates/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_failing_the_deletion.snap
View File

@ -38,7 +38,7 @@ doggos [0,1,]
[timestamp] [0,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentDeletion":1},"indexUids":{"doggos":1}}, }
0 {uid: 0, details: {"providedIds":2,"deletedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentDeletion":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_last_successful_addition.snap
View File

@ -42,7 +42,7 @@ doggos: { number_of_documents: 3, field_distribution: {"catto": 1, "doggo": 2, "
[timestamp] [1,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentDeletion":1},"indexUids":{"doggos":1}}, }
0 {uid: 0, details: {"providedIds":2,"deletedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentDeletion":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":3,"indexedDocuments":3}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:

2
crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/document_addition_failed.snap
View File

@ -35,7 +35,7 @@ doggos [0,]
[timestamp] [0,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_removing_the_documents.snap
View File

@ -55,7 +55,7 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
### All Batches:
0 {uid: 0, details: {"filterableAttributes":["catto"]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"settingsUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":3,"indexedDocuments":3}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {"providedIds":1,"deletedDocuments":2,"originalFilter":"\"catto EXISTS\""}, stats: {"totalNbTasks":4,"status":{"succeeded":2,"failed":2},"types":{"documentDeletion":4},"indexUids":{"doggos":4}}, }
2 {uid: 2, details: {"providedIds":1,"deletedDocuments":2,"originalFilter":"true&\"id = 2\"&\"catto EXISTS\""}, stats: {"totalNbTasks":4,"status":{"succeeded":2,"failed":2},"types":{"documentDeletion":4},"indexUids":{"doggos":4}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap
View File

@ -35,7 +35,7 @@ catto [0,]
[timestamp] [0,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, }
0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap
View File

@ -35,7 +35,7 @@ catto [0,]
[timestamp] [0,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, }
0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/query_batches_simple/end.snap
View File

@ -50,7 +50,7 @@ doggo: { number_of_documents: 0, field_distribution: {} }
### All Batches:
0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, }
1 {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"whalo":1}}, }
2 {uid: 2, details: {"primaryKey":"fish"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"whalo":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

4
crates/index-scheduler/src/snapshots/lib.rs/query_batches_special_rules/after-processing-everything.snap
View File

@ -55,8 +55,8 @@ doggo: { number_of_documents: 0, field_distribution: {} }
### All Batches:
0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, }
1 {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, }
3 {uid: 3, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, }
2 {uid: 2, details: {"swaps":[{"indexes":["catto","doggo"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, }
3 {uid: 3, details: {"swaps":[{"indexes":["catto","whalo"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/query_tasks_simple/end.snap
View File

@ -50,7 +50,7 @@ doggo: { number_of_documents: 0, field_distribution: {} }
### All Batches:
0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, }
1 {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"whalo":1}}, }
2 {uid: 2, details: {"primaryKey":"fish"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"whalo":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/swap_indexes_errors/first_swap_failed.snap
View File

@ -66,7 +66,7 @@ d: { number_of_documents: 0, field_distribution: {} }
1 {uid: 1, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"b":1}}, }
2 {uid: 2, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"c":1}}, }
3 {uid: 3, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"d":1}}, }
4 {uid: 4, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, }
4 {uid: 4, details: {"swaps":[{"indexes":["a","b"]},{"indexes":["c","e"]},{"indexes":["d","f"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index/after_processing_the_10_tasks.snap
View File

@ -53,7 +53,7 @@ doggos [0,1,2,3,4,5,6,7,8,9,]
[timestamp] [0,1,2,3,4,5,6,7,8,9,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":10,"status":{"failed":10},"types":{"documentAdditionOrUpdate":10},"indexUids":{"doggos":10}}, }
0 {uid: 0, details: {"receivedDocuments":10,"indexedDocuments":0}, stats: {"totalNbTasks":10,"status":{"failed":10},"types":{"documentAdditionOrUpdate":10},"indexUids":{"doggos":10}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,1,2,3,4,5,6,7,8,9,]

20
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/all_tasks_processed.snap
View File

@ -71,16 +71,16 @@ doggos [0,1,2,3,4,5,6,7,8,9,]
[timestamp] [9,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
3 {uid: 3, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
4 {uid: 4, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
5 {uid: 5, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
6 {uid: 6, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
7 {uid: 7, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
8 {uid: 8, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
9 {uid: 9, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
3 {uid: 3, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
4 {uid: 4, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
5 {uid: 5, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
6 {uid: 6, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
7 {uid: 7, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
8 {uid: 8, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
9 {uid: 9, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

10
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/five_tasks_processed.snap
View File

@ -61,11 +61,11 @@ doggos [0,1,2,3,4,5,6,7,8,9,]
[timestamp] [4,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
3 {uid: 3, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
4 {uid: 4, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
3 {uid: 3, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
4 {uid: 4, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/all_tasks_processed.snap
View File

@ -57,7 +57,7 @@ doggos: { number_of_documents: 9, field_distribution: {"doggo": 9, "id": 9} }
[timestamp] [1,2,3,4,5,6,7,8,9,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":9,"indexedDocuments":9}, stats: {"totalNbTasks":9,"status":{"succeeded":9},"types":{"documentAdditionOrUpdate":9},"indexUids":{"doggos":9}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/only_first_task_failed.snap
View File

@ -53,7 +53,7 @@ doggos [0,1,2,3,4,5,6,7,8,9,]
[timestamp] [0,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

4
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/fifth_task_succeeds.snap
View File

@ -51,9 +51,9 @@ doggos: { number_of_documents: 2, field_distribution: {"doggo": 2, "id": 2} }
[timestamp] [4,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, }
0 {uid: 0, details: {"receivedDocuments":2,"indexedDocuments":0}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
3 {uid: 3, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/first_and_second_task_fails.snap
View File

@ -44,7 +44,7 @@ doggos: { number_of_documents: 0, field_distribution: {} }
[timestamp] [0,1,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, }
0 {uid: 0, details: {"receivedDocuments":2,"indexedDocuments":0}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,1,]

4
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/fourth_task_fails.snap
View File

@ -49,9 +49,9 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
[timestamp] [3,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, }
0 {uid: 0, details: {"receivedDocuments":2,"indexedDocuments":0}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,1,]

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/third_task_succeeds.snap
View File

@ -47,7 +47,7 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
[timestamp] [2,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, }
0 {uid: 0, details: {"receivedDocuments":2,"indexedDocuments":0}, stats: {"totalNbTasks":2,"status":{"failed":2},"types":{"documentAdditionOrUpdate":2},"indexUids":{"doggos":2}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/second_task_fails.snap
View File

@ -44,7 +44,7 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

4
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/third_task_fails.snap
View File

@ -46,8 +46,8 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/second_and_third_tasks_fails.snap
View File

@ -44,7 +44,7 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

4
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/all_other_tasks_succeeds.snap
View File

@ -53,8 +53,8 @@ doggos: { number_of_documents: 4, field_distribution: {"doggo": 4, "paw": 4} }
[timestamp] [3,4,5,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
3 {uid: 3, details: {"receivedDocuments":3,"indexedDocuments":3}, stats: {"totalNbTasks":3,"status":{"succeeded":3},"types":{"documentAdditionOrUpdate":3},"indexUids":{"doggos":3}}, }
----------------------------------------------------------------------

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/first_task_fails.snap
View File

@ -46,7 +46,7 @@ doggos: { number_of_documents: 0, field_distribution: {} }
[timestamp] [0,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

4
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/second_task_fails.snap
View File

@ -48,8 +48,8 @@ doggos: { number_of_documents: 0, field_distribution: {} }
[timestamp] [1,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

4
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/third_task_succeeds.snap
View File

@ -51,8 +51,8 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "paw": 1} }
[timestamp] [2,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/all_other_tasks_succeeds.snap
View File

@ -54,7 +54,7 @@ doggos: { number_of_documents: 5, field_distribution: {"doggo": 5, "doggoid": 5}
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
3 {uid: 3, details: {"receivedDocuments":3,"indexedDocuments":3}, stats: {"totalNbTasks":3,"status":{"succeeded":3},"types":{"documentAdditionOrUpdate":3},"indexUids":{"doggos":3}}, }
----------------------------------------------------------------------

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/second_task_fails.snap
View File

@ -50,7 +50,7 @@ doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "doggoid": 1}
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

2
crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/third_task_succeeds.snap
View File

@ -52,7 +52,7 @@ doggos: { number_of_documents: 2, field_distribution: {"doggo": 2, "doggoid": 2}
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
1 {uid: 1, details: {"receivedDocuments":1,"indexedDocuments":0}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
2 {uid: 2, details: {"receivedDocuments":1,"indexedDocuments":1}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"documentAdditionOrUpdate":1},"indexUids":{"doggos":1}}, }
----------------------------------------------------------------------
### Batch to tasks mapping:

27
crates/index-scheduler/src/utils.rs
View File

@ -12,7 +12,7 @@ use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status
use roaring::{MultiOps, RoaringBitmap};
use time::OffsetDateTime;
use crate::{Error, IndexScheduler, Result, Task, TaskId, BEI128};
use crate::{Error, IndexScheduler, ProcessingTasks, Result, Task, TaskId, BEI128};
/// This structure contains all the information required to write a batch in the database without reading the tasks.
/// It'll stay in RAM so it must be small.
@ -106,10 +106,8 @@ impl ProcessingBatch {
self.statuses.insert(task.status);
// Craft an aggregation of the details of all the tasks encountered in this batch.
if task.status != Status::Failed {
if let Some(ref details) = task.details {
self.details.accumulate(&DetailsView::from(details.clone()));
}
if let Some(ref details) = task.details {
self.details.accumulate(&DetailsView::from(details.clone()));
}
self.stats.total_nb_tasks += 1;
*self.stats.status.entry(task.status).or_default() += 1;
@ -118,6 +116,16 @@ impl ProcessingBatch {
*self.stats.index_uids.entry(index_uid.to_string()).or_default() += 1;
}
}
pub fn to_batch(&self) -> Batch {
Batch {
uid: self.uid,
details: self.details.clone(),
stats: self.stats.clone(),
started_at: self.started_at,
finished_at: self.finished_at,
}
}
}
impl IndexScheduler {
@ -243,13 +251,18 @@ impl IndexScheduler {
pub(crate) fn get_existing_batches(
&self,
rtxn: &RoTxn,
processing: &ProcessingTasks,
tasks: impl IntoIterator<Item = BatchId>,
) -> Result<Vec<Batch>> {
tasks
.into_iter()
.map(|batch_id| {
self.get_batch(rtxn, batch_id)
.and_then(|task| task.ok_or(Error::CorruptedTaskQueue))
if Some(batch_id) == processing.batch.as_ref().map(|batch| batch.uid) {
Ok(processing.batch.as_ref().unwrap().to_batch())
} else {
self.get_batch(rtxn, batch_id)
.and_then(|task| task.ok_or(Error::CorruptedTaskQueue))
}
})
.collect::<Result<_>>()
}

2
crates/meilisearch/src/routes/batches.rs
View File

@ -42,7 +42,7 @@ async fn get_batch(
let task_view = BatchView::from_batch(batch);
Ok(HttpResponse::Ok().json(task_view))
} else {
Err(index_scheduler::Error::TaskNotFound(batch_uid).into())
Err(index_scheduler::Error::BatchNotFound(batch_uid).into())
}
}

100
crates/meilisearch/tests/batches/mod.rs
View File

@ -2,8 +2,6 @@ mod errors;
use meili_snap::insta::assert_json_snapshot;
use meili_snap::snapshot;
use time::format_description::well_known::Rfc3339;
use time::OffsetDateTime;
use crate::common::Server;
use crate::json;
@ -17,7 +15,7 @@ async fn error_get_unexisting_batch_status() {
let (response, code) = index.get_batch(1).await;
let expected_response = json!({
"message": "batch `1` not found.",
"message": "Batch `1` not found.",
"code": "batch_not_found",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#batch_not_found"
@ -32,19 +30,9 @@ async fn get_batch_status() {
let server = Server::new().await;
let index = server.index("test");
index.create(None).await;
index
.add_documents(
json!([{
"id": 1,
"content": "foobar",
}]),
None,
)
.await;
index.wait_task(0).await;
let (_response, code) = index.get_batch(1).await;
let (_response, code) = index.get_batch(0).await;
assert_eq!(code, 200);
// TODO check response format, as per #48
}
#[actix_rt::test]
@ -241,49 +229,6 @@ async fn get_batch_filter_error() {
"#);
}
macro_rules! assert_valid_summarized_batch {
($response:expr, $batch_type:literal, $index:literal) => {{
assert_eq!($response.as_object().unwrap().len(), 5);
assert!($response["batchUid"].as_u64().is_some());
assert_eq!($response["indexUid"], $index);
assert_eq!($response["status"], "enqueued");
assert_eq!($response["type"], $batch_type);
let date = $response["enqueuedAt"].as_str().expect("missing date");
OffsetDateTime::parse(date, &Rfc3339).unwrap();
}};
}
#[actix_web::test]
async fn test_summarized_batch_view() {
let server = Server::new().await;
let index = server.index("test");
let (response, _) = index.create(None).await;
assert_valid_summarized_batch!(response, "indexCreation", "test");
let (response, _) = index.update(None).await;
assert_valid_summarized_batch!(response, "indexUpdate", "test");
let (response, _) = index.update_settings(json!({})).await;
assert_valid_summarized_batch!(response, "settingsUpdate", "test");
let (response, _) = index.update_documents(json!([{"id": 1}]), None).await;
assert_valid_summarized_batch!(response, "documentAdditionOrUpdate", "test");
let (response, _) = index.add_documents(json!([{"id": 1}]), None).await;
assert_valid_summarized_batch!(response, "documentAdditionOrUpdate", "test");
let (response, _) = index.delete_document(1).await;
assert_valid_summarized_batch!(response, "documentDeletion", "test");
let (response, _) = index.clear_all_documents().await;
assert_valid_summarized_batch!(response, "documentDeletion", "test");
let (response, _) = index.delete().await;
assert_valid_summarized_batch!(response, "indexDeletion", "test");
}
#[actix_web::test]
async fn test_summarized_document_addition_or_update() {
let server = Server::new().await;
@ -361,7 +306,10 @@ async fn test_summarized_delete_documents_by_batch() {
@r#"
{
"uid": 0,
"details": {},
"details": {
"providedIds": 3,
"deletedDocuments": 0
},
"stats": {
"totalNbTasks": 1,
"status": {
@ -425,7 +373,11 @@ async fn test_summarized_delete_documents_by_filter() {
@r#"
{
"uid": 0,
"details": {},
"details": {
"providedIds": 0,
"deletedDocuments": 0,
"originalFilter": "\"doggo = bernese\""
},
"stats": {
"totalNbTasks": 1,
"status": {
@ -453,7 +405,11 @@ async fn test_summarized_delete_documents_by_filter() {
@r#"
{
"uid": 2,
"details": {},
"details": {
"providedIds": 0,
"deletedDocuments": 0,
"originalFilter": "\"doggo = bernese\""
},
"stats": {
"totalNbTasks": 1,
"status": {
@ -517,7 +473,10 @@ async fn test_summarized_delete_document_by_id() {
@r#"
{
"uid": 0,
"details": {},
"details": {
"providedIds": 1,
"deletedDocuments": 0
},
"stats": {
"totalNbTasks": 1,
"status": {
@ -663,7 +622,9 @@ async fn test_summarized_index_creation() {
@r#"
{
"uid": 1,
"details": {},
"details": {
"primaryKey": "doggos"
},
"stats": {
"totalNbTasks": 1,
"status": {
@ -832,7 +793,9 @@ async fn test_summarized_index_update() {
@r#"
{
"uid": 1,
"details": {},
"details": {
"primaryKey": "bones"
},
"stats": {
"totalNbTasks": 1,
"status": {
@ -926,7 +889,16 @@ async fn test_summarized_index_swap() {
@r#"
{
"uid": 0,
"details": {},
"details": {
"swaps": [
{
"indexes": [
"doggos",
"cattos"
]
}
]
},
"stats": {
"totalNbTasks": 1,
"status": {