make the generated filter valid

This commit is contained in:
Tamo 2023-04-26 13:55:02 +02:00 committed by Louis Dureuil
parent 9ca6f59546
commit dcbfecf42c
No known key found for this signature in database
8 changed files with 451 additions and 278 deletions

View File

@ -51,6 +51,7 @@ use meilisearch_types::milli::{self, CboRoaringBitmapCodec, Index, RoaringBitmap
use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task}; use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
use synchronoise::SignalEvent; use synchronoise::SignalEvent;
use time::format_description::well_known::Rfc3339;
use time::OffsetDateTime; use time::OffsetDateTime;
use utils::{filter_out_references_to_newer_tasks, keep_tasks_within_datetimes, map_bound}; use utils::{filter_out_references_to_newer_tasks, keep_tasks_within_datetimes, map_bound};
use uuid::Uuid; use uuid::Uuid;
@ -1118,7 +1119,6 @@ impl IndexScheduler {
let finished = self.status.get(&rtxn, &Status::Succeeded)?.unwrap_or_default() let finished = self.status.get(&rtxn, &Status::Succeeded)?.unwrap_or_default()
| self.status.get(&rtxn, &Status::Failed)?.unwrap_or_default() | self.status.get(&rtxn, &Status::Failed)?.unwrap_or_default()
| self.status.get(&rtxn, &Status::Canceled)?.unwrap_or_default(); | self.status.get(&rtxn, &Status::Canceled)?.unwrap_or_default();
drop(rtxn);
let to_delete = RoaringBitmap::from_iter(finished.into_iter().rev().take(100_000)); let to_delete = RoaringBitmap::from_iter(finished.into_iter().rev().take(100_000));
@ -1135,11 +1135,15 @@ impl IndexScheduler {
to_delete.len() to_delete.len()
); );
// it's safe to unwrap here because we checked the len above
let newest_task_id = to_delete.iter().last().unwrap();
let task = self.get_task(&rtxn, newest_task_id)?.ok_or(Error::CorruptedTaskQueue)?;
drop(rtxn);
self.register(KindWithContent::TaskDeletion { self.register(KindWithContent::TaskDeletion {
query: format!( query: format!(
"?from={},limit={},status=succeeded,failed,canceled", "?beforeEnqueuedAt={},status=succeeded,failed,canceled",
to_delete.iter().last().unwrap_or(u32::MAX), task.enqueued_at.format(&Rfc3339).map_err(|_| Error::CorruptedTaskQueue)?,
to_delete.len(),
), ),
tasks: to_delete, tasks: to_delete,
})?; })?;
@ -1404,7 +1408,7 @@ mod tests {
use big_s::S; use big_s::S;
use crossbeam::channel::RecvTimeoutError; use crossbeam::channel::RecvTimeoutError;
use file_store::File; use file_store::File;
use meili_snap::snapshot; use meili_snap::{json_string, snapshot};
use meilisearch_auth::AuthFilter; use meilisearch_auth::AuthFilter;
use meilisearch_types::document_formats::DocumentFormatError; use meilisearch_types::document_formats::DocumentFormatError;
use meilisearch_types::error::ErrorCode; use meilisearch_types::error::ErrorCode;
@ -3860,8 +3864,6 @@ mod tests {
handle.advance_one_failed_batch(); handle.advance_one_failed_batch();
// at this point the max number of tasks is reached // at this point the max number of tasks is reached
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "max_number_of_tasks");
// we can still enqueue multiple tasks // we can still enqueue multiple tasks
index_scheduler index_scheduler
.register(KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None }) .register(KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None })
@ -3870,22 +3872,43 @@ mod tests {
.register(KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None }) .register(KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None })
.unwrap(); .unwrap();
// at this point the max number of tasks is reached let rtxn = index_scheduler.env.read_txn().unwrap();
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "above_the_max_number_of_tasks"); let tasks = index_scheduler.get_task_ids(&rtxn, &Query { ..Default::default() }).unwrap();
let tasks = index_scheduler.get_existing_tasks(&rtxn, tasks).unwrap();
snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]" }), name: "task_queue_is_full");
drop(rtxn);
// now we're above the max number of tasks
// and if we try to advance in the tick function a new task deletion should be enqueued // and if we try to advance in the tick function a new task deletion should be enqueued
handle.advance_till([Start, BatchCreated]); handle.advance_till([Start, BatchCreated]);
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "task_deletion_have_been_enqueued"); let rtxn = index_scheduler.env.read_txn().unwrap();
let tasks = index_scheduler.get_task_ids(&rtxn, &Query { ..Default::default() }).unwrap();
let tasks = index_scheduler.get_existing_tasks(&rtxn, tasks).unwrap();
snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_been_enqueued");
drop(rtxn);
handle.advance_till([InsideProcessBatch, ProcessBatchSucceeded, AfterProcessing]); handle.advance_till([InsideProcessBatch, ProcessBatchSucceeded, AfterProcessing]);
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "task_deletion_have_been_processed"); let rtxn = index_scheduler.env.read_txn().unwrap();
let tasks = index_scheduler.get_task_ids(&rtxn, &Query { ..Default::default() }).unwrap();
let tasks = index_scheduler.get_existing_tasks(&rtxn, tasks).unwrap();
snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_been_processed");
drop(rtxn);
handle.advance_one_failed_batch(); handle.advance_one_failed_batch();
// a new task deletion has been enqueued // a new task deletion has been enqueued
handle.advance_one_successful_batch(); handle.advance_one_successful_batch();
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_the_second_task_deletion"); let rtxn = index_scheduler.env.read_txn().unwrap();
let tasks = index_scheduler.get_task_ids(&rtxn, &Query { ..Default::default() }).unwrap();
let tasks = index_scheduler.get_existing_tasks(&rtxn, tasks).unwrap();
snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "after_the_second_task_deletion");
drop(rtxn);
handle.advance_one_failed_batch(); handle.advance_one_failed_batch();
handle.advance_one_successful_batch(); handle.advance_one_successful_batch();
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "everything_has_been_processed"); let rtxn = index_scheduler.env.read_txn().unwrap();
let tasks = index_scheduler.get_task_ids(&rtxn, &Query { ..Default::default() }).unwrap();
let tasks = index_scheduler.get_existing_tasks(&rtxn, tasks).unwrap();
snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "everything_has_been_processed");
drop(rtxn);
} }
} }

View File

@ -1,49 +0,0 @@
---
source: index-scheduler/src/lib.rs
---
### Autobatching Enabled = true
### Processing Tasks:
[]
----------------------------------------------------------------------
### All Tasks:
0 {uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggo", primary_key: None }}
1 {uid: 1, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggo", primary_key: None }}
2 {uid: 2, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggo", primary_key: None }}
3 {uid: 3, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggo", primary_key: None }}
----------------------------------------------------------------------
### Status:
enqueued [2,3,]
succeeded [0,]
failed [1,]
----------------------------------------------------------------------
### Kind:
"indexCreation" [0,1,2,3,]
----------------------------------------------------------------------
### Index Tasks:
doggo [0,1,2,3,]
----------------------------------------------------------------------
### Index Mapper:
doggo: { number_of_documents: 0, field_distribution: {} }
----------------------------------------------------------------------
### Canceled By:
----------------------------------------------------------------------
### Enqueued At:
[timestamp] [0,]
[timestamp] [1,]
[timestamp] [2,]
[timestamp] [3,]
----------------------------------------------------------------------
### Started At:
[timestamp] [0,]
[timestamp] [1,]
----------------------------------------------------------------------
### Finished At:
[timestamp] [0,]
[timestamp] [1,]
----------------------------------------------------------------------
### File Store:
----------------------------------------------------------------------

View File

@ -1,44 +1,68 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
--- ---
### Autobatching Enabled = true [
### Processing Tasks: {
[] "uid": 3,
---------------------------------------------------------------------- "enqueuedAt": "[date]",
### All Tasks: "startedAt": "[date]",
3 {uid: 3, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggo", primary_key: None }} "finishedAt": "[date]",
5 {uid: 5, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(2), original_filter: "?from=4,limit=2,status=succeeded,failed,canceled" }, kind: TaskDeletion { query: "?from=4,limit=2,status=succeeded,failed,canceled", tasks: RoaringBitmap<[2, 4]> }} "error": null,
---------------------------------------------------------------------- "canceledBy": null,
### Status: "details": {
enqueued [3,] "IndexInfo": {
succeeded [5,] "primary_key": null
failed [] }
---------------------------------------------------------------------- },
### Kind: "status": "enqueued",
"indexCreation" [3,] "kind": {
"taskDeletion" [5,] "indexCreation": {
---------------------------------------------------------------------- "index_uid": "doggo",
### Index Tasks: "primary_key": null
doggo [3,] }
---------------------------------------------------------------------- }
### Index Mapper: },
doggo: { number_of_documents: 0, field_distribution: {} } {
"uid": 5,
---------------------------------------------------------------------- "enqueuedAt": "[date]",
### Canceled By: "startedAt": "[date]",
"finishedAt": "[date]",
---------------------------------------------------------------------- "error": null,
### Enqueued At: "canceledBy": null,
[timestamp] [3,] "details": {
[timestamp] [5,] "TaskDeletion": {
---------------------------------------------------------------------- "matched_tasks": 2,
### Started At: "deleted_tasks": 2,
[timestamp] [5,] "original_filter": "[filter]"
---------------------------------------------------------------------- }
### Finished At: },
[timestamp] [5,] "status": "succeeded",
---------------------------------------------------------------------- "kind": {
### File Store: "taskDeletion": {
"query": "[query]",
---------------------------------------------------------------------- "tasks": [
58,
48,
0,
0,
1,
0,
0,
0,
0,
0,
1,
0,
16,
0,
0,
0,
2,
0,
4,
0
]
}
}
}
]

View File

@ -1,41 +1,48 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
--- ---
### Autobatching Enabled = true [
### Processing Tasks: {
[] "uid": 6,
---------------------------------------------------------------------- "enqueuedAt": "[date]",
### All Tasks: "startedAt": "[date]",
6 {uid: 6, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(2), original_filter: "?from=5,limit=2,status=succeeded,failed,canceled" }, kind: TaskDeletion { query: "?from=5,limit=2,status=succeeded,failed,canceled", tasks: RoaringBitmap<[3, 5]> }} "finishedAt": "[date]",
---------------------------------------------------------------------- "error": null,
### Status: "canceledBy": null,
enqueued [] "details": {
succeeded [6,] "TaskDeletion": {
failed [] "matched_tasks": 2,
---------------------------------------------------------------------- "deleted_tasks": 2,
### Kind: "original_filter": "[filter]"
"indexCreation" [] }
"taskDeletion" [6,] },
---------------------------------------------------------------------- "status": "succeeded",
### Index Tasks: "kind": {
---------------------------------------------------------------------- "taskDeletion": {
### Index Mapper: "query": "[query]",
doggo: { number_of_documents: 0, field_distribution: {} } "tasks": [
58,
---------------------------------------------------------------------- 48,
### Canceled By: 0,
0,
---------------------------------------------------------------------- 1,
### Enqueued At: 0,
[timestamp] [6,] 0,
---------------------------------------------------------------------- 0,
### Started At: 0,
[timestamp] [6,] 0,
---------------------------------------------------------------------- 1,
### Finished At: 0,
[timestamp] [6,] 16,
---------------------------------------------------------------------- 0,
### File Store: 0,
0,
---------------------------------------------------------------------- 3,
0,
5,
0
]
}
}
}
]

View File

@ -1,45 +0,0 @@
---
source: index-scheduler/src/lib.rs
---
### Autobatching Enabled = true
### Processing Tasks:
[]
----------------------------------------------------------------------
### All Tasks:
0 {uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggo", primary_key: None }}
1 {uid: 1, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggo", primary_key: None }}
----------------------------------------------------------------------
### Status:
enqueued []
succeeded [0,]
failed [1,]
----------------------------------------------------------------------
### Kind:
"indexCreation" [0,1,]
----------------------------------------------------------------------
### Index Tasks:
doggo [0,1,]
----------------------------------------------------------------------
### Index Mapper:
doggo: { number_of_documents: 0, field_distribution: {} }
----------------------------------------------------------------------
### Canceled By:
----------------------------------------------------------------------
### Enqueued At:
[timestamp] [0,]
[timestamp] [1,]
----------------------------------------------------------------------
### Started At:
[timestamp] [0,]
[timestamp] [1,]
----------------------------------------------------------------------
### Finished At:
[timestamp] [0,]
[timestamp] [1,]
----------------------------------------------------------------------
### File Store:
----------------------------------------------------------------------

View File

@ -1,52 +1,133 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
--- ---
### Autobatching Enabled = true [
### Processing Tasks: {
[4,] "uid": 0,
---------------------------------------------------------------------- "enqueuedAt": "[date]",
### All Tasks: "startedAt": "[date]",
0 {uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggo", primary_key: None }} "finishedAt": "[date]",
1 {uid: 1, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggo", primary_key: None }} "error": null,
2 {uid: 2, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggo", primary_key: None }} "canceledBy": null,
3 {uid: 3, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggo", primary_key: None }} "details": {
4 {uid: 4, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_filter: "?from=1,limit=2,status=succeeded,failed,canceled" }, kind: TaskDeletion { query: "?from=1,limit=2,status=succeeded,failed,canceled", tasks: RoaringBitmap<[0, 1]> }} "IndexInfo": {
---------------------------------------------------------------------- "primary_key": null
### Status: }
enqueued [2,3,4,] },
succeeded [0,] "status": "succeeded",
failed [1,] "kind": {
---------------------------------------------------------------------- "indexCreation": {
### Kind: "index_uid": "doggo",
"indexCreation" [0,1,2,3,] "primary_key": null
"taskDeletion" [4,] }
---------------------------------------------------------------------- }
### Index Tasks: },
doggo [0,1,2,3,] {
---------------------------------------------------------------------- "uid": 1,
### Index Mapper: "enqueuedAt": "[date]",
doggo: { number_of_documents: 0, field_distribution: {} } "startedAt": "[date]",
"finishedAt": "[date]",
---------------------------------------------------------------------- "error": {
### Canceled By: "message": "Index `doggo` already exists.",
"code": "index_already_exists",
---------------------------------------------------------------------- "type": "invalid_request",
### Enqueued At: "link": "https://docs.meilisearch.com/errors#index_already_exists"
[timestamp] [0,] },
[timestamp] [1,] "canceledBy": null,
[timestamp] [2,] "details": {
[timestamp] [3,] "IndexInfo": {
[timestamp] [4,] "primary_key": null
---------------------------------------------------------------------- }
### Started At: },
[timestamp] [0,] "status": "failed",
[timestamp] [1,] "kind": {
---------------------------------------------------------------------- "indexCreation": {
### Finished At: "index_uid": "doggo",
[timestamp] [0,] "primary_key": null
[timestamp] [1,] }
---------------------------------------------------------------------- }
### File Store: },
{
---------------------------------------------------------------------- "uid": 2,
"enqueuedAt": "[date]",
"startedAt": "[date]",
"finishedAt": "[date]",
"error": null,
"canceledBy": null,
"details": {
"IndexInfo": {
"primary_key": null
}
},
"status": "enqueued",
"kind": {
"indexCreation": {
"index_uid": "doggo",
"primary_key": null
}
}
},
{
"uid": 3,
"enqueuedAt": "[date]",
"startedAt": "[date]",
"finishedAt": "[date]",
"error": null,
"canceledBy": null,
"details": {
"IndexInfo": {
"primary_key": null
}
},
"status": "enqueued",
"kind": {
"indexCreation": {
"index_uid": "doggo",
"primary_key": null
}
}
},
{
"uid": 4,
"enqueuedAt": "[date]",
"startedAt": "[date]",
"finishedAt": "[date]",
"error": null,
"canceledBy": null,
"details": {
"TaskDeletion": {
"matched_tasks": 2,
"deleted_tasks": null,
"original_filter": "[filter]"
}
},
"status": "enqueued",
"kind": {
"taskDeletion": {
"query": "[query]",
"tasks": [
58,
48,
0,
0,
1,
0,
0,
0,
0,
0,
1,
0,
16,
0,
0,
0,
0,
0,
1,
0
]
}
}
}
]

View File

@ -1,46 +1,88 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
--- ---
### Autobatching Enabled = true [
### Processing Tasks: {
[] "uid": 2,
---------------------------------------------------------------------- "enqueuedAt": "[date]",
### All Tasks: "startedAt": "[date]",
2 {uid: 2, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggo", primary_key: None }} "finishedAt": "[date]",
3 {uid: 3, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggo", primary_key: None }} "error": null,
4 {uid: 4, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(2), original_filter: "?from=1,limit=2,status=succeeded,failed,canceled" }, kind: TaskDeletion { query: "?from=1,limit=2,status=succeeded,failed,canceled", tasks: RoaringBitmap<[0, 1]> }} "canceledBy": null,
---------------------------------------------------------------------- "details": {
### Status: "IndexInfo": {
enqueued [2,3,] "primary_key": null
succeeded [4,] }
failed [] },
---------------------------------------------------------------------- "status": "enqueued",
### Kind: "kind": {
"indexCreation" [2,3,] "indexCreation": {
"taskDeletion" [4,] "index_uid": "doggo",
---------------------------------------------------------------------- "primary_key": null
### Index Tasks: }
doggo [2,3,] }
---------------------------------------------------------------------- },
### Index Mapper: {
doggo: { number_of_documents: 0, field_distribution: {} } "uid": 3,
"enqueuedAt": "[date]",
---------------------------------------------------------------------- "startedAt": "[date]",
### Canceled By: "finishedAt": "[date]",
"error": null,
---------------------------------------------------------------------- "canceledBy": null,
### Enqueued At: "details": {
[timestamp] [2,] "IndexInfo": {
[timestamp] [3,] "primary_key": null
[timestamp] [4,] }
---------------------------------------------------------------------- },
### Started At: "status": "enqueued",
[timestamp] [4,] "kind": {
---------------------------------------------------------------------- "indexCreation": {
### Finished At: "index_uid": "doggo",
[timestamp] [4,] "primary_key": null
---------------------------------------------------------------------- }
### File Store: }
},
---------------------------------------------------------------------- {
"uid": 4,
"enqueuedAt": "[date]",
"startedAt": "[date]",
"finishedAt": "[date]",
"error": null,
"canceledBy": null,
"details": {
"TaskDeletion": {
"matched_tasks": 2,
"deleted_tasks": 2,
"original_filter": "[filter]"
}
},
"status": "succeeded",
"kind": {
"taskDeletion": {
"query": "[query]",
"tasks": [
58,
48,
0,
0,
1,
0,
0,
0,
0,
0,
1,
0,
16,
0,
0,
0,
0,
0,
1,
0
]
}
}
}
]

View File

@ -0,0 +1,90 @@
---
source: index-scheduler/src/lib.rs
---
[
{
"uid": 0,
"enqueuedAt": "[date]",
"startedAt": "[date]",
"finishedAt": "[date]",
"error": null,
"canceledBy": null,
"details": {
"IndexInfo": {
"primary_key": null
}
},
"status": "succeeded",
"kind": {
"indexCreation": {
"index_uid": "doggo",
"primary_key": null
}
}
},
{
"uid": 1,
"enqueuedAt": "[date]",
"startedAt": "[date]",
"finishedAt": "[date]",
"error": {
"message": "Index `doggo` already exists.",
"code": "index_already_exists",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#index_already_exists"
},
"canceledBy": null,
"details": {
"IndexInfo": {
"primary_key": null
}
},
"status": "failed",
"kind": {
"indexCreation": {
"index_uid": "doggo",
"primary_key": null
}
}
},
{
"uid": 2,
"enqueuedAt": "[date]",
"startedAt": "[date]",
"finishedAt": "[date]",
"error": null,
"canceledBy": null,
"details": {
"IndexInfo": {
"primary_key": null
}
},
"status": "enqueued",
"kind": {
"indexCreation": {
"index_uid": "doggo",
"primary_key": null
}
}
},
{
"uid": 3,
"enqueuedAt": "[date]",
"startedAt": "[date]",
"finishedAt": "[date]",
"error": null,
"canceledBy": null,
"details": {
"IndexInfo": {
"primary_key": null
}
},
"status": "enqueued",
"kind": {
"indexCreation": {
"index_uid": "doggo",
"primary_key": null
}
}
}
]