From 7b93ba40bd7cfe18cb481b0a0a8b1e8e7dc8e88d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Lecrenier?= Date: Thu, 27 Oct 2022 11:17:50 +0200 Subject: [PATCH 1/8] Reimplement task queries to account for special index swap rules --- index-scheduler/src/batch.rs | 10 +- index-scheduler/src/lib.rs | 251 ++++++++++++++---- .../all_tasks_processed.snap | 59 ++++ .../all_tasks_processed.snap | 46 ++++ .../all_tasks_processed.snap | 49 ++++ .../end.snap | 0 .../start.snap | 0 .../query_tasks_special_rules/start.snap | 42 +++ meilisearch-auth/src/lib.rs | 21 ++ meilisearch-http/src/routes/indexes/mod.rs | 13 +- meilisearch-http/src/routes/mod.rs | 9 +- meilisearch-http/src/routes/tasks.rs | 77 ++---- 12 files changed, 452 insertions(+), 125 deletions(-) create mode 100644 index-scheduler/src/snapshots/lib.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap create mode 100644 index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/all_tasks_processed.snap create mode 100644 index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/all_tasks_processed.snap rename index-scheduler/src/snapshots/lib.rs/{query_processing_tasks => query_tasks_simple}/end.snap (100%) rename index-scheduler/src/snapshots/lib.rs/{query_processing_tasks => query_tasks_simple}/start.snap (100%) create mode 100644 index-scheduler/src/snapshots/lib.rs/query_tasks_special_rules/start.snap diff --git a/index-scheduler/src/batch.rs b/index-scheduler/src/batch.rs index edbf2cae0..a1a45df9f 100644 --- a/index-scheduler/src/batch.rs +++ b/index-scheduler/src/batch.rs @@ -41,7 +41,7 @@ use uuid::Uuid; use crate::autobatcher::{self, BatchKind}; use crate::utils::{self, swap_index_uid_in_task}; -use crate::{Error, IndexScheduler, Query, Result, TaskId}; +use crate::{Error, IndexScheduler, Result, TaskId}; /// Represents a combination of tasks that can all be processed at the same time. /// @@ -854,12 +854,10 @@ impl IndexScheduler { return Err(Error::IndexNotFound(rhs.to_owned())); } - // 2. Get the task set for index = name. - let mut index_lhs_task_ids = - self.get_task_ids(&Query::default().with_index(lhs.to_owned()))?; + // 2. Get the task set for index = name that appeared before the index swap task + let mut index_lhs_task_ids = self.index_tasks(&wtxn, lhs)?; index_lhs_task_ids.remove_range(task_id..); - let mut index_rhs_task_ids = - self.get_task_ids(&Query::default().with_index(rhs.to_owned()))?; + let mut index_rhs_task_ids = self.index_tasks(&wtxn, rhs)?; index_rhs_task_ids.remove_range(task_id..); // 3. before_name -> new_name in the task's KindWithContent diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index b60ea8718..6b8037e4e 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -41,7 +41,7 @@ pub use error::Error; use file_store::FileStore; use meilisearch_types::error::ResponseError; use meilisearch_types::heed::types::{OwnedType, SerdeBincode, SerdeJson, Str}; -use meilisearch_types::heed::{self, Database, Env}; +use meilisearch_types::heed::{self, Database, Env, RoTxn}; use meilisearch_types::milli; use meilisearch_types::milli::documents::DocumentsBatchBuilder; use meilisearch_types::milli::update::IndexerConfig; @@ -393,6 +393,10 @@ impl IndexScheduler { Ok(this) } + pub fn read_txn(&self) -> Result { + self.env.read_txn().map_err(|e| e.into()) + } + /// Start the run loop for the given index scheduler. /// /// This function will execute in a different thread and must be called @@ -442,10 +446,8 @@ impl IndexScheduler { self.index_mapper.indexes(&rtxn) } - /// Return the task ids matched by the given query. - pub fn get_task_ids(&self, query: &Query) -> Result { - let rtxn = self.env.read_txn()?; - + /// Return the task ids matched by the given query from the index scheduler's point of view. + pub(crate) fn get_task_ids(&self, rtxn: &RoTxn, query: &Query) -> Result { let ProcessingTasks { started_at: started_at_processing, processing: processing_tasks } = self.processing_tasks.read().unwrap().clone(); @@ -561,10 +563,72 @@ impl IndexScheduler { Ok(tasks) } - /// Returns the tasks matched by the given query. - pub fn get_tasks(&self, query: Query) -> Result> { - let tasks = self.get_task_ids(&query)?; + /// Return true iff there is at least one task associated with this index + /// that is processing. + pub fn is_index_processing(&self, index: &str) -> Result { let rtxn = self.env.read_txn()?; + let processing_tasks = self.processing_tasks.read().unwrap().processing.clone(); + let index_tasks = self.index_tasks(&rtxn, index)?; + + let nbr_index_processing_tasks = processing_tasks.intersection_len(&index_tasks); + + Ok(nbr_index_processing_tasks > 0) + } + + /// Return the task ids matching the query from the user's point of view. + /// + /// There are two differences between an internal query and a query executed by + /// the user. + /// + /// 1. IndexSwap tasks are not publicly associated with any index, but they are associated + /// with many indexes internally. + /// 2. The user may not have the rights to access the tasks (internally) associated wuth all indexes. + pub fn get_task_ids_from_authorized_indexes( + &self, + rtxn: &RoTxn, + query: &Query, + authorized_indexes: &Option>, + ) -> Result { + let mut tasks = self.get_task_ids(rtxn, &query)?; + + // If the query contains a list of index_uid, then we must exclude IndexSwap tasks + // from the result (because it is not publicly associated with any index) + if query.index_uid.is_some() { + tasks -= self.get_kind(rtxn, Kind::IndexSwap)? + } + + // Any task that is internally associated with a non-authorized index + // must be discarded. + if let Some(authorized_indexes) = authorized_indexes { + let all_indexes_iter = self.index_tasks.iter(rtxn)?; + for iter_el in all_indexes_iter { + let (index, index_tasks) = iter_el?; + if !authorized_indexes.contains(&index.to_owned()) { + tasks -= index_tasks; + } + } + } + + Ok(tasks) + } + + /// Return the tasks matching the query from the user's point of view. + /// + /// There are two differences between an internal query and a query executed by + /// the user. + /// + /// 1. IndexSwap tasks are not publicly associated with any index, but they are associated + /// with many indexes internally. + /// 2. The user may not have the rights to access the tasks (internally) associated wuth all indexes. + pub fn get_tasks_from_authorized_indexes( + &self, + query: Query, + authorized_indexes: Option>, + ) -> Result> { + let rtxn = self.env.read_txn()?; + + let tasks = + self.get_task_ids_from_authorized_indexes(&rtxn, &query, &authorized_indexes)?; let tasks = self.get_existing_tasks( &rtxn, @@ -1187,12 +1251,7 @@ mod tests { handle.wait_till(Breakpoint::AfterProcessing); index_scheduler.assert_internally_consistent(); - let mut tasks = index_scheduler.get_tasks(Query::default()).unwrap(); - tasks.reverse(); - assert_eq!(tasks.len(), 3); - assert_eq!(tasks[0].status, Status::Succeeded); - assert_eq!(tasks[1].status, Status::Succeeded); - assert_eq!(tasks[2].status, Status::Succeeded); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "all_tasks_processed"); } #[test] @@ -1231,13 +1290,7 @@ mod tests { handle.wait_till(Breakpoint::AfterProcessing); index_scheduler.assert_internally_consistent(); - let mut tasks = index_scheduler.get_tasks(Query::default()).unwrap(); - tasks.reverse(); - assert_eq!(tasks.len(), 4); - assert_eq!(tasks[0].status, Status::Succeeded); - assert_eq!(tasks[1].status, Status::Succeeded); - assert_eq!(tasks[2].status, Status::Succeeded); - assert_eq!(tasks[3].status, Status::Succeeded); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "all_tasks_processed"); } #[test] @@ -1493,15 +1546,7 @@ mod tests { index_scheduler.assert_internally_consistent(); } - let mut tasks = index_scheduler.get_tasks(Query::default()).unwrap(); - tasks.reverse(); - assert_eq!(tasks.len(), 6); - assert_eq!(tasks[0].status, Status::Succeeded); - assert_eq!(tasks[1].status, Status::Succeeded); - assert_eq!(tasks[2].status, Status::Succeeded); - assert_eq!(tasks[3].status, Status::Succeeded); - assert_eq!(tasks[4].status, Status::Succeeded); - assert_eq!(tasks[5].status, Status::Succeeded); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "all_tasks_processed"); } #[test] @@ -2054,37 +2099,45 @@ mod tests { snapshot!(snapshot_index_scheduler(&index_scheduler), name: "finished"); + let rtxn = index_scheduler.env.read_txn().unwrap(); let query = Query { limit: Some(0), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[]"); let query = Query { limit: Some(1), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[2,]"); let query = Query { limit: Some(2), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[1,2,]"); let query = Query { from: Some(1), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[0,1,]"); let query = Query { from: Some(2), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[0,1,2,]"); let query = Query { from: Some(1), limit: Some(1), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[1,]"); let query = Query { from: Some(1), limit: Some(2), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[0,1,]"); } #[test] - fn query_processing_tasks() { + fn query_tasks_simple() { let start_time = OffsetDateTime::now_utc(); let (index_scheduler, handle) = @@ -2101,19 +2154,24 @@ mod tests { handle.wait_till(Breakpoint::BatchCreated); + let rtxn = index_scheduler.env.read_txn().unwrap(); + let query = Query { status: Some(vec![Status::Processing]), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[0,]"); // only the processing tasks in the first tick let query = Query { status: Some(vec![Status::Enqueued]), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[1,2,]"); // only the enqueued tasks in the first tick let query = Query { status: Some(vec![Status::Enqueued, Status::Processing]), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[0,1,2,]"); // both enqueued and processing tasks in the first tick let query = Query { @@ -2121,7 +2179,8 @@ mod tests { after_started_at: Some(start_time), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); // both enqueued and processing tasks in the first tick, but limited to those with a started_at // that comes after the start of the test, which should excludes the enqueued tasks snapshot!(snapshot_bitmap(&tasks), @"[0,]"); @@ -2131,7 +2190,8 @@ mod tests { before_started_at: Some(start_time), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); // both enqueued and processing tasks in the first tick, but limited to those with a started_at // that comes before the start of the test, which should excludes all of them snapshot!(snapshot_bitmap(&tasks), @"[]"); @@ -2142,7 +2202,8 @@ mod tests { before_started_at: Some(start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); // both enqueued and processing tasks in the first tick, but limited to those with a started_at // that comes after the start of the test and before one minute after the start of the test, // which should exclude the enqueued tasks and include the only processing task @@ -2150,6 +2211,8 @@ mod tests { handle.wait_till(Breakpoint::BatchCreated); + let rtxn = index_scheduler.env.read_txn().unwrap(); + let second_start_time = OffsetDateTime::now_utc(); let query = Query { @@ -2158,7 +2221,8 @@ mod tests { before_started_at: Some(start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); // both succeeded and processing tasks in the first tick, but limited to those with a started_at // that comes after the start of the test and before one minute after the start of the test, // which should include all tasks @@ -2169,7 +2233,8 @@ mod tests { before_started_at: Some(start_time), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); // both succeeded and processing tasks in the first tick, but limited to those with a started_at // that comes before the start of the test, which should exclude all tasks snapshot!(snapshot_bitmap(&tasks), @"[]"); @@ -2180,7 +2245,8 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); // both succeeded and processing tasks in the first tick, but limited to those with a started_at // that comes after the start of the second part of the test and before one minute after the // second start of the test, which should exclude all tasks @@ -2188,7 +2254,11 @@ mod tests { // now we make one more batch, the started_at field of the new tasks will be past `second_start_time` handle.wait_till(Breakpoint::BatchCreated); - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + + let rtxn = index_scheduler.env.read_txn().unwrap(); + + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); // we run the same query to verify that, and indeed find that the last task is matched snapshot!(snapshot_bitmap(&tasks), @"[2,]"); @@ -2198,15 +2268,19 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); // enqueued, succeeded, or processing tasks started after the second part of the test, should // again only return the last task snapshot!(snapshot_bitmap(&tasks), @"[2,]"); handle.wait_till(Breakpoint::AfterProcessing); + let rtxn = index_scheduler.read_txn().unwrap(); + // now the last task should have failed snapshot!(snapshot_index_scheduler(&index_scheduler), name: "end"); - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); // so running the last query should return nothing snapshot!(snapshot_bitmap(&tasks), @"[]"); @@ -2216,7 +2290,8 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); // but the same query on failed tasks should return the last task snapshot!(snapshot_bitmap(&tasks), @"[2,]"); @@ -2226,7 +2301,8 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); // but the same query on failed tasks should return the last task snapshot!(snapshot_bitmap(&tasks), @"[2,]"); @@ -2237,7 +2313,8 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); // same query but with an invalid uid snapshot!(snapshot_bitmap(&tasks), @"[]"); @@ -2248,11 +2325,77 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler.get_task_ids(&query).unwrap(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); // same query but with a valid uid snapshot!(snapshot_bitmap(&tasks), @"[2,]"); } + #[test] + fn query_tasks_special_rules() { + let (index_scheduler, handle) = + IndexScheduler::test(true, vec![(3, FailureLocation::InsideProcessBatch)]); + + let kind = index_creation_task("catto", "mouse"); + let _task = index_scheduler.register(kind).unwrap(); + let kind = index_creation_task("doggo", "sheep"); + let _task = index_scheduler.register(kind).unwrap(); + let kind = KindWithContent::IndexSwap { + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }], + }; + let _task = index_scheduler.register(kind).unwrap(); + let kind = KindWithContent::IndexSwap { + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()) }], + }; + let _task = index_scheduler.register(kind).unwrap(); + + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "start"); + + handle.wait_till(Breakpoint::BatchCreated); + + let rtxn = index_scheduler.env.read_txn().unwrap(); + + let query = Query { index_uid: Some(vec!["catto".to_owned()]), ..Default::default() }; + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); + // only the first task associated with catto is returned, the indexSwap tasks are excluded! + snapshot!(snapshot_bitmap(&tasks), @"[0,]"); + + let query = Query { index_uid: Some(vec!["catto".to_owned()]), ..Default::default() }; + let tasks = index_scheduler + .get_task_ids_from_authorized_indexes(&rtxn, &query, &Some(vec!["doggo".to_owned()])) + .unwrap(); + // we have asked for only the tasks associated with catto, but are only authorized to retrieve the tasks + // associated with doggo -> empty result + snapshot!(snapshot_bitmap(&tasks), @"[]"); + + let query = Query::default(); + let tasks = index_scheduler + .get_task_ids_from_authorized_indexes(&rtxn, &query, &Some(vec!["doggo".to_owned()])) + .unwrap(); + // we asked for all the tasks, but we are only authorized to retrieve the doggo tasks + // -> only the index creation of doggo should be returned + snapshot!(snapshot_bitmap(&tasks), @"[1,]"); + + let query = Query::default(); + let tasks = index_scheduler + .get_task_ids_from_authorized_indexes( + &rtxn, + &query, + &Some(vec!["catto".to_owned(), "doggo".to_owned()]), + ) + .unwrap(); + // we asked for all the tasks, but we are only authorized to retrieve the doggo and catto tasks + // -> all tasks except the swap of catto with whalo are returned + snapshot!(snapshot_bitmap(&tasks), @"[0,1,2,]"); + + let query = Query::default(); + let tasks = + index_scheduler.get_task_ids_from_authorized_indexes(&rtxn, &query, &None).unwrap(); + // we asked for all the tasks with all index authorized -> all tasks returned + snapshot!(snapshot_bitmap(&tasks), @"[0,1,2,3,]"); + } + #[test] fn fail_in_create_batch_for_index_creation() { let (index_scheduler, handle) = diff --git a/index-scheduler/src/snapshots/lib.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap b/index-scheduler/src/snapshots/lib.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap new file mode 100644 index 000000000..8541c7c1b --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap @@ -0,0 +1,59 @@ +--- +source: index-scheduler/src/lib.rs +--- +### Autobatching Enabled = true +### Processing Tasks: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +2 {uid: 2, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "girafos", primary_key: None }} +3 {uid: 3, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} +4 {uid: 4, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "cattos" }} +5 {uid: 5, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "girafos" }} +---------------------------------------------------------------------- +### Status: +enqueued [] +succeeded [0,1,2,3,4,5,] +---------------------------------------------------------------------- +### Kind: +"documentDeletion" [3,4,5,] +"indexCreation" [0,1,2,] +---------------------------------------------------------------------- +### Index Tasks: +cattos [1,4,] +doggos [0,3,] +girafos [2,5,] +---------------------------------------------------------------------- +### Index Mapper: +["cattos", "doggos", "girafos"] +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +[timestamp] [4,] +[timestamp] [5,] +---------------------------------------------------------------------- +### Started At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +[timestamp] [4,] +[timestamp] [5,] +---------------------------------------------------------------------- +### Finished At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +[timestamp] [4,] +[timestamp] [5,] +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- + diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/all_tasks_processed.snap b/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/all_tasks_processed.snap new file mode 100644 index 000000000..c75964581 --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/all_tasks_processed.snap @@ -0,0 +1,46 @@ +--- +source: index-scheduler/src/lib.rs +--- +### Autobatching Enabled = true +### Processing Tasks: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +2 {uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: IndexDeletion { index_uid: "doggos" }} +---------------------------------------------------------------------- +### Status: +enqueued [] +succeeded [0,1,2,] +---------------------------------------------------------------------- +### Kind: +"indexCreation" [0,1,] +"indexDeletion" [2,] +---------------------------------------------------------------------- +### Index Tasks: +cattos [1,] +doggos [0,2,] +---------------------------------------------------------------------- +### Index Mapper: +["cattos"] +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +---------------------------------------------------------------------- +### Started At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +---------------------------------------------------------------------- +### Finished At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- + diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/all_tasks_processed.snap b/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/all_tasks_processed.snap new file mode 100644 index 000000000..44ce75ebb --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/all_tasks_processed.snap @@ -0,0 +1,49 @@ +--- +source: index-scheduler/src/lib.rs +--- +### Autobatching Enabled = false +### Processing Tasks: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} +2 {uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} +3 {uid: 3, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} +---------------------------------------------------------------------- +### Status: +enqueued [] +succeeded [0,1,2,3,] +---------------------------------------------------------------------- +### Kind: +"documentDeletion" [1,2,3,] +"indexCreation" [0,] +---------------------------------------------------------------------- +### Index Tasks: +doggos [0,1,2,3,] +---------------------------------------------------------------------- +### Index Mapper: +["doggos"] +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +---------------------------------------------------------------------- +### Started At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +---------------------------------------------------------------------- +### Finished At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- + diff --git a/index-scheduler/src/snapshots/lib.rs/query_processing_tasks/end.snap b/index-scheduler/src/snapshots/lib.rs/query_tasks_simple/end.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/query_processing_tasks/end.snap rename to index-scheduler/src/snapshots/lib.rs/query_tasks_simple/end.snap diff --git a/index-scheduler/src/snapshots/lib.rs/query_processing_tasks/start.snap b/index-scheduler/src/snapshots/lib.rs/query_tasks_simple/start.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/query_processing_tasks/start.snap rename to index-scheduler/src/snapshots/lib.rs/query_tasks_simple/start.snap diff --git a/index-scheduler/src/snapshots/lib.rs/query_tasks_special_rules/start.snap b/index-scheduler/src/snapshots/lib.rs/query_tasks_special_rules/start.snap new file mode 100644 index 000000000..caab362d7 --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/query_tasks_special_rules/start.snap @@ -0,0 +1,42 @@ +--- +source: index-scheduler/src/lib.rs +--- +### Autobatching Enabled = true +### Processing Tasks: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} +3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }} +---------------------------------------------------------------------- +### Status: +enqueued [0,1,2,3,] +---------------------------------------------------------------------- +### Kind: +"indexCreation" [0,1,] +"indexSwap" [2,3,] +---------------------------------------------------------------------- +### Index Tasks: +catto [0,2,3,] +doggo [1,2,] +whalo [3,] +---------------------------------------------------------------------- +### Index Mapper: +[] +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +---------------------------------------------------------------------- +### Started At: +---------------------------------------------------------------------- +### Finished At: +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- + diff --git a/meilisearch-auth/src/lib.rs b/meilisearch-auth/src/lib.rs index 12d810aec..020a2821c 100644 --- a/meilisearch-auth/src/lib.rs +++ b/meilisearch-auth/src/lib.rs @@ -215,6 +215,27 @@ impl SearchRules { } } } + + /// Return the list of indexes such that `self.is_index_authorized(index) == true`, + /// or `None` if all indexes satisfy this condition. + pub fn authorized_indexes(&self) -> Option> { + match self { + SearchRules::Set(set) => { + if set.contains("*") { + None + } else { + Some(set.iter().cloned().collect()) + } + } + SearchRules::Map(map) => { + if map.contains_key("*") { + None + } else { + Some(map.keys().cloned().collect()) + } + } + } + } } impl IntoIterator for SearchRules { diff --git a/meilisearch-http/src/routes/indexes/mod.rs b/meilisearch-http/src/routes/indexes/mod.rs index d370483c6..e8fca0cf8 100644 --- a/meilisearch-http/src/routes/indexes/mod.rs +++ b/meilisearch-http/src/routes/indexes/mod.rs @@ -1,11 +1,11 @@ use actix_web::web::Data; use actix_web::{web, HttpRequest, HttpResponse}; -use index_scheduler::{IndexScheduler, Query}; +use index_scheduler::IndexScheduler; use log::debug; use meilisearch_types::error::ResponseError; use meilisearch_types::index_uid::IndexUid; use meilisearch_types::milli::{self, FieldDistribution, Index}; -use meilisearch_types::tasks::{KindWithContent, Status}; +use meilisearch_types::tasks::KindWithContent; use serde::{Deserialize, Serialize}; use serde_json::json; use time::OffsetDateTime; @@ -202,14 +202,7 @@ impl IndexStats { index_uid: String, ) -> Result { // we check if there is currently a task processing associated with this index. - let processing_task = index_scheduler.get_tasks(Query { - status: Some(vec![Status::Processing]), - index_uid: Some(vec![index_uid.clone()]), - limit: Some(1), - ..Query::default() - })?; - let is_processing = !processing_task.is_empty(); - + let is_processing = index_scheduler.is_index_processing(&index_uid)?; let index = index_scheduler.index(&index_uid)?; let rtxn = index.read_txn()?; Ok(IndexStats { diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 4463aee5e..81e100214 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -270,11 +270,10 @@ pub fn create_all_stats( let mut last_task: Option = None; let mut indexes = BTreeMap::new(); let mut database_size = 0; - let processing_task = index_scheduler.get_tasks(Query { - status: Some(vec![Status::Processing]), - limit: Some(1), - ..Query::default() - })?; + let processing_task = index_scheduler.get_tasks_from_authorized_indexes( + Query { status: Some(vec![Status::Processing]), limit: Some(1), ..Query::default() }, + search_rules.authorized_indexes(), + )?; let processing_index = processing_task.first().and_then(|task| task.index_uid()); for (name, index) in index_scheduler.indexes()? { if !search_rules.is_index_authorized(&name) { diff --git a/meilisearch-http/src/routes/tasks.rs b/meilisearch-http/src/routes/tasks.rs index 59f1a1f68..1d4b6a054 100644 --- a/meilisearch-http/src/routes/tasks.rs +++ b/meilisearch-http/src/routes/tasks.rs @@ -291,8 +291,11 @@ async fn cancel_tasks( return Err(index_scheduler::Error::TaskCancelationWithEmptyQuery.into()); } - let filtered_query = filter_out_inaccessible_indexes_from_query(&index_scheduler, &query); - let tasks = index_scheduler.get_task_ids(&filtered_query)?; + let tasks = index_scheduler.get_task_ids_from_authorized_indexes( + &index_scheduler.read_txn()?, + &query, + &index_scheduler.filters().search_rules.authorized_indexes(), + )?; let task_cancelation = KindWithContent::TaskCancelation { query: req.query_string().to_string(), tasks }; @@ -348,8 +351,11 @@ async fn delete_tasks( return Err(index_scheduler::Error::TaskDeletionWithEmptyQuery.into()); } - let filtered_query = filter_out_inaccessible_indexes_from_query(&index_scheduler, &query); - let tasks = index_scheduler.get_task_ids(&filtered_query)?; + let tasks = index_scheduler.get_task_ids_from_authorized_indexes( + &index_scheduler.read_txn()?, + &query, + &index_scheduler.filters().search_rules.authorized_indexes(), + )?; let task_deletion = KindWithContent::TaskDeletion { query: req.query_string().to_string(), tasks }; @@ -425,10 +431,15 @@ async fn get_tasks( before_finished_at, after_finished_at, }; - let query = filter_out_inaccessible_indexes_from_query(&index_scheduler, &query); - let mut tasks_results: Vec = - index_scheduler.get_tasks(query)?.into_iter().map(|t| TaskView::from_task(&t)).collect(); + let mut tasks_results: Vec = index_scheduler + .get_tasks_from_authorized_indexes( + query, + index_scheduler.filters().search_rules.authorized_indexes(), + )? + .into_iter() + .map(|t| TaskView::from_task(&t)) + .collect(); // If we were able to fetch the number +1 tasks we asked // it means that there is more to come. @@ -454,17 +465,16 @@ async fn get_task( analytics.publish("Tasks Seen".to_string(), json!({ "per_task_uid": true }), Some(&req)); - let search_rules = &index_scheduler.filters().search_rules; - let mut filters = index_scheduler::Query::default(); - if !search_rules.is_index_authorized("*") { - for (index, _policy) in search_rules.clone() { - filters = filters.with_index(index); - } - } + let mut query = index_scheduler::Query::default(); + query.uid = Some(vec![task_id]); - filters.uid = Some(vec![task_id]); - - if let Some(task) = index_scheduler.get_tasks(filters)?.first() { + if let Some(task) = index_scheduler + .get_tasks_from_authorized_indexes( + query, + index_scheduler.filters().search_rules.authorized_indexes(), + )? + .first() + { let task_view = TaskView::from_task(task); Ok(HttpResponse::Ok().json(task_view)) } else { @@ -472,39 +482,6 @@ async fn get_task( } } -fn filter_out_inaccessible_indexes_from_query( - index_scheduler: &GuardedData, Data>, - query: &Query, -) -> Query { - let mut query = query.clone(); - - // First remove all indexes from the query, we will add them back later - let indexes = query.index_uid.take(); - - let search_rules = &index_scheduler.filters().search_rules; - - // We filter on potential indexes and make sure that the search filter - // restrictions are also applied. - match indexes { - Some(indexes) => { - for name in indexes.iter() { - if search_rules.is_index_authorized(name) { - query = query.with_index(name.to_string()); - } - } - } - None => { - if !search_rules.is_index_authorized("*") { - for (index, _policy) in search_rules.clone() { - query = query.with_index(index.to_string()); - } - } - } - }; - - query -} - pub(crate) mod date_deserializer { use time::format_description::well_known::Rfc3339; use time::macros::format_description; From 4f4fc20acfe52115c8cca178193c895f3b150f2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Lecrenier?= Date: Thu, 27 Oct 2022 13:00:30 +0200 Subject: [PATCH 2/8] Make clippy happy --- index-scheduler/src/batch.rs | 4 ++-- index-scheduler/src/lib.rs | 16 ++++++++-------- meilisearch-http/src/routes/tasks.rs | 6 ++++-- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/index-scheduler/src/batch.rs b/index-scheduler/src/batch.rs index a1a45df9f..90ac4f8df 100644 --- a/index-scheduler/src/batch.rs +++ b/index-scheduler/src/batch.rs @@ -855,9 +855,9 @@ impl IndexScheduler { } // 2. Get the task set for index = name that appeared before the index swap task - let mut index_lhs_task_ids = self.index_tasks(&wtxn, lhs)?; + let mut index_lhs_task_ids = self.index_tasks(wtxn, lhs)?; index_lhs_task_ids.remove_range(task_id..); - let mut index_rhs_task_ids = self.index_tasks(&wtxn, rhs)?; + let mut index_rhs_task_ids = self.index_tasks(wtxn, rhs)?; index_rhs_task_ids.remove_range(task_id..); // 3. before_name -> new_name in the task's KindWithContent diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 6b8037e4e..67ce09ec0 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -451,7 +451,7 @@ impl IndexScheduler { let ProcessingTasks { started_at: started_at_processing, processing: processing_tasks } = self.processing_tasks.read().unwrap().clone(); - let mut tasks = self.all_task_ids(&rtxn)?; + let mut tasks = self.all_task_ids(rtxn)?; if let Some(from) = &query.from { tasks.remove_range(from.saturating_add(1)..); @@ -465,7 +465,7 @@ impl IndexScheduler { Status::Processing => { status_tasks |= &processing_tasks; } - status => status_tasks |= &self.get_status(&rtxn, *status)?, + status => status_tasks |= &self.get_status(rtxn, *status)?, }; } if !status.contains(&Status::Processing) { @@ -482,7 +482,7 @@ impl IndexScheduler { if let Some(kind) = &query.kind { let mut kind_tasks = RoaringBitmap::new(); for kind in kind { - kind_tasks |= self.get_kind(&rtxn, *kind)?; + kind_tasks |= self.get_kind(rtxn, *kind)?; } tasks &= &kind_tasks; } @@ -490,7 +490,7 @@ impl IndexScheduler { if let Some(index) = &query.index_uid { let mut index_tasks = RoaringBitmap::new(); for index in index { - index_tasks |= self.index_tasks(&rtxn, index)?; + index_tasks |= self.index_tasks(rtxn, index)?; } tasks &= &index_tasks; } @@ -531,7 +531,7 @@ impl IndexScheduler { }; keep_tasks_within_datetimes( - &rtxn, + rtxn, &mut filtered_non_processing_tasks, self.started_at, query.after_started_at, @@ -541,7 +541,7 @@ impl IndexScheduler { }; keep_tasks_within_datetimes( - &rtxn, + rtxn, &mut tasks, self.enqueued_at, query.after_enqueued_at, @@ -549,7 +549,7 @@ impl IndexScheduler { )?; keep_tasks_within_datetimes( - &rtxn, + rtxn, &mut tasks, self.finished_at, query.after_finished_at, @@ -589,7 +589,7 @@ impl IndexScheduler { query: &Query, authorized_indexes: &Option>, ) -> Result { - let mut tasks = self.get_task_ids(rtxn, &query)?; + let mut tasks = self.get_task_ids(rtxn, query)?; // If the query contains a list of index_uid, then we must exclude IndexSwap tasks // from the result (because it is not publicly associated with any index) diff --git a/meilisearch-http/src/routes/tasks.rs b/meilisearch-http/src/routes/tasks.rs index 1d4b6a054..110f6d40b 100644 --- a/meilisearch-http/src/routes/tasks.rs +++ b/meilisearch-http/src/routes/tasks.rs @@ -465,8 +465,10 @@ async fn get_task( analytics.publish("Tasks Seen".to_string(), json!({ "per_task_uid": true }), Some(&req)); - let mut query = index_scheduler::Query::default(); - query.uid = Some(vec![task_id]); + let query = index_scheduler::Query { + uid: Some(vec![task_id]), + ..Query::default() + }; if let Some(task) = index_scheduler .get_tasks_from_authorized_indexes( From 4d9e9f4a9d0c98e152f6e870baae7eaa2d5391ed Mon Sep 17 00:00:00 2001 From: Irevoire Date: Thu, 27 Oct 2022 13:12:42 +0200 Subject: [PATCH 3/8] isolate the search in another task In case there is a failure on milli's side that should avoid blocking the tokio main thread --- meilisearch-http/src/routes/indexes/search.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/routes/indexes/search.rs b/meilisearch-http/src/routes/indexes/search.rs index 8c901035d..af5da44a0 100644 --- a/meilisearch-http/src/routes/indexes/search.rs +++ b/meilisearch-http/src/routes/indexes/search.rs @@ -153,7 +153,7 @@ pub async fn search_with_url_query( let mut aggregate = SearchAggregator::from_query(&query, &req); let index = index_scheduler.index(&index_uid)?; - let search_result = perform_search(&index, query); + let search_result = tokio::task::spawn_blocking(move || perform_search(&index, query)).await?; if let Ok(ref search_result) = search_result { aggregate.succeed(search_result); } @@ -185,7 +185,7 @@ pub async fn search_with_post( let mut aggregate = SearchAggregator::from_query(&query, &req); let index = index_scheduler.index(&index_uid)?; - let search_result = perform_search(&index, query); + let search_result = tokio::task::spawn_blocking(move || perform_search(&index, query)).await?; if let Ok(ref search_result) = search_result { aggregate.succeed(search_result); } From 68f80dbacf77152fb4ade3e654b18f8f53691955 Mon Sep 17 00:00:00 2001 From: curquiza Date: Thu, 27 Oct 2022 11:35:44 +0000 Subject: [PATCH 4/8] Update version for the next release (v0.30.0) in Cargo.toml files --- Cargo.lock | 16 ++++++++-------- dump/Cargo.toml | 2 +- file-store/Cargo.toml | 2 +- index-scheduler/Cargo.toml | 2 +- meili-snap/Cargo.toml | 2 +- meilisearch-auth/Cargo.toml | 2 +- meilisearch-http/Cargo.toml | 2 +- meilisearch-types/Cargo.toml | 2 +- permissive-json-pointer/Cargo.toml | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f43356f6..01eb9cd36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1101,7 +1101,7 @@ dependencies = [ [[package]] name = "dump" -version = "0.29.0" +version = "0.30.0" dependencies = [ "anyhow", "big_s", @@ -1310,7 +1310,7 @@ dependencies = [ [[package]] name = "file-store" -version = "0.1.0" +version = "0.30.0" dependencies = [ "faux", "tempfile", @@ -1767,7 +1767,7 @@ dependencies = [ [[package]] name = "index-scheduler" -version = "0.1.0" +version = "0.30.0" dependencies = [ "anyhow", "big_s", @@ -2257,7 +2257,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "meili-snap" -version = "0.1.0" +version = "0.30.0" dependencies = [ "insta", "md5", @@ -2266,7 +2266,7 @@ dependencies = [ [[package]] name = "meilisearch-auth" -version = "0.29.1" +version = "0.30.0" dependencies = [ "enum-iterator", "hmac", @@ -2283,7 +2283,7 @@ dependencies = [ [[package]] name = "meilisearch-http" -version = "0.29.1" +version = "0.30.0" dependencies = [ "actix-cors", "actix-http", @@ -2366,7 +2366,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "0.29.1" +version = "0.30.0" dependencies = [ "actix-web", "anyhow", @@ -2747,7 +2747,7 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "permissive-json-pointer" -version = "0.29.1" +version = "0.30.0" dependencies = [ "big_s", "serde_json", diff --git a/dump/Cargo.toml b/dump/Cargo.toml index c7bf76879..c5dc10949 100644 --- a/dump/Cargo.toml +++ b/dump/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dump" -version = "0.29.0" +version = "0.30.0" edition = "2021" [dependencies] diff --git a/file-store/Cargo.toml b/file-store/Cargo.toml index 3dafcbecc..0110a00b1 100644 --- a/file-store/Cargo.toml +++ b/file-store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-store" -version = "0.1.0" +version = "0.30.0" edition = "2021" [dependencies] diff --git a/index-scheduler/Cargo.toml b/index-scheduler/Cargo.toml index 3b29d158f..e46712703 100644 --- a/index-scheduler/Cargo.toml +++ b/index-scheduler/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "index-scheduler" -version = "0.1.0" +version = "0.30.0" edition = "2021" [dependencies] diff --git a/meili-snap/Cargo.toml b/meili-snap/Cargo.toml index 292b60cfa..6706bf693 100644 --- a/meili-snap/Cargo.toml +++ b/meili-snap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meili-snap" -version = "0.1.0" +version = "0.30.0" edition = "2021" [dependencies] diff --git a/meilisearch-auth/Cargo.toml b/meilisearch-auth/Cargo.toml index e673c2f9a..f0b73b539 100644 --- a/meilisearch-auth/Cargo.toml +++ b/meilisearch-auth/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meilisearch-auth" -version = "0.29.1" +version = "0.30.0" edition = "2021" [dependencies] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index dd1c9d7e5..6a4cc0336 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -4,7 +4,7 @@ description = "Meilisearch HTTP server" edition = "2021" license = "MIT" name = "meilisearch-http" -version = "0.29.1" +version = "0.30.0" [[bin]] name = "meilisearch" diff --git a/meilisearch-types/Cargo.toml b/meilisearch-types/Cargo.toml index 62d0e6ebb..3b5346438 100644 --- a/meilisearch-types/Cargo.toml +++ b/meilisearch-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meilisearch-types" -version = "0.29.1" +version = "0.30.0" authors = ["marin "] edition = "2021" diff --git a/permissive-json-pointer/Cargo.toml b/permissive-json-pointer/Cargo.toml index e504a1b68..b036a0c88 100644 --- a/permissive-json-pointer/Cargo.toml +++ b/permissive-json-pointer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "permissive-json-pointer" -version = "0.29.1" +version = "0.30.0" edition = "2021" description = "A permissive json pointer" readme = "README.md" From 44d6f3e7a02f924f95e32f5a705e6e5b6bfb1129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 27 Oct 2022 13:50:05 +0200 Subject: [PATCH 5/8] Reconsider the Windows tests --- meilisearch-http/tests/auth/tenant_token.rs | 8 -------- meilisearch-http/tests/dumps/mod.rs | 11 ----------- meilisearch-http/tests/index/delete_index.rs | 1 - 3 files changed, 20 deletions(-) diff --git a/meilisearch-http/tests/auth/tenant_token.rs b/meilisearch-http/tests/auth/tenant_token.rs index 3206a6553..fbf9d2b49 100644 --- a/meilisearch-http/tests/auth/tenant_token.rs +++ b/meilisearch-http/tests/auth/tenant_token.rs @@ -203,7 +203,6 @@ macro_rules! compute_forbidden_search { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn search_authorized_simple_token() { let tenant_tokens = vec![ hashmap! { @@ -252,7 +251,6 @@ async fn search_authorized_simple_token() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn search_authorized_filter_token() { let tenant_tokens = vec![ hashmap! { @@ -306,7 +304,6 @@ async fn search_authorized_filter_token() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn filter_search_authorized_filter_token() { let tenant_tokens = vec![ hashmap! { @@ -360,7 +357,6 @@ async fn filter_search_authorized_filter_token() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn error_search_token_forbidden_parent_key() { let tenant_tokens = vec![ hashmap! { @@ -393,7 +389,6 @@ async fn error_search_token_forbidden_parent_key() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn error_search_forbidden_token() { let tenant_tokens = vec![ // bad index @@ -448,7 +443,6 @@ async fn error_search_forbidden_token() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn error_access_forbidden_routes() { let mut server = Server::new_auth().await; server.use_api_key("MASTER_KEY"); @@ -483,7 +477,6 @@ async fn error_access_forbidden_routes() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn error_access_expired_parent_key() { use std::{thread, time}; let mut server = Server::new_auth().await; @@ -523,7 +516,6 @@ async fn error_access_expired_parent_key() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn error_access_modified_token() { let mut server = Server::new_auth().await; server.use_api_key("MASTER_KEY"); diff --git a/meilisearch-http/tests/dumps/mod.rs b/meilisearch-http/tests/dumps/mod.rs index fa0b929a3..b219f8318 100644 --- a/meilisearch-http/tests/dumps/mod.rs +++ b/meilisearch-http/tests/dumps/mod.rs @@ -8,7 +8,6 @@ use crate::common::{default_settings, GetAllDocumentsOptions, Server}; // all the following test are ignored on windows. See #2364 #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn import_dump_v1() { let temp = tempfile::tempdir().unwrap(); @@ -25,7 +24,6 @@ async fn import_dump_v1() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn import_dump_v2_movie_raw() { let temp = tempfile::tempdir().unwrap(); @@ -87,7 +85,6 @@ async fn import_dump_v2_movie_raw() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn import_dump_v2_movie_with_settings() { let temp = tempfile::tempdir().unwrap(); @@ -151,7 +148,6 @@ async fn import_dump_v2_movie_with_settings() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn import_dump_v2_rubygems_with_settings() { let temp = tempfile::tempdir().unwrap(); @@ -215,7 +211,6 @@ async fn import_dump_v2_rubygems_with_settings() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn import_dump_v3_movie_raw() { let temp = tempfile::tempdir().unwrap(); @@ -277,7 +272,6 @@ async fn import_dump_v3_movie_raw() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn import_dump_v3_movie_with_settings() { let temp = tempfile::tempdir().unwrap(); @@ -341,7 +335,6 @@ async fn import_dump_v3_movie_with_settings() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn import_dump_v3_rubygems_with_settings() { let temp = tempfile::tempdir().unwrap(); @@ -405,7 +398,6 @@ async fn import_dump_v3_rubygems_with_settings() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn import_dump_v4_movie_raw() { let temp = tempfile::tempdir().unwrap(); @@ -467,7 +459,6 @@ async fn import_dump_v4_movie_raw() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn import_dump_v4_movie_with_settings() { let temp = tempfile::tempdir().unwrap(); @@ -531,7 +522,6 @@ async fn import_dump_v4_movie_with_settings() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn import_dump_v4_rubygems_with_settings() { let temp = tempfile::tempdir().unwrap(); @@ -595,7 +585,6 @@ async fn import_dump_v4_rubygems_with_settings() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn import_dump_v5() { let temp = tempfile::tempdir().unwrap(); diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs index f3cdf6631..b6efc7a68 100644 --- a/meilisearch-http/tests/index/delete_index.rs +++ b/meilisearch-http/tests/index/delete_index.rs @@ -44,7 +44,6 @@ async fn error_delete_unexisting_index() { } #[actix_rt::test] -#[cfg_attr(target_os = "windows", ignore)] async fn loop_delete_add_documents() { let server = Server::new().await; let index = server.index("test"); From e67673bd1231e69ccd1c014a992386cc9c496650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 27 Oct 2022 14:34:45 +0200 Subject: [PATCH 6/8] Ingore the dumps v1 test on Windows --- meilisearch-http/tests/dumps/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/meilisearch-http/tests/dumps/mod.rs b/meilisearch-http/tests/dumps/mod.rs index b219f8318..3f783a1e3 100644 --- a/meilisearch-http/tests/dumps/mod.rs +++ b/meilisearch-http/tests/dumps/mod.rs @@ -8,6 +8,7 @@ use crate::common::{default_settings, GetAllDocumentsOptions, Server}; // all the following test are ignored on windows. See #2364 #[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] async fn import_dump_v1() { let temp = tempfile::tempdir().unwrap(); From 2c31d7c50af287958505fc10cefc478e46c224bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Lecrenier?= Date: Thu, 27 Oct 2022 16:23:50 +0200 Subject: [PATCH 7/8] Apply review suggestions --- index-scheduler/src/lib.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 67ce09ec0..4d95aa8e5 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -100,7 +100,7 @@ pub struct Query { } impl Query { - /// Return `true` iff every field of the query is set to `None`, such that the query + /// Return `true` if every field of the query is set to `None`, such that the query /// matches all tasks. pub fn is_empty(&self) -> bool { matches!( @@ -569,9 +569,7 @@ impl IndexScheduler { let rtxn = self.env.read_txn()?; let processing_tasks = self.processing_tasks.read().unwrap().processing.clone(); let index_tasks = self.index_tasks(&rtxn, index)?; - let nbr_index_processing_tasks = processing_tasks.intersection_len(&index_tasks); - Ok(nbr_index_processing_tasks > 0) } @@ -582,7 +580,7 @@ impl IndexScheduler { /// /// 1. IndexSwap tasks are not publicly associated with any index, but they are associated /// with many indexes internally. - /// 2. The user may not have the rights to access the tasks (internally) associated wuth all indexes. + /// 2. The user may not have the rights to access the tasks (internally) associated with all indexes. pub fn get_task_ids_from_authorized_indexes( &self, rtxn: &RoTxn, @@ -601,8 +599,8 @@ impl IndexScheduler { // must be discarded. if let Some(authorized_indexes) = authorized_indexes { let all_indexes_iter = self.index_tasks.iter(rtxn)?; - for iter_el in all_indexes_iter { - let (index, index_tasks) = iter_el?; + for result in all_indexes_iter { + let (index, index_tasks) = result?; if !authorized_indexes.contains(&index.to_owned()) { tasks -= index_tasks; } @@ -619,7 +617,7 @@ impl IndexScheduler { /// /// 1. IndexSwap tasks are not publicly associated with any index, but they are associated /// with many indexes internally. - /// 2. The user may not have the rights to access the tasks (internally) associated wuth all indexes. + /// 2. The user may not have the rights to access the tasks (internally) associated with all indexes. pub fn get_tasks_from_authorized_indexes( &self, query: Query, From 8dd79426565660e211431e43ff38e667b502092c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Lecrenier?= Date: Thu, 27 Oct 2022 16:23:12 +0200 Subject: [PATCH 8/8] Cargo fmt --- meilisearch-http/src/routes/tasks.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/meilisearch-http/src/routes/tasks.rs b/meilisearch-http/src/routes/tasks.rs index 110f6d40b..0c9a49a3c 100644 --- a/meilisearch-http/src/routes/tasks.rs +++ b/meilisearch-http/src/routes/tasks.rs @@ -465,10 +465,7 @@ async fn get_task( analytics.publish("Tasks Seen".to_string(), json!({ "per_task_uid": true }), Some(&req)); - let query = index_scheduler::Query { - uid: Some(vec![task_id]), - ..Query::default() - }; + let query = index_scheduler::Query { uid: Some(vec![task_id]), ..Query::default() }; if let Some(task) = index_scheduler .get_tasks_from_authorized_indexes(