From 1dd33af8a3fbef70a1fd7edf394cda04360f1f0a Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 26 Dec 2024 17:16:52 +0100 Subject: [PATCH] add the batches --- crates/meilisearch-types/src/batch_view.rs | 4 +- crates/meilisearch-types/src/batches.rs | 4 +- crates/meilisearch-types/src/task_view.rs | 1 + crates/meilisearch/src/routes/batches.rs | 123 ++++++++++++++++++++- crates/meilisearch/src/routes/features.rs | 2 +- crates/meilisearch/src/routes/mod.rs | 40 ++++--- crates/milli/src/progress.rs | 7 +- 7 files changed, 156 insertions(+), 25 deletions(-) diff --git a/crates/meilisearch-types/src/batch_view.rs b/crates/meilisearch-types/src/batch_view.rs index 08d25413c..112abd1dd 100644 --- a/crates/meilisearch-types/src/batch_view.rs +++ b/crates/meilisearch-types/src/batch_view.rs @@ -1,13 +1,15 @@ use milli::progress::ProgressView; use serde::Serialize; use time::{Duration, OffsetDateTime}; +use utoipa::ToSchema; use crate::batches::{Batch, BatchId, BatchStats}; use crate::task_view::DetailsView; use crate::tasks::serialize_duration; -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct BatchView { pub uid: BatchId, pub progress: Option, diff --git a/crates/meilisearch-types/src/batches.rs b/crates/meilisearch-types/src/batches.rs index 664dafa7a..7910a5af4 100644 --- a/crates/meilisearch-types/src/batches.rs +++ b/crates/meilisearch-types/src/batches.rs @@ -3,6 +3,7 @@ use std::collections::BTreeMap; use milli::progress::ProgressView; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; +use utoipa::ToSchema; use crate::task_view::DetailsView; use crate::tasks::{Kind, Status}; @@ -25,8 +26,9 @@ pub struct Batch { pub finished_at: Option, } -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive(Default, Debug, Clone, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct BatchStats { pub total_nb_tasks: BatchId, pub status: BTreeMap, diff --git a/crates/meilisearch-types/src/task_view.rs b/crates/meilisearch-types/src/task_view.rs index 467408097..23af055d6 100644 --- a/crates/meilisearch-types/src/task_view.rs +++ b/crates/meilisearch-types/src/task_view.rs @@ -69,6 +69,7 @@ impl TaskView { #[derive(Default, Debug, PartialEq, Eq, Clone, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct DetailsView { /// Number of documents received for documentAdditionOrUpdate task. #[serde(skip_serializing_if = "Option::is_none")] diff --git a/crates/meilisearch/src/routes/batches.rs b/crates/meilisearch/src/routes/batches.rs index 36bf31605..a5fc77a3d 100644 --- a/crates/meilisearch/src/routes/batches.rs +++ b/crates/meilisearch/src/routes/batches.rs @@ -8,17 +8,77 @@ use meilisearch_types::deserr::DeserrQueryParamError; use meilisearch_types::error::ResponseError; use meilisearch_types::keys::actions; use serde::Serialize; +use utoipa::{OpenApi, ToSchema}; use super::tasks::TasksFilterQuery; use super::ActionPolicy; use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; +#[derive(OpenApi)] +#[openapi( + paths(get_batch, get_batches), + tags(( + name = "Batches", + description = "The /batches route gives information about the progress of batches of asynchronous operations.", + external_docs(url = "https://www.meilisearch.com/docs/reference/api/batches"), + + )), +)] +pub struct BatchesApi; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::resource("").route(web::get().to(SeqHandler(get_batches)))) .service(web::resource("/{batch_id}").route(web::get().to(SeqHandler(get_batch)))); } +/// Get one batch +/// +/// Get a single batch. +#[utoipa::path( + get, + path = "/{batchUid}", + tag = "Batches", + security(("Bearer" = ["tasks.get", "tasks.*", "*"])), + params( + ("batchUid" = String, Path, example = "8685", description = "The unique batch id", nullable = false), + ), + responses( + (status = OK, description = "Return the batch", body = BatchView, content_type = "application/json", example = json!( + { + "uid": 1, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "progress": null, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "INDEX_NAME": 1 + } + }, + "duration": "PT0.364788S", + "startedAt": "2024-12-10T15:48:49.672141Z", + "finishedAt": "2024-12-10T15:48:50.036929Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] async fn get_batch( index_scheduler: GuardedData, Data>, batch_uid: web::Path, @@ -39,14 +99,14 @@ async fn get_batch( let (batches, _) = index_scheduler.get_batches_from_authorized_indexes(&query, filters)?; if let Some(batch) = batches.first() { - let task_view = BatchView::from_batch(batch); - Ok(HttpResponse::Ok().json(task_view)) + let batch_view = BatchView::from_batch(batch); + Ok(HttpResponse::Ok().json(batch_view)) } else { Err(index_scheduler::Error::BatchNotFound(batch_uid).into()) } } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] pub struct AllBatches { results: Vec, total: u64, @@ -55,6 +115,63 @@ pub struct AllBatches { next: Option, } +/// Get batches +/// +/// List all batches, regardless of index. The batch objects are contained in the results array. +/// Batches are always returned in descending order of uid. This means that by default, the most recently created batch objects appear first. +/// Batch results are paginated and can be filtered with query parameters. +#[utoipa::path( + get, + path = "/", + tag = "Batches", + security(("Bearer" = ["tasks.get", "tasks.*", "*"])), + params(TasksFilterQuery), + responses( + (status = OK, description = "Return the batches", body = AllBatches, content_type = "application/json", example = json!( + { + "results": [ + { + "uid": 2, + "details": { + "stopWords": [ + "of", + "the" + ] + }, + "progress": null, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "INDEX_NAME": 1 + } + }, + "duration": "PT0.110083S", + "startedAt": "2024-12-10T15:49:04.995321Z", + "finishedAt": "2024-12-10T15:49:05.105404Z" + } + ], + "total": 3, + "limit": 1, + "from": 2, + "next": 1 + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] async fn get_batches( index_scheduler: GuardedData, Data>, params: AwebQueryParameter, diff --git a/crates/meilisearch/src/routes/features.rs b/crates/meilisearch/src/routes/features.rs index 8347a9496..506e73b2e 100644 --- a/crates/meilisearch/src/routes/features.rs +++ b/crates/meilisearch/src/routes/features.rs @@ -17,7 +17,7 @@ use crate::extractors::sequential_extractor::SeqHandler; #[derive(OpenApi)] #[openapi( - paths(get_features), + paths(get_features, patch_features), tags(( name = "Experimental features", description = "The `/experimental-features` route allows you to activate or deactivate some of Meilisearch's experimental features. diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index b597b82a1..b97d129db 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -2,6 +2,9 @@ use std::collections::BTreeMap; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; +use crate::milli::progress::ProgressStepView; +use crate::milli::progress::ProgressView; +use crate::routes::batches::AllBatches; use crate::routes::features::RuntimeTogglableFeatures; use crate::routes::indexes::documents::DocumentEditionByFunction; use crate::routes::multi_search::SearchResults; @@ -16,6 +19,8 @@ use actix_web::web::Data; use actix_web::{web, HttpRequest, HttpResponse}; use index_scheduler::IndexScheduler; use meilisearch_auth::AuthController; +use meilisearch_types::batch_view::BatchView; +use meilisearch_types::batches::BatchStats; use meilisearch_types::error::{Code, ErrorType, ResponseError}; use meilisearch_types::index_uid::IndexUid; use meilisearch_types::keys::CreateApiKey; @@ -63,6 +68,7 @@ pub mod tasks; #[openapi( nest( (path = "/tasks", api = tasks::TaskApi), + (path = "/batches", api = batches::BatchesApi), (path = "/indexes", api = indexes::IndexesApi), // We must stop the search path here because the rest must be configured by each route individually (path = "/indexes", api = indexes::search::SearchApi), @@ -80,29 +86,29 @@ pub mod tasks; (name = "Stats", description = "Stats gives extended information and metrics about indexes and the Meilisearch database."), ), modifiers(&OpenApiAuth), - components(schemas(RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) + components(schemas(AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) )] pub struct MeilisearchApi; pub fn configure(cfg: &mut web::ServiceConfig) { let openapi = MeilisearchApi::openapi(); - cfg.service(web::scope("/tasks").configure(tasks::configure)) // done - .service(web::scope("/batches").configure(batches::configure)) // TODO - .service(Scalar::with_url("/scalar", openapi.clone())) // done - .service(RapiDoc::with_openapi("/api-docs/openapi.json", openapi.clone()).path("/rapidoc")) // done - .service(Redoc::with_url("/redoc", openapi)) // done - .service(web::resource("/health").route(web::get().to(get_health))) // done - .service(web::scope("/logs").configure(logs::configure)) // done - .service(web::scope("/keys").configure(api_key::configure)) // done - .service(web::scope("/dumps").configure(dump::configure)) // done - .service(web::scope("/snapshots").configure(snapshot::configure)) // done - .service(web::resource("/stats").route(web::get().to(get_stats))) // done - .service(web::resource("/version").route(web::get().to(get_version))) // done - .service(web::scope("/indexes").configure(indexes::configure)) // done - .service(web::scope("/multi-search").configure(multi_search::configure)) // done - .service(web::scope("/swap-indexes").configure(swap_indexes::configure)) // done - .service(web::scope("/metrics").configure(metrics::configure)) // done + cfg.service(web::scope("/tasks").configure(tasks::configure)) + .service(web::scope("/batches").configure(batches::configure)) + .service(Scalar::with_url("/scalar", openapi.clone())) + .service(RapiDoc::with_openapi("/api-docs/openapi.json", openapi.clone()).path("/rapidoc")) + .service(Redoc::with_url("/redoc", openapi)) + .service(web::resource("/health").route(web::get().to(get_health))) + .service(web::scope("/logs").configure(logs::configure)) + .service(web::scope("/keys").configure(api_key::configure)) + .service(web::scope("/dumps").configure(dump::configure)) + .service(web::scope("/snapshots").configure(snapshot::configure)) + .service(web::resource("/stats").route(web::get().to(get_stats))) + .service(web::resource("/version").route(web::get().to(get_version))) + .service(web::scope("/indexes").configure(indexes::configure)) + .service(web::scope("/multi-search").configure(multi_search::configure)) + .service(web::scope("/swap-indexes").configure(swap_indexes::configure)) + .service(web::scope("/metrics").configure(metrics::configure)) .service(web::scope("/experimental-features").configure(features::configure)); } diff --git a/crates/milli/src/progress.rs b/crates/milli/src/progress.rs index accc2cf56..622ec9842 100644 --- a/crates/milli/src/progress.rs +++ b/crates/milli/src/progress.rs @@ -4,6 +4,7 @@ use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, RwLock}; use serde::Serialize; +use utoipa::ToSchema; pub trait Step: 'static + Send + Sync { fn name(&self) -> Cow<'static, str>; @@ -136,15 +137,17 @@ macro_rules! make_atomic_progress { make_atomic_progress!(Document alias AtomicDocumentStep => "document" ); make_atomic_progress!(Payload alias AtomicPayloadStep => "payload" ); -#[derive(Debug, Serialize, Clone)] +#[derive(Debug, Serialize, Clone, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct ProgressView { pub steps: Vec, pub percentage: f32, } -#[derive(Debug, Serialize, Clone)] +#[derive(Debug, Serialize, Clone, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct ProgressStepView { pub current_step: Cow<'static, str>, pub finished: u32,