From fa2b96b9a5e80fc5ba18a5d78d30113ca0ee6d37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 19 Dec 2023 12:18:45 +0100 Subject: [PATCH] Add an Authorization Header along with the webhook calls --- index-scheduler/src/insta_snapshot.rs | 1 + index-scheduler/src/lib.rs | 17 ++++++++++++++++- meilisearch/src/analytics/segment_analytics.rs | 3 +++ meilisearch/src/lib.rs | 1 + meilisearch/src/option.rs | 12 ++++++++++++ 5 files changed, 33 insertions(+), 1 deletion(-) diff --git a/index-scheduler/src/insta_snapshot.rs b/index-scheduler/src/insta_snapshot.rs index 9261bf66d..0adda43ff 100644 --- a/index-scheduler/src/insta_snapshot.rs +++ b/index-scheduler/src/insta_snapshot.rs @@ -38,6 +38,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { auth_path: _, version_file_path: _, webhook_url: _, + webhook_authorization_header: _, test_breakpoint_sdr: _, planned_failures: _, run_loop_iteration: _, diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 2ad263ca4..296f8add1 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -245,7 +245,10 @@ pub struct IndexSchedulerOptions { pub snapshots_path: PathBuf, /// The path to the folder containing the dumps. pub dumps_path: PathBuf, + /// The URL on which we must send the tasks statuses pub webhook_url: Option, + /// The value we will send into the Authorization HTTP header on the webhook URL + pub webhook_authorization_header: Option, /// The maximum size, in bytes, of the task index. pub task_db_size: usize, /// The size, in bytes, with which a meilisearch index is opened the first time of each meilisearch index. @@ -330,6 +333,8 @@ pub struct IndexScheduler { /// The webhook url we should send tasks to after processing every batches. pub(crate) webhook_url: Option, + /// The Authorization header to send to the webhook URL. + pub(crate) webhook_authorization_header: Option, /// A frame to output the indexation profiling files to disk. pub(crate) puffin_frame: Arc, @@ -397,6 +402,7 @@ impl IndexScheduler { auth_path: self.auth_path.clone(), version_file_path: self.version_file_path.clone(), webhook_url: self.webhook_url.clone(), + webhook_authorization_header: self.webhook_authorization_header.clone(), currently_updating_index: self.currently_updating_index.clone(), embedders: self.embedders.clone(), #[cfg(test)] @@ -497,6 +503,7 @@ impl IndexScheduler { auth_path: options.auth_path, version_file_path: options.version_file_path, webhook_url: options.webhook_url, + webhook_authorization_header: options.webhook_authorization_header, currently_updating_index: Arc::new(RwLock::new(None)), embedders: Default::default(), @@ -1338,8 +1345,15 @@ impl IndexScheduler { written: 0, }; + // let reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); let reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); - if let Err(e) = ureq::post(url).set("Content-Encoding", "gzip").send(reader) { + let request = ureq::post(url).set("Content-Encoding", "gzip"); + let request = match &self.webhook_authorization_header { + Some(header) => request.set("Authorization", header), + None => request, + }; + + if let Err(e) = request.send(reader) { log::error!("While sending data to the webhook: {e}"); } } @@ -1761,6 +1775,7 @@ mod tests { snapshots_path: tempdir.path().join("snapshots"), dumps_path: tempdir.path().join("dumps"), webhook_url: None, + webhook_authorization_header: None, task_db_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose. index_base_map_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose. enable_mdb_writemap: false, diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs index 0cdb18540..86a5eddb9 100644 --- a/meilisearch/src/analytics/segment_analytics.rs +++ b/meilisearch/src/analytics/segment_analytics.rs @@ -265,6 +265,7 @@ struct Infos { http_addr: bool, http_payload_size_limit: Byte, task_queue_webhook: bool, + task_webhook_authorization_header: bool, log_level: String, max_indexing_memory: MaxMemory, max_indexing_threads: MaxThreads, @@ -292,6 +293,7 @@ impl From for Infos { master_key: _, env, task_webhook_url, + task_webhook_authorization_header, max_index_size: _, max_task_db_size: _, http_payload_size_limit, @@ -346,6 +348,7 @@ impl From for Infos { http_payload_size_limit, experimental_max_number_of_batched_tasks, task_queue_webhook: task_webhook_url.is_some(), + task_webhook_authorization_header: task_webhook_authorization_header.is_some(), log_level: log_level.to_string(), max_indexing_memory, max_indexing_threads, diff --git a/meilisearch/src/lib.rs b/meilisearch/src/lib.rs index 3698e5da4..f1111962c 100644 --- a/meilisearch/src/lib.rs +++ b/meilisearch/src/lib.rs @@ -229,6 +229,7 @@ fn open_or_create_database_unchecked( snapshots_path: opt.snapshot_dir.clone(), dumps_path: opt.dump_dir.clone(), webhook_url: opt.task_webhook_url.as_ref().map(|url| url.to_string()), + webhook_authorization_header: opt.task_webhook_authorization_header.clone(), task_db_size: opt.max_task_db_size.get_bytes() as usize, index_base_map_size: opt.max_index_size.get_bytes() as usize, enable_mdb_writemap: opt.experimental_reduce_indexing_memory_usage, diff --git a/meilisearch/src/option.rs b/meilisearch/src/option.rs index abb2bab6c..a0672c9cf 100644 --- a/meilisearch/src/option.rs +++ b/meilisearch/src/option.rs @@ -30,6 +30,7 @@ const MEILI_HTTP_ADDR: &str = "MEILI_HTTP_ADDR"; const MEILI_MASTER_KEY: &str = "MEILI_MASTER_KEY"; const MEILI_ENV: &str = "MEILI_ENV"; const MEILI_TASK_WEBHOOK_URL: &str = "MEILI_TASK_WEBHOOK_URL"; +const MEILI_TASK_WEBHOOK_AUTHORIZATION_HEADER: &str = "MEILI_TASK_WEBHOOK_AUTHORIZATION_HEADER"; #[cfg(feature = "analytics")] const MEILI_NO_ANALYTICS: &str = "MEILI_NO_ANALYTICS"; const MEILI_HTTP_PAYLOAD_SIZE_LIMIT: &str = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT"; @@ -162,6 +163,10 @@ pub struct Opt { #[clap(long, env = MEILI_TASK_WEBHOOK_URL)] pub task_webhook_url: Option, + /// The Authorization header to send on the webhook URL whenever a task finishes so a third party can be notified. + #[clap(long, env = MEILI_TASK_WEBHOOK_AUTHORIZATION_HEADER)] + pub task_webhook_authorization_header: Option, + /// Deactivates Meilisearch's built-in telemetry when provided. /// /// Meilisearch automatically collects data from all instances that do not opt out using this flag. @@ -382,6 +387,7 @@ impl Opt { master_key, env, task_webhook_url, + task_webhook_authorization_header, max_index_size: _, max_task_db_size: _, http_payload_size_limit, @@ -419,6 +425,12 @@ impl Opt { if let Some(task_webhook_url) = task_webhook_url { export_to_env_if_not_present(MEILI_TASK_WEBHOOK_URL, task_webhook_url.to_string()); } + if let Some(task_webhook_authorization_header) = task_webhook_authorization_header { + export_to_env_if_not_present( + MEILI_TASK_WEBHOOK_AUTHORIZATION_HEADER, + task_webhook_authorization_header, + ); + } #[cfg(feature = "analytics")] {