From 1a7684912e6b0696454351815deeb42149e2daae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Fri, 17 Jan 2025 13:53:00 +0100 Subject: [PATCH 01/14] feat(sources)!: add custom auth strategy for components with HTTP server This adds `custom` auth strategy for components with HTTP server (`http_server`, `datadog_agent`, `opentelemetry`, `prometheus`) besides the default basic auth. This is a breaking change because `strategy` is now required for auth - for existing configurations `strategy: "basic"` needs to be added. Related: #22213 --- src/{sources/util => common}/http/error.rs | 1 + src/common/http/mod.rs | 12 + src/common/http/server_auth.rs | 391 +++++++++++++++++++++ src/common/mod.rs | 6 + src/config/sink.rs | 2 + src/config/source.rs | 3 + src/sources/datadog_agent/logs.rs | 8 +- src/sources/datadog_agent/metrics.rs | 3 +- src/sources/datadog_agent/mod.rs | 3 +- src/sources/datadog_agent/traces.rs | 6 +- src/sources/heroku_logs.rs | 22 +- src/sources/http_server.rs | 119 ++++++- src/sources/opentelemetry/http.rs | 3 +- src/sources/prometheus/pushgateway.rs | 6 +- src/sources/prometheus/remote_write.rs | 5 +- src/sources/socket/mod.rs | 1 + src/sources/util/http/auth.rs | 85 ----- src/sources/util/http/encoding.rs | 3 +- src/sources/util/http/mod.rs | 8 - src/sources/util/http/prelude.rs | 26 +- src/sources/util/mod.rs | 5 - src/topology/builder.rs | 9 +- 22 files changed, 579 insertions(+), 148 deletions(-) rename src/{sources/util => common}/http/error.rs (98%) create mode 100644 src/common/http/mod.rs create mode 100644 src/common/http/server_auth.rs delete mode 100644 src/sources/util/http/auth.rs diff --git a/src/sources/util/http/error.rs b/src/common/http/error.rs similarity index 98% rename from src/sources/util/http/error.rs rename to src/common/http/error.rs index 06298558b2abd..da39e86cf1156 100644 --- a/src/sources/util/http/error.rs +++ b/src/common/http/error.rs @@ -1,3 +1,4 @@ +#![allow(missing_docs)] use std::{error::Error, fmt}; use serde::Serialize; diff --git a/src/common/http/mod.rs b/src/common/http/mod.rs new file mode 100644 index 0000000000000..8608f5e153dcb --- /dev/null +++ b/src/common/http/mod.rs @@ -0,0 +1,12 @@ +//! Common module between modules that use HTTP +#[cfg(all( + feature = "sources-utils-http-auth", + feature = "sources-utils-http-error" +))] +pub mod server_auth; + +#[cfg(feature = "sources-utils-http-error")] +mod error; + +#[cfg(feature = "sources-utils-http-error")] +pub use error::ErrorMessage; diff --git a/src/common/http/server_auth.rs b/src/common/http/server_auth.rs new file mode 100644 index 0000000000000..9aefcf6e6d1e1 --- /dev/null +++ b/src/common/http/server_auth.rs @@ -0,0 +1,391 @@ +//! Shared authentication config between components that use HTTP. +use bytes::Bytes; +use headers::{authorization::Credentials, Authorization}; +use http::{header::AUTHORIZATION, HeaderMap, HeaderValue, StatusCode}; +use vector_config::configurable_component; +use vector_lib::{ + compile_vrl, + event::{Event, LogEvent, VrlTarget}, + sensitive_string::SensitiveString, + TimeZone, +}; +use vrl::{ + compiler::{runtime::Runtime, CompilationResult, CompileConfig, Program}, + core::Value, + diagnostic::Formatter, + prelude::TypeState, + value::{KeyString, ObjectMap}, +}; + +use super::ErrorMessage; + +/// Configuration of the authentication strategy for server mode sinks and sources. +/// +/// HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an +/// HTTP header without any additional encryption beyond what is provided by the transport itself. +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde(deny_unknown_fields, rename_all = "snake_case", tag = "strategy")] +#[configurable(metadata(docs::enum_tag_description = "The authentication strategy to use."))] +pub enum HttpServerAuthConfig { + /// Basic authentication. + /// + /// The username and password are concatenated and encoded via [base64][base64]. + /// + /// [base64]: https://en.wikipedia.org/wiki/Base64 + Basic { + /// The basic authentication username. + #[configurable(metadata(docs::examples = "${USERNAME}"))] + #[configurable(metadata(docs::examples = "username"))] + user: String, + + /// The basic authentication password. + #[configurable(metadata(docs::examples = "${PASSWORD}"))] + #[configurable(metadata(docs::examples = "password"))] + password: SensitiveString, + }, + + /// Custom authentication using VRL code. + /// + /// Takes in request and validates it using VRL code. + Custom { + /// The VRL boolean expression. + source: String, + }, +} + +impl HttpServerAuthConfig { + /// Builds an auth matcher based on provided configuration. + /// Used to validate configuration if needed, before passing it to the + /// actual component for usage. + pub fn build( + &self, + enrichment_tables: &vector_lib::enrichment::TableRegistry, + ) -> crate::Result { + match self { + HttpServerAuthConfig::Basic { user, password } => { + Ok(HttpServerAuthMatcher::AuthHeader( + Authorization::basic(user, password.inner()).0.encode(), + "Invalid username/password", + )) + } + HttpServerAuthConfig::Custom { source } => { + let functions = vrl::stdlib::all() + .into_iter() + .chain(vector_lib::enrichment::vrl_functions()) + .chain(vector_vrl_functions::all()) + .collect::>(); + + let state = TypeState::default(); + + let mut config = CompileConfig::default(); + config.set_custom(enrichment_tables.clone()); + config.set_read_only(); + + let CompilationResult { + program, + warnings, + config: _, + } = compile_vrl(source, &functions, &state, config).map_err(|diagnostics| { + Formatter::new(source, diagnostics).colored().to_string() + })?; + + if !program.final_type_info().result.is_boolean() { + return Err("VRL conditions must return a boolean.".into()); + } + + if !warnings.is_empty() { + let warnings = Formatter::new(source, warnings).colored().to_string(); + warn!(message = "VRL compilation warning.", %warnings); + } + + Ok(HttpServerAuthMatcher::Vrl { program }) + } + } + } +} + +/// Built auth matcher with validated configuration +/// Can be used directly in a component to validate authentication in HTTP requests +#[derive(Clone, Debug)] +pub enum HttpServerAuthMatcher { + /// Matcher for comparing exact value of Authorization header + AuthHeader(HeaderValue, &'static str), + /// Matcher for running VRL script for requests, to allow for custom validation + Vrl { + /// Compiled VRL script + program: Program, + }, +} + +impl HttpServerAuthMatcher { + #[cfg(test)] + fn auth_header(self) -> (HeaderValue, &'static str) { + match self { + HttpServerAuthMatcher::AuthHeader(header_value, error_message) => { + (header_value, error_message) + } + HttpServerAuthMatcher::Vrl { .. } => { + panic!("Expected HttpServerAuthMatcher::AuthHeader") + } + } + } + + /// Compares passed headers to the matcher + pub fn handle_auth(&self, headers: &HeaderMap) -> Result<(), ErrorMessage> { + match self { + HttpServerAuthMatcher::AuthHeader(expected, err_message) => { + if let Some(header) = headers.get(AUTHORIZATION) { + if expected == header { + Ok(()) + } else { + Err(ErrorMessage::new( + StatusCode::UNAUTHORIZED, + err_message.to_string(), + )) + } + } else { + Err(ErrorMessage::new( + StatusCode::UNAUTHORIZED, + "No authorization header".to_owned(), + )) + } + } + HttpServerAuthMatcher::Vrl { program } => { + let mut target = VrlTarget::new( + Event::Log(LogEvent::from_map( + ObjectMap::from([( + "headers".into(), + Value::Object( + headers + .iter() + .map(|(k, v)| { + ( + KeyString::from(k.to_string()), + Value::Bytes(Bytes::copy_from_slice(v.as_bytes())), + ) + }) + .collect::(), + ), + )]), + Default::default(), + )), + program.info(), + false, + ); + let timezone = TimeZone::default(); + + let result = Runtime::default().resolve(&mut target, program, &timezone); + match result.map_err(|e| { + warn!("Handling auth failed: {}", e); + ErrorMessage::new(StatusCode::UNAUTHORIZED, "Auth failed".to_owned()) + })? { + vrl::core::Value::Boolean(result) => { + if result { + Ok(()) + } else { + Err(ErrorMessage::new( + StatusCode::UNAUTHORIZED, + "Auth failed".to_owned(), + )) + } + } + _ => Err(ErrorMessage::new( + StatusCode::UNAUTHORIZED, + "Invalid return value".to_owned(), + )), + } + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::test_util::random_string; + use indoc::indoc; + + use super::*; + + #[test] + fn build_basic_auth_should_always_work() { + let basic_auth = HttpServerAuthConfig::Basic { + user: random_string(16), + password: random_string(16).into(), + }; + + let matcher = basic_auth.build(&Default::default()); + + assert!(matcher.is_ok()); + assert!(matches!( + matcher.unwrap(), + HttpServerAuthMatcher::AuthHeader { .. } + )); + } + + #[test] + fn build_basic_auth_should_use_username_password_related_message() { + let basic_auth = HttpServerAuthConfig::Basic { + user: random_string(16), + password: random_string(16).into(), + }; + + let (_, error_message) = basic_auth.build(&Default::default()).unwrap().auth_header(); + assert_eq!("Invalid username/password", error_message); + } + + #[test] + fn build_basic_auth_should_use_encode_basic_header() { + let user = random_string(16); + let password = random_string(16); + let basic_auth = HttpServerAuthConfig::Basic { + user: user.clone(), + password: password.clone().into(), + }; + + let (header, _) = basic_auth.build(&Default::default()).unwrap().auth_header(); + assert_eq!(Authorization::basic(&user, &password).0.encode(), header); + } + + #[test] + fn build_custom_should_fail_on_invalid_source() { + let custom_auth = HttpServerAuthConfig::Custom { + source: "invalid VRL source".to_string(), + }; + + assert!(custom_auth.build(&Default::default()).is_err()); + } + + #[test] + fn build_custom_should_fail_on_non_boolean_return_type() { + let custom_auth = HttpServerAuthConfig::Custom { + source: indoc! {r#" + .success = true + . + "#} + .to_string(), + }; + + assert!(custom_auth.build(&Default::default()).is_err()); + } + + #[test] + fn build_custom_should_success_on_proper_source_with_boolean_return_type() { + let custom_auth = HttpServerAuthConfig::Custom { + source: indoc! {r#" + .headers.authorization == "Basic test" + "#} + .to_string(), + }; + + assert!(custom_auth.build(&Default::default()).is_ok()); + } + + #[test] + fn basic_auth_matcher_should_return_401_when_missing_auth_header() { + let basic_auth = HttpServerAuthConfig::Basic { + user: random_string(16), + password: random_string(16).into(), + }; + + let matcher = basic_auth.build(&Default::default()).unwrap(); + + let result = matcher.handle_auth(&HeaderMap::new()); + + assert!(result.is_err()); + let error = result.unwrap_err(); + assert_eq!(401, error.code()); + assert_eq!("No authorization header", error.message()); + } + + #[test] + fn basic_auth_matcher_should_return_401_and_with_wrong_credentials() { + let basic_auth = HttpServerAuthConfig::Basic { + user: random_string(16), + password: random_string(16).into(), + }; + + let matcher = basic_auth.build(&Default::default()).unwrap(); + + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, HeaderValue::from_static("Basic wrong")); + let result = matcher.handle_auth(&headers); + + assert!(result.is_err()); + let error = result.unwrap_err(); + assert_eq!(401, error.code()); + assert_eq!("Invalid username/password", error.message()); + } + + #[test] + fn basic_auth_matcher_should_return_ok_for_correct_credentials() { + let user = random_string(16); + let password = random_string(16); + let basic_auth = HttpServerAuthConfig::Basic { + user: user.clone(), + password: password.clone().into(), + }; + + let matcher = basic_auth.build(&Default::default()).unwrap(); + + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + Authorization::basic(&user, &password).0.encode(), + ); + let result = matcher.handle_auth(&headers); + + assert!(result.is_ok()); + } + + #[test] + fn custom_auth_matcher_should_return_ok_for_true_vrl_script_result() { + let custom_auth = HttpServerAuthConfig::Custom { + source: r#".headers.authorization == "test""#.to_string(), + }; + + let matcher = custom_auth.build(&Default::default()).unwrap(); + + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, HeaderValue::from_static("test")); + let result = matcher.handle_auth(&headers); + + assert!(result.is_ok()); + } + + #[test] + fn custom_auth_matcher_should_return_401_for_false_vrl_script_result() { + let custom_auth = HttpServerAuthConfig::Custom { + source: r#".headers.authorization == "test""#.to_string(), + }; + + let matcher = custom_auth.build(&Default::default()).unwrap(); + + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, HeaderValue::from_static("wrong value")); + let result = matcher.handle_auth(&headers); + + assert!(result.is_err()); + let error = result.unwrap_err(); + assert_eq!(401, error.code()); + assert_eq!("Auth failed", error.message()); + } + + #[test] + fn custom_auth_matcher_should_return_401_for_failed_script_execution() { + let custom_auth = HttpServerAuthConfig::Custom { + source: "abort".to_string(), + }; + + let matcher = custom_auth.build(&Default::default()).unwrap(); + + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, HeaderValue::from_static("test")); + let result = matcher.handle_auth(&headers); + + assert!(result.is_err()); + let error = result.unwrap_err(); + assert_eq!(401, error.code()); + assert_eq!("Auth failed", error.message()); + } +} diff --git a/src/common/mod.rs b/src/common/mod.rs index 6e29cbeebeadf..cfaf80c4f9354 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -20,3 +20,9 @@ pub(crate) mod s3; #[cfg(any(feature = "transforms-log_to_metric", feature = "sinks-loki"))] pub(crate) mod expansion; + +#[cfg(any( + feature = "sources-utils-http-auth", + feature = "sources-utils-http-error" +))] +pub mod http; diff --git a/src/config/sink.rs b/src/config/sink.rs index a322863f11926..c3e60ffc3b07c 100644 --- a/src/config/sink.rs +++ b/src/config/sink.rs @@ -241,6 +241,7 @@ dyn_clone::clone_trait_object!(SinkConfig); pub struct SinkContext { pub healthcheck: SinkHealthcheckOptions, pub globals: GlobalOptions, + pub enrichment_tables: vector_lib::enrichment::TableRegistry, pub proxy: ProxyConfig, pub schema: schema::Options, pub app_name: String, @@ -256,6 +257,7 @@ impl Default for SinkContext { Self { healthcheck: Default::default(), globals: Default::default(), + enrichment_tables: Default::default(), proxy: Default::default(), schema: Default::default(), app_name: crate::get_app_name().to_string(), diff --git a/src/config/source.rs b/src/config/source.rs index 1ad0e46de30c2..8b91413115b7a 100644 --- a/src/config/source.rs +++ b/src/config/source.rs @@ -124,6 +124,7 @@ dyn_clone::clone_trait_object!(SourceConfig); pub struct SourceContext { pub key: ComponentKey, pub globals: GlobalOptions, + pub enrichment_tables: vector_lib::enrichment::TableRegistry, pub shutdown: ShutdownSignal, pub out: SourceSender, pub proxy: ProxyConfig, @@ -153,6 +154,7 @@ impl SourceContext { Self { key: key.clone(), globals: GlobalOptions::default(), + enrichment_tables: Default::default(), shutdown: shutdown_signal, out, proxy: Default::default(), @@ -173,6 +175,7 @@ impl SourceContext { Self { key: ComponentKey::from("default"), globals: GlobalOptions::default(), + enrichment_tables: Default::default(), shutdown: ShutdownSignal::noop(), out, proxy: Default::default(), diff --git a/src/sources/datadog_agent/logs.rs b/src/sources/datadog_agent/logs.rs index 50ee4aebb2d14..58c8b5369f029 100644 --- a/src/sources/datadog_agent/logs.rs +++ b/src/sources/datadog_agent/logs.rs @@ -13,14 +13,12 @@ use vrl::core::Value; use warp::{filters::BoxedFilter, path as warp_path, path::FullPath, reply::Response, Filter}; use crate::common::datadog::DDTAGS; +use crate::common::http::ErrorMessage; use crate::{ event::Event, internal_events::DatadogAgentJsonParseError, - sources::{ - datadog_agent::{ - handle_request, ApiKeyQueryParams, DatadogAgentConfig, DatadogAgentSource, LogMsg, - }, - util::ErrorMessage, + sources::datadog_agent::{ + handle_request, ApiKeyQueryParams, DatadogAgentConfig, DatadogAgentSource, LogMsg, }, SourceSender, }; diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index fd3bcdfe66e83..a27400e995ef1 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -14,6 +14,7 @@ use vector_lib::{ EstimatedJsonEncodedSizeOf, }; +use crate::common::http::ErrorMessage; use crate::{ common::datadog::{DatadogMetricType, DatadogSeriesMetric}, config::log_schema, @@ -28,7 +29,7 @@ use crate::{ ddmetric_proto::{metric_payload, Metadata, MetricPayload, SketchPayload}, handle_request, ApiKeyQueryParams, DatadogAgentSource, }, - util::{extract_tag_key_and_value, ErrorMessage}, + util::extract_tag_key_and_value, }, SourceSender, }; diff --git a/src/sources/datadog_agent/mod.rs b/src/sources/datadog_agent/mod.rs index 5d53300fbdd2c..abb5463608641 100644 --- a/src/sources/datadog_agent/mod.rs +++ b/src/sources/datadog_agent/mod.rs @@ -47,6 +47,7 @@ use vrl::value::kind::Collection; use vrl::value::Kind; use warp::{filters::BoxedFilter, reject::Rejection, reply::Response, Filter, Reply}; +use crate::common::http::ErrorMessage; use crate::http::{build_http_trace_layer, KeepaliveConfig, MaxConnectionAgeLayer}; use crate::{ codecs::{Decoder, DecodingConfig}, @@ -58,7 +59,7 @@ use crate::{ internal_events::{HttpBytesReceived, HttpDecompressError, StreamClosedError}, schema, serde::{bool_or_struct, default_decoding, default_framing_message_based}, - sources::{self, util::ErrorMessage}, + sources::{self}, tls::{MaybeTlsSettings, TlsEnableableConfig}, SourceSender, }; diff --git a/src/sources/datadog_agent/traces.rs b/src/sources/datadog_agent/traces.rs index a9bef1086560c..769ec25a16d1c 100644 --- a/src/sources/datadog_agent/traces.rs +++ b/src/sources/datadog_agent/traces.rs @@ -12,11 +12,11 @@ use warp::{filters::BoxedFilter, path, path::FullPath, reply::Response, Filter, use vector_lib::internal_event::{CountByteSize, InternalEventHandle as _}; use vector_lib::EstimatedJsonEncodedSizeOf; +use crate::common::http::ErrorMessage; use crate::{ event::{Event, ObjectMap, TraceEvent, Value}, - sources::{ - datadog_agent::{ddtrace_proto, handle_request, ApiKeyQueryParams, DatadogAgentSource}, - util::ErrorMessage, + sources::datadog_agent::{ + ddtrace_proto, handle_request, ApiKeyQueryParams, DatadogAgentSource, }, SourceSender, }; diff --git a/src/sources/heroku_logs.rs b/src/sources/heroku_logs.rs index f019a980c8e67..14d4ab169c837 100644 --- a/src/sources/heroku_logs.rs +++ b/src/sources/heroku_logs.rs @@ -25,6 +25,7 @@ use vector_lib::{ use crate::{ codecs::{Decoder, DecodingConfig}, + common::http::{server_auth::HttpServerAuthConfig, ErrorMessage}, config::{ log_schema, GenerateConfig, Resource, SourceAcknowledgementsConfig, SourceConfig, SourceContext, SourceOutput, @@ -37,7 +38,7 @@ use crate::{ http_server::{build_param_matcher, remove_duplicates, HttpConfigParamKind}, util::{ http::{add_query_parameters, HttpMethod}, - ErrorMessage, HttpSource, HttpSourceAuthConfig, + HttpSource, }, }, tls::TlsEnableableConfig, @@ -70,7 +71,7 @@ pub struct LogplexConfig { tls: Option, #[configurable(derived)] - auth: Option, + auth: Option, #[configurable(derived)] #[serde(default = "default_framing_message_based")] @@ -438,7 +439,8 @@ mod tests { }; use vrl::value::{kind::Collection, Kind}; - use super::{HttpSourceAuthConfig, LogplexConfig}; + use super::LogplexConfig; + use crate::common::http::server_auth::HttpServerAuthConfig; use crate::{ config::{log_schema, SourceConfig, SourceContext}, serde::{default_decoding, default_framing_message_based}, @@ -455,7 +457,7 @@ mod tests { } async fn source( - auth: Option, + auth: Option, query_parameters: Vec, status: EventStatus, acknowledgements: bool, @@ -488,13 +490,13 @@ mod tests { async fn send( address: SocketAddr, body: &str, - auth: Option, + auth: Option, query: &str, ) -> u16 { let len = body.lines().count(); let mut req = reqwest::Client::new().post(format!("http://{}/events?{}", address, query)); - if let Some(auth) = auth { - req = req.basic_auth(auth.username, Some(auth.password.inner())); + if let Some(HttpServerAuthConfig::Basic { user, password }) = auth { + req = req.basic_auth(user, Some(password.inner())); } req.header("Logplex-Msg-Count", len) .header("Logplex-Frame-Id", "frame-foo") @@ -507,9 +509,9 @@ mod tests { .as_u16() } - fn make_auth() -> HttpSourceAuthConfig { - HttpSourceAuthConfig { - username: random_string(16), + fn make_auth() -> HttpServerAuthConfig { + HttpServerAuthConfig::Basic { + user: random_string(16), password: random_string(16).into(), } } diff --git a/src/sources/http_server.rs b/src/sources/http_server.rs index 068f5e3d9f340..6220234a5efd0 100644 --- a/src/sources/http_server.rs +++ b/src/sources/http_server.rs @@ -1,3 +1,4 @@ +use crate::common::http::{server_auth::HttpServerAuthConfig, ErrorMessage}; use std::{collections::HashMap, net::SocketAddr}; use bytes::{Bytes, BytesMut}; @@ -31,7 +32,7 @@ use crate::{ serde::{bool_or_struct, default_decoding}, sources::util::{ http::{add_headers, add_query_parameters, HttpMethod}, - Encoding, ErrorMessage, HttpSource, HttpSourceAuthConfig, + Encoding, HttpSource, }, tls::TlsEnableableConfig, }; @@ -114,7 +115,7 @@ pub struct SimpleHttpConfig { query_parameters: Vec, #[configurable(derived)] - auth: Option, + auth: Option, /// Whether or not to treat the configured `path` as an absolute path. /// @@ -532,6 +533,9 @@ mod tests { Compression, }; use futures::Stream; + use headers::authorization::Credentials; + use headers::Authorization; + use http::header::AUTHORIZATION; use http::{HeaderMap, Method, StatusCode, Uri}; use similar_asserts::assert_eq; use vector_lib::codecs::{ @@ -545,6 +549,7 @@ mod tests { use vector_lib::schema::Definition; use vrl::value::{kind::Collection, Kind, ObjectMap}; + use crate::common::http::server_auth::HttpServerAuthConfig; use crate::sources::http_server::HttpMethod; use crate::{ components::validation::prelude::*, @@ -573,6 +578,7 @@ mod tests { path: &'a str, method: &'a str, response_code: StatusCode, + auth: Option, strict_path: bool, status: EventStatus, acknowledgements: bool, @@ -599,7 +605,7 @@ mod tests { query_parameters, response_code, tls: None, - auth: None, + auth, strict_path, path_key, host_key, @@ -711,6 +717,7 @@ mod tests { "/", "POST", StatusCode::OK, + None, true, EventStatus::Delivered, true, @@ -757,6 +764,7 @@ mod tests { "/", "POST", StatusCode::OK, + None, true, EventStatus::Delivered, true, @@ -796,6 +804,7 @@ mod tests { "/", "POST", StatusCode::OK, + None, true, EventStatus::Delivered, true, @@ -829,6 +838,7 @@ mod tests { "/", "POST", StatusCode::OK, + None, true, EventStatus::Delivered, true, @@ -867,6 +877,7 @@ mod tests { "/", "POST", StatusCode::OK, + None, true, EventStatus::Delivered, true, @@ -912,6 +923,7 @@ mod tests { "/", "POST", StatusCode::OK, + None, true, EventStatus::Delivered, true, @@ -963,6 +975,7 @@ mod tests { "/", "POST", StatusCode::OK, + None, true, EventStatus::Delivered, true, @@ -1049,6 +1062,7 @@ mod tests { "/", "POST", StatusCode::OK, + None, true, EventStatus::Delivered, true, @@ -1095,6 +1109,7 @@ mod tests { "/", "POST", StatusCode::OK, + None, true, EventStatus::Delivered, true, @@ -1137,6 +1152,7 @@ mod tests { "/", "POST", StatusCode::OK, + None, true, EventStatus::Delivered, true, @@ -1176,6 +1192,7 @@ mod tests { "/", "POST", StatusCode::OK, + None, true, EventStatus::Delivered, true, @@ -1232,6 +1249,7 @@ mod tests { "/", "POST", StatusCode::OK, + None, true, EventStatus::Delivered, true, @@ -1263,6 +1281,7 @@ mod tests { "/event/path", "POST", StatusCode::OK, + None, true, EventStatus::Delivered, true, @@ -1304,6 +1323,7 @@ mod tests { "/event", "POST", StatusCode::OK, + None, false, EventStatus::Delivered, true, @@ -1365,6 +1385,7 @@ mod tests { "/", "POST", StatusCode::OK, + None, true, EventStatus::Delivered, true, @@ -1390,6 +1411,7 @@ mod tests { "/", "POST", StatusCode::ACCEPTED, + None, true, EventStatus::Delivered, true, @@ -1424,6 +1446,7 @@ mod tests { "/", "POST", StatusCode::OK, + None, true, EventStatus::Rejected, true, @@ -1455,6 +1478,7 @@ mod tests { "/", "POST", StatusCode::OK, + None, true, EventStatus::Rejected, false, @@ -1488,6 +1512,7 @@ mod tests { "/", "GET", StatusCode::OK, + None, true, EventStatus::Delivered, true, @@ -1499,6 +1524,94 @@ mod tests { assert_eq!(200, send_request(addr, "GET", "", "/").await); } + #[tokio::test] + async fn returns_401_when_required_auth_is_missing() { + components::init_test(); + let (_rx, addr) = source( + vec![], + vec![], + "http_path", + "remote_ip", + "/", + "GET", + StatusCode::OK, + Some(HttpServerAuthConfig::Basic { + user: "test".to_string(), + password: "test".to_string().into(), + }), + true, + EventStatus::Delivered, + true, + None, + None, + ) + .await; + + assert_eq!(401, send_request(addr, "GET", "", "/").await); + } + + #[tokio::test] + async fn returns_401_when_required_auth_is_wrong() { + components::init_test(); + let (_rx, addr) = source( + vec![], + vec![], + "http_path", + "remote_ip", + "/", + "POST", + StatusCode::OK, + Some(HttpServerAuthConfig::Basic { + user: "test".to_string(), + password: "test".to_string().into(), + }), + true, + EventStatus::Delivered, + true, + None, + None, + ) + .await; + + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + Authorization::basic("wrong", "test").0.encode(), + ); + assert_eq!(401, send_with_headers(addr, "", headers).await); + } + + #[tokio::test] + async fn http_get_with_correct_auth() { + components::init_test(); + let (_rx, addr) = source( + vec![], + vec![], + "http_path", + "remote_ip", + "/", + "POST", + StatusCode::OK, + Some(HttpServerAuthConfig::Basic { + user: "test".to_string(), + password: "test".to_string().into(), + }), + true, + EventStatus::Delivered, + true, + None, + None, + ) + .await; + + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + Authorization::basic("test", "test").0.encode(), + ); + assert_eq!(200, send_with_headers(addr, "", headers).await); + } + #[test] fn output_schema_definition_vector_namespace() { let config = SimpleHttpConfig { diff --git a/src/sources/opentelemetry/http.rs b/src/sources/opentelemetry/http.rs index 0d78294226906..97231ef0f6c20 100644 --- a/src/sources/opentelemetry/http.rs +++ b/src/sources/opentelemetry/http.rs @@ -27,6 +27,7 @@ use warp::{ filters::BoxedFilter, http::HeaderMap, reject::Rejection, reply::Response, Filter, Reply, }; +use crate::common::http::ErrorMessage; use crate::http::{KeepaliveConfig, MaxConnectionAgeLayer}; use crate::sources::http_server::HttpConfigParamKind; use crate::sources::util::add_headers; @@ -35,7 +36,7 @@ use crate::{ http::build_http_trace_layer, internal_events::{EventsReceived, StreamClosedError}, shutdown::ShutdownSignal, - sources::util::{decode, ErrorMessage}, + sources::util::decode, tls::MaybeTlsSettings, SourceSender, }; diff --git a/src/sources/prometheus/pushgateway.rs b/src/sources/prometheus/pushgateway.rs index 537487dd52038..88291d4513abe 100644 --- a/src/sources/prometheus/pushgateway.rs +++ b/src/sources/prometheus/pushgateway.rs @@ -22,6 +22,8 @@ use vector_lib::configurable::configurable_component; use warp::http::HeaderMap; use super::parser; +use crate::common::http::server_auth::HttpServerAuthConfig; +use crate::common::http::ErrorMessage; use crate::http::KeepaliveConfig; use crate::{ config::{ @@ -31,7 +33,7 @@ use crate::{ serde::bool_or_struct, sources::{ self, - util::{http::HttpMethod, ErrorMessage, HttpSource, HttpSourceAuthConfig}, + util::{http::HttpMethod, HttpSource}, }, tls::TlsEnableableConfig, }; @@ -54,7 +56,7 @@ pub struct PrometheusPushgatewayConfig { #[configurable(derived)] #[configurable(metadata(docs::advanced))] - auth: Option, + auth: Option, #[configurable(derived)] #[serde(default, deserialize_with = "bool_or_struct")] diff --git a/src/sources/prometheus/remote_write.rs b/src/sources/prometheus/remote_write.rs index e4a8c9bff39bd..ba7613d600fc0 100644 --- a/src/sources/prometheus/remote_write.rs +++ b/src/sources/prometheus/remote_write.rs @@ -9,6 +9,7 @@ use warp::http::{HeaderMap, StatusCode}; use super::parser; use crate::{ + common::http::{server_auth::HttpServerAuthConfig, ErrorMessage}, config::{ GenerateConfig, SourceAcknowledgementsConfig, SourceConfig, SourceContext, SourceOutput, }, @@ -18,7 +19,7 @@ use crate::{ serde::bool_or_struct, sources::{ self, - util::{decode, http::HttpMethod, ErrorMessage, HttpSource, HttpSourceAuthConfig}, + util::{decode, http::HttpMethod, HttpSource}, }, tls::TlsEnableableConfig, }; @@ -41,7 +42,7 @@ pub struct PrometheusRemoteWriteConfig { #[configurable(derived)] #[configurable(metadata(docs::advanced))] - auth: Option, + auth: Option, #[configurable(derived)] #[serde(default, deserialize_with = "bool_or_struct")] diff --git a/src/sources/socket/mod.rs b/src/sources/socket/mod.rs index 13bac01881824..efd23de8a987c 100644 --- a/src/sources/socket/mod.rs +++ b/src/sources/socket/mod.rs @@ -991,6 +991,7 @@ mod test { .build(SourceContext { key: source_key.clone(), globals: GlobalOptions::default(), + enrichment_tables: Default::default(), shutdown: shutdown_signal, out: sender, proxy: Default::default(), diff --git a/src/sources/util/http/auth.rs b/src/sources/util/http/auth.rs deleted file mode 100644 index f902cf3734db8..0000000000000 --- a/src/sources/util/http/auth.rs +++ /dev/null @@ -1,85 +0,0 @@ -use std::convert::TryFrom; - -use headers::{Authorization, HeaderMapExt}; -use vector_lib::configurable::configurable_component; -use vector_lib::sensitive_string::SensitiveString; -use warp::http::HeaderMap; - -#[cfg(any( - feature = "sources-utils-http-prelude", - feature = "sources-utils-http-auth" -))] -use super::error::ErrorMessage; - -/// HTTP Basic authentication configuration. -#[configurable_component] -#[derive(Clone, Debug)] -pub struct HttpSourceAuthConfig { - /// The username for basic authentication. - #[configurable(metadata(docs::examples = "AzureDiamond"))] - #[configurable(metadata(docs::examples = "admin"))] - pub username: String, - - /// The password for basic authentication. - #[configurable(metadata(docs::examples = "hunter2"))] - #[configurable(metadata(docs::examples = "${PASSWORD}"))] - pub password: SensitiveString, -} - -impl TryFrom> for HttpSourceAuth { - type Error = String; - - fn try_from(auth: Option<&HttpSourceAuthConfig>) -> Result { - match auth { - Some(auth) => { - let mut headers = HeaderMap::new(); - headers.typed_insert(Authorization::basic( - auth.username.as_str(), - auth.password.inner(), - )); - match headers.get("authorization") { - Some(value) => { - let token = value - .to_str() - .map_err(|error| format!("Failed stringify HeaderValue: {:?}", error))? - .to_owned(); - Ok(HttpSourceAuth { token: Some(token) }) - } - None => Err("Authorization headers wasn't generated".to_owned()), - } - } - None => Ok(HttpSourceAuth { token: None }), - } - } -} - -#[derive(Clone, Debug)] -pub struct HttpSourceAuth { - #[allow(unused)] // triggered by check-component-features - pub(self) token: Option, -} - -impl HttpSourceAuth { - #[allow(unused)] // triggered by check-component-features - pub fn is_valid(&self, header: &Option) -> Result<(), ErrorMessage> { - use warp::http::StatusCode; - - match (&self.token, header) { - (Some(token1), Some(token2)) => { - if token1 == token2 { - Ok(()) - } else { - Err(ErrorMessage::new( - StatusCode::UNAUTHORIZED, - "Invalid username/password".to_owned(), - )) - } - } - (Some(_), None) => Err(ErrorMessage::new( - StatusCode::UNAUTHORIZED, - "No authorization header".to_owned(), - )), - (None, _) => Ok(()), - } - } -} diff --git a/src/sources/util/http/encoding.rs b/src/sources/util/http/encoding.rs index 39051f67acd82..43f2916623a8e 100644 --- a/src/sources/util/http/encoding.rs +++ b/src/sources/util/http/encoding.rs @@ -5,8 +5,7 @@ use flate2::read::{MultiGzDecoder, ZlibDecoder}; use snap::raw::Decoder as SnappyDecoder; use warp::http::StatusCode; -use super::error::ErrorMessage; -use crate::internal_events::HttpDecompressError; +use crate::{common::http::ErrorMessage, internal_events::HttpDecompressError}; pub fn decode(header: Option<&str>, mut body: Bytes) -> Result { if let Some(encodings) = header { diff --git a/src/sources/util/http/mod.rs b/src/sources/util/http/mod.rs index ae01187b78d8b..09b6a733477b0 100644 --- a/src/sources/util/http/mod.rs +++ b/src/sources/util/http/mod.rs @@ -1,9 +1,5 @@ -#[cfg(feature = "sources-utils-http-auth")] -mod auth; #[cfg(feature = "sources-utils-http-encoding")] mod encoding; -#[cfg(feature = "sources-utils-http-error")] -mod error; #[cfg(any( feature = "sources-http_server", feature = "sources-opentelemetry", @@ -20,12 +16,8 @@ mod prelude; ))] mod query; -#[cfg(feature = "sources-utils-http-auth")] -pub use auth::{HttpSourceAuth, HttpSourceAuthConfig}; #[cfg(feature = "sources-utils-http-encoding")] pub use encoding::decode; -#[cfg(feature = "sources-utils-http-error")] -pub use error::ErrorMessage; #[cfg(feature = "sources-utils-http-headers")] pub use headers::add_headers; pub use method::HttpMethod; diff --git a/src/sources/util/http/prelude.rs b/src/sources/util/http/prelude.rs index 1e452882a1b32..52ac4c8adf88a 100644 --- a/src/sources/util/http/prelude.rs +++ b/src/sources/util/http/prelude.rs @@ -1,10 +1,5 @@ -use std::{ - collections::HashMap, - convert::{Infallible, TryFrom}, - fmt, - net::SocketAddr, - time::Duration, -}; +use crate::common::http::{server_auth::HttpServerAuthConfig, ErrorMessage}; +use std::{collections::HashMap, convert::Infallible, fmt, net::SocketAddr, time::Duration}; use bytes::Bytes; use futures::{FutureExt, TryFutureExt}; @@ -38,11 +33,7 @@ use crate::{ SourceSender, }; -use super::{ - auth::{HttpSourceAuth, HttpSourceAuthConfig}, - encoding::decode, - error::ErrorMessage, -}; +use super::encoding::decode; pub trait HttpSource: Clone + Send + Sync + 'static { // This function can be defined to enrich events with additional HTTP @@ -79,14 +70,14 @@ pub trait HttpSource: Clone + Send + Sync + 'static { response_code: StatusCode, strict_path: bool, tls: Option<&TlsEnableableConfig>, - auth: Option<&HttpSourceAuthConfig>, + auth: Option<&HttpServerAuthConfig>, cx: SourceContext, acknowledgements: SourceAcknowledgementsConfig, keepalive_settings: KeepaliveConfig, ) -> crate::Result { let tls = MaybeTlsSettings::from_config(tls, true)?; let protocol = tls.http_protocol_name(); - let auth = HttpSourceAuth::try_from(auth)?; + let auth_matcher = auth.map(|a| a.build(&cx.enrichment_tables)).transpose()?; let path = path.to_owned(); let acknowledgements = cx.do_acknowledgements(acknowledgements); let enable_source_ip = self.enable_source_ip(); @@ -124,7 +115,6 @@ pub trait HttpSource: Clone + Send + Sync + 'static { }) .untuple_one() .and(warp::path::full()) - .and(warp::header::optional::("authorization")) .and(warp::header::optional::("content-encoding")) .and(warp::header::headers_cloned()) .and(warp::body::bytes()) @@ -132,7 +122,6 @@ pub trait HttpSource: Clone + Send + Sync + 'static { .and(warp::filters::ext::optional()) .and_then( move |path: FullPath, - auth_header, encoding_header: Option, headers: HeaderMap, body: Bytes, @@ -141,8 +130,9 @@ pub trait HttpSource: Clone + Send + Sync + 'static { debug!(message = "Handling HTTP request.", headers = ?headers); let http_path = path.as_str(); - let events = auth - .is_valid(&auth_header) + let events = auth_matcher + .as_ref() + .map_or(Ok(()), |a| a.handle_auth(&headers)) .and_then(|()| self.decode(encoding_header.as_deref(), body)) .and_then(|body| { emit!(HttpBytesReceived { diff --git a/src/sources/util/mod.rs b/src/sources/util/mod.rs index a441d4f6ba30c..45e773316c4cb 100644 --- a/src/sources/util/mod.rs +++ b/src/sources/util/mod.rs @@ -9,7 +9,6 @@ pub mod grpc; #[cfg(any( feature = "sources-utils-http-auth", feature = "sources-utils-http-encoding", - feature = "sources-utils-http-error", feature = "sources-utils-http-headers", feature = "sources-utils-http-prelude", feature = "sources-utils-http-query" @@ -59,12 +58,8 @@ pub use self::http::add_query_parameters; feature = "sources-utils-http-encoding" ))] pub use self::http::decode; -#[cfg(feature = "sources-utils-http-error")] -pub use self::http::ErrorMessage; #[cfg(feature = "sources-utils-http-prelude")] pub use self::http::HttpSource; -#[cfg(feature = "sources-utils-http-auth")] -pub use self::http::HttpSourceAuthConfig; #[cfg(any(feature = "sources-aws_sqs", feature = "sources-gcp_pubsub"))] pub use self::message_decoding::decode_message; diff --git a/src/topology/builder.rs b/src/topology/builder.rs index 8602e5cce6c13..a6163e9e713cb 100644 --- a/src/topology/builder.rs +++ b/src/topology/builder.rs @@ -111,7 +111,7 @@ impl<'a> Builder<'a> { /// Builds the new pieces of the topology found in `self.diff`. async fn build(mut self) -> Result> { let enrichment_tables = self.load_enrichment_tables().await; - let source_tasks = self.build_sources().await; + let source_tasks = self.build_sources(enrichment_tables).await; self.build_transforms(enrichment_tables).await; self.build_sinks(enrichment_tables).await; @@ -203,7 +203,10 @@ impl<'a> Builder<'a> { &ENRICHMENT_TABLES } - async fn build_sources(&mut self) -> HashMap { + async fn build_sources( + &mut self, + enrichment_tables: &vector_lib::enrichment::TableRegistry, + ) -> HashMap { let mut source_tasks = HashMap::new(); for (key, source) in self @@ -322,6 +325,7 @@ impl<'a> Builder<'a> { let context = SourceContext { key: key.clone(), globals: self.config.global.clone(), + enrichment_tables: enrichment_tables.clone(), shutdown: shutdown_signal, out: pipeline, proxy: ProxyConfig::merge_with_env(&self.config.global.proxy, &source.proxy), @@ -568,6 +572,7 @@ impl<'a> Builder<'a> { let cx = SinkContext { healthcheck, globals: self.config.global.clone(), + enrichment_tables: enrichment_tables.clone(), proxy: ProxyConfig::merge_with_env(&self.config.global.proxy, sink.proxy()), schema: self.config.schema, app_name: crate::get_app_name().to_string(), From cc60d2e238bd89e3d88a943007f0e3e95030658e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Fri, 17 Jan 2025 19:05:05 +0100 Subject: [PATCH 02/14] Generate components docs --- .../components/sources/base/heroku_logs.cue | 47 +++++++++++++++---- .../components/sources/base/http.cue | 47 +++++++++++++++---- .../components/sources/base/http_server.cue | 47 +++++++++++++++---- .../sources/base/prometheus_pushgateway.cue | 47 +++++++++++++++---- .../sources/base/prometheus_remote_write.cue | 47 +++++++++++++++---- 5 files changed, 195 insertions(+), 40 deletions(-) diff --git a/website/cue/reference/components/sources/base/heroku_logs.cue b/website/cue/reference/components/sources/base/heroku_logs.cue index 418f61da96d33..556df05f90d24 100644 --- a/website/cue/reference/components/sources/base/heroku_logs.cue +++ b/website/cue/reference/components/sources/base/heroku_logs.cue @@ -28,18 +28,49 @@ base: components: sources: heroku_logs: configuration: { type: string: examples: ["0.0.0.0:80", "localhost:80"] } auth: { - description: "HTTP Basic authentication configuration." - required: false + description: """ + Configuration of the authentication strategy for server mode sinks and sources. + + HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an + HTTP header without any additional encryption beyond what is provided by the transport itself. + """ + required: false type: object: options: { password: { - description: "The password for basic authentication." - required: true - type: string: examples: ["hunter2", "${PASSWORD}"] + description: "The basic authentication password." + relevant_when: "strategy = \"basic\"" + required: true + type: string: examples: ["${PASSWORD}", "password"] + } + source: { + description: "The VRL boolean expression." + relevant_when: "strategy = \"custom\"" + required: true + type: string: {} } - username: { - description: "The username for basic authentication." + strategy: { + description: "The authentication strategy to use." required: true - type: string: examples: ["AzureDiamond", "admin"] + type: string: enum: { + basic: """ + Basic authentication. + + The username and password are concatenated and encoded via [base64][base64]. + + [base64]: https://en.wikipedia.org/wiki/Base64 + """ + custom: """ + Custom authentication using VRL code. + + Takes in request and validates it using VRL code. + """ + } + } + user: { + description: "The basic authentication username." + relevant_when: "strategy = \"basic\"" + required: true + type: string: examples: ["${USERNAME}", "username"] } } } diff --git a/website/cue/reference/components/sources/base/http.cue b/website/cue/reference/components/sources/base/http.cue index d53ca9b9f93fd..d1c98b60b7db8 100644 --- a/website/cue/reference/components/sources/base/http.cue +++ b/website/cue/reference/components/sources/base/http.cue @@ -32,18 +32,49 @@ base: components: sources: http: configuration: { type: string: examples: ["0.0.0.0:80", "localhost:80"] } auth: { - description: "HTTP Basic authentication configuration." - required: false + description: """ + Configuration of the authentication strategy for server mode sinks and sources. + + HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an + HTTP header without any additional encryption beyond what is provided by the transport itself. + """ + required: false type: object: options: { password: { - description: "The password for basic authentication." - required: true - type: string: examples: ["hunter2", "${PASSWORD}"] + description: "The basic authentication password." + relevant_when: "strategy = \"basic\"" + required: true + type: string: examples: ["${PASSWORD}", "password"] + } + source: { + description: "The VRL boolean expression." + relevant_when: "strategy = \"custom\"" + required: true + type: string: {} } - username: { - description: "The username for basic authentication." + strategy: { + description: "The authentication strategy to use." required: true - type: string: examples: ["AzureDiamond", "admin"] + type: string: enum: { + basic: """ + Basic authentication. + + The username and password are concatenated and encoded via [base64][base64]. + + [base64]: https://en.wikipedia.org/wiki/Base64 + """ + custom: """ + Custom authentication using VRL code. + + Takes in request and validates it using VRL code. + """ + } + } + user: { + description: "The basic authentication username." + relevant_when: "strategy = \"basic\"" + required: true + type: string: examples: ["${USERNAME}", "username"] } } } diff --git a/website/cue/reference/components/sources/base/http_server.cue b/website/cue/reference/components/sources/base/http_server.cue index 543a97a42c96c..ceebf070a0c56 100644 --- a/website/cue/reference/components/sources/base/http_server.cue +++ b/website/cue/reference/components/sources/base/http_server.cue @@ -32,18 +32,49 @@ base: components: sources: http_server: configuration: { type: string: examples: ["0.0.0.0:80", "localhost:80"] } auth: { - description: "HTTP Basic authentication configuration." - required: false + description: """ + Configuration of the authentication strategy for server mode sinks and sources. + + HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an + HTTP header without any additional encryption beyond what is provided by the transport itself. + """ + required: false type: object: options: { password: { - description: "The password for basic authentication." - required: true - type: string: examples: ["hunter2", "${PASSWORD}"] + description: "The basic authentication password." + relevant_when: "strategy = \"basic\"" + required: true + type: string: examples: ["${PASSWORD}", "password"] + } + source: { + description: "The VRL boolean expression." + relevant_when: "strategy = \"custom\"" + required: true + type: string: {} } - username: { - description: "The username for basic authentication." + strategy: { + description: "The authentication strategy to use." required: true - type: string: examples: ["AzureDiamond", "admin"] + type: string: enum: { + basic: """ + Basic authentication. + + The username and password are concatenated and encoded via [base64][base64]. + + [base64]: https://en.wikipedia.org/wiki/Base64 + """ + custom: """ + Custom authentication using VRL code. + + Takes in request and validates it using VRL code. + """ + } + } + user: { + description: "The basic authentication username." + relevant_when: "strategy = \"basic\"" + required: true + type: string: examples: ["${USERNAME}", "username"] } } } diff --git a/website/cue/reference/components/sources/base/prometheus_pushgateway.cue b/website/cue/reference/components/sources/base/prometheus_pushgateway.cue index 9d00229d003e4..2c4221cc23bb5 100644 --- a/website/cue/reference/components/sources/base/prometheus_pushgateway.cue +++ b/website/cue/reference/components/sources/base/prometheus_pushgateway.cue @@ -42,18 +42,49 @@ base: components: sources: prometheus_pushgateway: configuration: { type: bool: default: false } auth: { - description: "HTTP Basic authentication configuration." - required: false + description: """ + Configuration of the authentication strategy for server mode sinks and sources. + + HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an + HTTP header without any additional encryption beyond what is provided by the transport itself. + """ + required: false type: object: options: { password: { - description: "The password for basic authentication." - required: true - type: string: examples: ["hunter2", "${PASSWORD}"] + description: "The basic authentication password." + relevant_when: "strategy = \"basic\"" + required: true + type: string: examples: ["${PASSWORD}", "password"] } - username: { - description: "The username for basic authentication." + source: { + description: "The VRL boolean expression." + relevant_when: "strategy = \"custom\"" + required: true + type: string: {} + } + strategy: { + description: "The authentication strategy to use." required: true - type: string: examples: ["AzureDiamond", "admin"] + type: string: enum: { + basic: """ + Basic authentication. + + The username and password are concatenated and encoded via [base64][base64]. + + [base64]: https://en.wikipedia.org/wiki/Base64 + """ + custom: """ + Custom authentication using VRL code. + + Takes in request and validates it using VRL code. + """ + } + } + user: { + description: "The basic authentication username." + relevant_when: "strategy = \"basic\"" + required: true + type: string: examples: ["${USERNAME}", "username"] } } } diff --git a/website/cue/reference/components/sources/base/prometheus_remote_write.cue b/website/cue/reference/components/sources/base/prometheus_remote_write.cue index 60530cc9e0a23..10e169e0a6e5f 100644 --- a/website/cue/reference/components/sources/base/prometheus_remote_write.cue +++ b/website/cue/reference/components/sources/base/prometheus_remote_write.cue @@ -32,18 +32,49 @@ base: components: sources: prometheus_remote_write: configuration: { type: string: examples: ["0.0.0.0:9090"] } auth: { - description: "HTTP Basic authentication configuration." - required: false + description: """ + Configuration of the authentication strategy for server mode sinks and sources. + + HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an + HTTP header without any additional encryption beyond what is provided by the transport itself. + """ + required: false type: object: options: { password: { - description: "The password for basic authentication." - required: true - type: string: examples: ["hunter2", "${PASSWORD}"] + description: "The basic authentication password." + relevant_when: "strategy = \"basic\"" + required: true + type: string: examples: ["${PASSWORD}", "password"] } - username: { - description: "The username for basic authentication." + source: { + description: "The VRL boolean expression." + relevant_when: "strategy = \"custom\"" + required: true + type: string: {} + } + strategy: { + description: "The authentication strategy to use." required: true - type: string: examples: ["AzureDiamond", "admin"] + type: string: enum: { + basic: """ + Basic authentication. + + The username and password are concatenated and encoded via [base64][base64]. + + [base64]: https://en.wikipedia.org/wiki/Base64 + """ + custom: """ + Custom authentication using VRL code. + + Takes in request and validates it using VRL code. + """ + } + } + user: { + description: "The basic authentication username." + relevant_when: "strategy = \"basic\"" + required: true + type: string: examples: ["${USERNAME}", "username"] } } } From 340d53ba97df26f7c8b5cec08a60bea872dd489f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Fri, 17 Jan 2025 19:09:15 +0100 Subject: [PATCH 03/14] Add changelog entry --- changelog.d/22236_custom_server_auth_strategy.breaking.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 changelog.d/22236_custom_server_auth_strategy.breaking.md diff --git a/changelog.d/22236_custom_server_auth_strategy.breaking.md b/changelog.d/22236_custom_server_auth_strategy.breaking.md new file mode 100644 index 0000000000000..cc2cfe6e6efcf --- /dev/null +++ b/changelog.d/22236_custom_server_auth_strategy.breaking.md @@ -0,0 +1,8 @@ +Custom authorization strategy is now supported for sources running +HTTP servers (`http_server` source, `prometheus` source, `datadog_agent`, etc.). + +Since there are now multiple authorization strategies, if you are using `auth` in any +of these supported components, you now also need to add `strategy: "basic"`, together with +`user` and `password`. + +authors: esensar From 7609793b57a8d759415a36dcf87df40ffd595070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Fri, 17 Jan 2025 19:55:21 +0100 Subject: [PATCH 04/14] Rename `user` to `username` to match old config --- src/common/http/server_auth.rs | 29 ++++++++++++++++------------- src/sources/heroku_logs.rs | 6 +++--- src/sources/http_server.rs | 6 +++--- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/common/http/server_auth.rs b/src/common/http/server_auth.rs index 9aefcf6e6d1e1..aff3921db64a1 100644 --- a/src/common/http/server_auth.rs +++ b/src/common/http/server_auth.rs @@ -37,7 +37,7 @@ pub enum HttpServerAuthConfig { /// The basic authentication username. #[configurable(metadata(docs::examples = "${USERNAME}"))] #[configurable(metadata(docs::examples = "username"))] - user: String, + username: String, /// The basic authentication password. #[configurable(metadata(docs::examples = "${PASSWORD}"))] @@ -63,9 +63,9 @@ impl HttpServerAuthConfig { enrichment_tables: &vector_lib::enrichment::TableRegistry, ) -> crate::Result { match self { - HttpServerAuthConfig::Basic { user, password } => { + HttpServerAuthConfig::Basic { username, password } => { Ok(HttpServerAuthMatcher::AuthHeader( - Authorization::basic(user, password.inner()).0.encode(), + Authorization::basic(username, password.inner()).0.encode(), "Invalid username/password", )) } @@ -210,7 +210,7 @@ mod tests { #[test] fn build_basic_auth_should_always_work() { let basic_auth = HttpServerAuthConfig::Basic { - user: random_string(16), + username: random_string(16), password: random_string(16).into(), }; @@ -226,7 +226,7 @@ mod tests { #[test] fn build_basic_auth_should_use_username_password_related_message() { let basic_auth = HttpServerAuthConfig::Basic { - user: random_string(16), + username: random_string(16), password: random_string(16).into(), }; @@ -236,15 +236,18 @@ mod tests { #[test] fn build_basic_auth_should_use_encode_basic_header() { - let user = random_string(16); + let username = random_string(16); let password = random_string(16); let basic_auth = HttpServerAuthConfig::Basic { - user: user.clone(), + username: username.clone(), password: password.clone().into(), }; let (header, _) = basic_auth.build(&Default::default()).unwrap().auth_header(); - assert_eq!(Authorization::basic(&user, &password).0.encode(), header); + assert_eq!( + Authorization::basic(&username, &password).0.encode(), + header + ); } #[test] @@ -284,7 +287,7 @@ mod tests { #[test] fn basic_auth_matcher_should_return_401_when_missing_auth_header() { let basic_auth = HttpServerAuthConfig::Basic { - user: random_string(16), + username: random_string(16), password: random_string(16).into(), }; @@ -301,7 +304,7 @@ mod tests { #[test] fn basic_auth_matcher_should_return_401_and_with_wrong_credentials() { let basic_auth = HttpServerAuthConfig::Basic { - user: random_string(16), + username: random_string(16), password: random_string(16).into(), }; @@ -319,10 +322,10 @@ mod tests { #[test] fn basic_auth_matcher_should_return_ok_for_correct_credentials() { - let user = random_string(16); + let username = random_string(16); let password = random_string(16); let basic_auth = HttpServerAuthConfig::Basic { - user: user.clone(), + username: username.clone(), password: password.clone().into(), }; @@ -331,7 +334,7 @@ mod tests { let mut headers = HeaderMap::new(); headers.insert( AUTHORIZATION, - Authorization::basic(&user, &password).0.encode(), + Authorization::basic(&username, &password).0.encode(), ); let result = matcher.handle_auth(&headers); diff --git a/src/sources/heroku_logs.rs b/src/sources/heroku_logs.rs index 14d4ab169c837..c4bde0c2898e1 100644 --- a/src/sources/heroku_logs.rs +++ b/src/sources/heroku_logs.rs @@ -495,8 +495,8 @@ mod tests { ) -> u16 { let len = body.lines().count(); let mut req = reqwest::Client::new().post(format!("http://{}/events?{}", address, query)); - if let Some(HttpServerAuthConfig::Basic { user, password }) = auth { - req = req.basic_auth(user, Some(password.inner())); + if let Some(HttpServerAuthConfig::Basic { username, password }) = auth { + req = req.basic_auth(username, Some(password.inner())); } req.header("Logplex-Msg-Count", len) .header("Logplex-Frame-Id", "frame-foo") @@ -511,7 +511,7 @@ mod tests { fn make_auth() -> HttpServerAuthConfig { HttpServerAuthConfig::Basic { - user: random_string(16), + username: random_string(16), password: random_string(16).into(), } } diff --git a/src/sources/http_server.rs b/src/sources/http_server.rs index 6220234a5efd0..6ca55f9487dae 100644 --- a/src/sources/http_server.rs +++ b/src/sources/http_server.rs @@ -1536,7 +1536,7 @@ mod tests { "GET", StatusCode::OK, Some(HttpServerAuthConfig::Basic { - user: "test".to_string(), + username: "test".to_string(), password: "test".to_string().into(), }), true, @@ -1562,7 +1562,7 @@ mod tests { "POST", StatusCode::OK, Some(HttpServerAuthConfig::Basic { - user: "test".to_string(), + username: "test".to_string(), password: "test".to_string().into(), }), true, @@ -1593,7 +1593,7 @@ mod tests { "POST", StatusCode::OK, Some(HttpServerAuthConfig::Basic { - user: "test".to_string(), + username: "test".to_string(), password: "test".to_string().into(), }), true, From cf09b122b865e9eaabe8a1997c68d3b84f82cce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Mon, 20 Jan 2025 15:31:45 +0100 Subject: [PATCH 05/14] Apply docs suggestions from code review Co-authored-by: Esther Kim --- website/cue/reference/components/sources/base/heroku_logs.cue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/cue/reference/components/sources/base/heroku_logs.cue b/website/cue/reference/components/sources/base/heroku_logs.cue index 556df05f90d24..756eb9c1c4472 100644 --- a/website/cue/reference/components/sources/base/heroku_logs.cue +++ b/website/cue/reference/components/sources/base/heroku_logs.cue @@ -31,7 +31,7 @@ base: components: sources: heroku_logs: configuration: { description: """ Configuration of the authentication strategy for server mode sinks and sources. - HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an + Use the HTTP authentication with HTTPS only. The authentication credentials are passed as an HTTP header without any additional encryption beyond what is provided by the transport itself. """ required: false @@ -55,7 +55,7 @@ base: components: sources: heroku_logs: configuration: { basic: """ Basic authentication. - The username and password are concatenated and encoded via [base64][base64]. + The username and password are concatenated and encoded using [base64][base64]. [base64]: https://en.wikipedia.org/wiki/Base64 """ From 622c81d4dcf82a96a2c6be9838a5c492558b7e06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Mon, 20 Jan 2025 15:39:01 +0100 Subject: [PATCH 06/14] Update all docs based on suggestions --- src/common/http/server_auth.rs | 4 ++-- .../cue/reference/components/sources/base/heroku_logs.cue | 2 +- website/cue/reference/components/sources/base/http.cue | 6 +++--- .../cue/reference/components/sources/base/http_server.cue | 6 +++--- .../components/sources/base/prometheus_pushgateway.cue | 6 +++--- .../components/sources/base/prometheus_remote_write.cue | 6 +++--- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/common/http/server_auth.rs b/src/common/http/server_auth.rs index aff3921db64a1..31a2694f41d03 100644 --- a/src/common/http/server_auth.rs +++ b/src/common/http/server_auth.rs @@ -21,7 +21,7 @@ use super::ErrorMessage; /// Configuration of the authentication strategy for server mode sinks and sources. /// -/// HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an +/// Use the HTTP authentication with HTTPS only. The authentication credentials are passed as an /// HTTP header without any additional encryption beyond what is provided by the transport itself. #[configurable_component] #[derive(Clone, Debug, Eq, PartialEq)] @@ -30,7 +30,7 @@ use super::ErrorMessage; pub enum HttpServerAuthConfig { /// Basic authentication. /// - /// The username and password are concatenated and encoded via [base64][base64]. + /// The username and password are concatenated and encoded using [base64][base64]. /// /// [base64]: https://en.wikipedia.org/wiki/Base64 Basic { diff --git a/website/cue/reference/components/sources/base/heroku_logs.cue b/website/cue/reference/components/sources/base/heroku_logs.cue index 756eb9c1c4472..58b949c77cf6f 100644 --- a/website/cue/reference/components/sources/base/heroku_logs.cue +++ b/website/cue/reference/components/sources/base/heroku_logs.cue @@ -66,7 +66,7 @@ base: components: sources: heroku_logs: configuration: { """ } } - user: { + username: { description: "The basic authentication username." relevant_when: "strategy = \"basic\"" required: true diff --git a/website/cue/reference/components/sources/base/http.cue b/website/cue/reference/components/sources/base/http.cue index d1c98b60b7db8..72804db5342a0 100644 --- a/website/cue/reference/components/sources/base/http.cue +++ b/website/cue/reference/components/sources/base/http.cue @@ -35,7 +35,7 @@ base: components: sources: http: configuration: { description: """ Configuration of the authentication strategy for server mode sinks and sources. - HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an + Use the HTTP authentication with HTTPS only. The authentication credentials are passed as an HTTP header without any additional encryption beyond what is provided by the transport itself. """ required: false @@ -59,7 +59,7 @@ base: components: sources: http: configuration: { basic: """ Basic authentication. - The username and password are concatenated and encoded via [base64][base64]. + The username and password are concatenated and encoded using [base64][base64]. [base64]: https://en.wikipedia.org/wiki/Base64 """ @@ -70,7 +70,7 @@ base: components: sources: http: configuration: { """ } } - user: { + username: { description: "The basic authentication username." relevant_when: "strategy = \"basic\"" required: true diff --git a/website/cue/reference/components/sources/base/http_server.cue b/website/cue/reference/components/sources/base/http_server.cue index ceebf070a0c56..f002d03bd90ca 100644 --- a/website/cue/reference/components/sources/base/http_server.cue +++ b/website/cue/reference/components/sources/base/http_server.cue @@ -35,7 +35,7 @@ base: components: sources: http_server: configuration: { description: """ Configuration of the authentication strategy for server mode sinks and sources. - HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an + Use the HTTP authentication with HTTPS only. The authentication credentials are passed as an HTTP header without any additional encryption beyond what is provided by the transport itself. """ required: false @@ -59,7 +59,7 @@ base: components: sources: http_server: configuration: { basic: """ Basic authentication. - The username and password are concatenated and encoded via [base64][base64]. + The username and password are concatenated and encoded using [base64][base64]. [base64]: https://en.wikipedia.org/wiki/Base64 """ @@ -70,7 +70,7 @@ base: components: sources: http_server: configuration: { """ } } - user: { + username: { description: "The basic authentication username." relevant_when: "strategy = \"basic\"" required: true diff --git a/website/cue/reference/components/sources/base/prometheus_pushgateway.cue b/website/cue/reference/components/sources/base/prometheus_pushgateway.cue index 2c4221cc23bb5..7b0088a0da41a 100644 --- a/website/cue/reference/components/sources/base/prometheus_pushgateway.cue +++ b/website/cue/reference/components/sources/base/prometheus_pushgateway.cue @@ -45,7 +45,7 @@ base: components: sources: prometheus_pushgateway: configuration: { description: """ Configuration of the authentication strategy for server mode sinks and sources. - HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an + Use the HTTP authentication with HTTPS only. The authentication credentials are passed as an HTTP header without any additional encryption beyond what is provided by the transport itself. """ required: false @@ -69,7 +69,7 @@ base: components: sources: prometheus_pushgateway: configuration: { basic: """ Basic authentication. - The username and password are concatenated and encoded via [base64][base64]. + The username and password are concatenated and encoded using [base64][base64]. [base64]: https://en.wikipedia.org/wiki/Base64 """ @@ -80,7 +80,7 @@ base: components: sources: prometheus_pushgateway: configuration: { """ } } - user: { + username: { description: "The basic authentication username." relevant_when: "strategy = \"basic\"" required: true diff --git a/website/cue/reference/components/sources/base/prometheus_remote_write.cue b/website/cue/reference/components/sources/base/prometheus_remote_write.cue index 10e169e0a6e5f..aa4f9b2db275a 100644 --- a/website/cue/reference/components/sources/base/prometheus_remote_write.cue +++ b/website/cue/reference/components/sources/base/prometheus_remote_write.cue @@ -35,7 +35,7 @@ base: components: sources: prometheus_remote_write: configuration: { description: """ Configuration of the authentication strategy for server mode sinks and sources. - HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an + Use the HTTP authentication with HTTPS only. The authentication credentials are passed as an HTTP header without any additional encryption beyond what is provided by the transport itself. """ required: false @@ -59,7 +59,7 @@ base: components: sources: prometheus_remote_write: configuration: { basic: """ Basic authentication. - The username and password are concatenated and encoded via [base64][base64]. + The username and password are concatenated and encoded using [base64][base64]. [base64]: https://en.wikipedia.org/wiki/Base64 """ @@ -70,7 +70,7 @@ base: components: sources: prometheus_remote_write: configuration: { """ } } - user: { + username: { description: "The basic authentication username." relevant_when: "strategy = \"basic\"" required: true From 9ad0b464fa8b017b9f37894f70963cedc74a9746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Wed, 29 Jan 2025 18:02:23 +0100 Subject: [PATCH 07/14] Prevent breaking change by defaulting to `basic` strategy for `HttpServerAuthConfig` --- src/common/http/server_auth.rs | 134 ++++++++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 2 deletions(-) diff --git a/src/common/http/server_auth.rs b/src/common/http/server_auth.rs index 31a2694f41d03..bdeb416d16921 100644 --- a/src/common/http/server_auth.rs +++ b/src/common/http/server_auth.rs @@ -1,7 +1,13 @@ //! Shared authentication config between components that use HTTP. +use std::{collections::HashMap, fmt}; + use bytes::Bytes; use headers::{authorization::Credentials, Authorization}; use http::{header::AUTHORIZATION, HeaderMap, HeaderValue, StatusCode}; +use serde::{ + de::{Error, MapAccess, Visitor}, + Deserialize, +}; use vector_config::configurable_component; use vector_lib::{ compile_vrl, @@ -23,9 +29,8 @@ use super::ErrorMessage; /// /// Use the HTTP authentication with HTTPS only. The authentication credentials are passed as an /// HTTP header without any additional encryption beyond what is provided by the transport itself. -#[configurable_component] +#[configurable_component(no_deser)] #[derive(Clone, Debug, Eq, PartialEq)] -#[serde(deny_unknown_fields, rename_all = "snake_case", tag = "strategy")] #[configurable(metadata(docs::enum_tag_description = "The authentication strategy to use."))] pub enum HttpServerAuthConfig { /// Basic authentication. @@ -54,6 +59,74 @@ pub enum HttpServerAuthConfig { }, } +// Custom deserializer implementation to default `strategy` to `basic` +impl<'de> Deserialize<'de> for HttpServerAuthConfig { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct HttpServerAuthConfigVisitor; + + const FIELD_KEYS: [&str; 4] = ["strategy", "username", "password", "source"]; + + impl<'de> Visitor<'de> for HttpServerAuthConfigVisitor { + type Value = HttpServerAuthConfig; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a valid authentication strategy (basic or custom)") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut fields: HashMap<&str, String> = HashMap::default(); + + while let Some(key) = map.next_key::()? { + if let Some(field_index) = FIELD_KEYS.iter().position(|k| *k == key.as_str()) { + if fields.contains_key(FIELD_KEYS[field_index]) { + return Err(Error::duplicate_field(FIELD_KEYS[field_index])); + } + fields.insert(FIELD_KEYS[field_index], map.next_value()?); + } else { + return Err(Error::unknown_field(&key, &FIELD_KEYS)); + } + } + + // Default to "basic" if strategy is missing + let strategy = fields + .get("strategy") + .map(String::as_str) + .unwrap_or_else(|| "basic"); + + match strategy { + "basic" => { + let username = fields + .remove("username") + .ok_or_else(|| Error::missing_field("username"))?; + let password = fields + .remove("password") + .ok_or_else(|| Error::missing_field("password"))?; + Ok(HttpServerAuthConfig::Basic { + username, + password: SensitiveString::from(password), + }) + } + "custom" => { + let source = fields + .remove("source") + .ok_or_else(|| Error::missing_field("source"))?; + Ok(HttpServerAuthConfig::Custom { source }) + } + _ => Err(Error::unknown_variant(strategy, &["basic", "custom"])), + } + } + } + + deserializer.deserialize_map(HttpServerAuthConfigVisitor) + } +} + impl HttpServerAuthConfig { /// Builds an auth matcher based on provided configuration. /// Used to validate configuration if needed, before passing it to the @@ -207,6 +280,63 @@ mod tests { use super::*; + #[test] + fn config_should_default_to_basic() { + let config = indoc! { r#" + username: foo + password: bar + "# + }; + + let config: HttpServerAuthConfig = serde_yaml::from_str(config).unwrap(); + + assert!(matches!(config, HttpServerAuthConfig::Basic { .. })); + if let HttpServerAuthConfig::Basic { username, password } = config { + assert_eq!(username, "foo"); + assert_eq!(password.inner(), "bar"); + } else { + unreachable!(); + } + } + + #[test] + fn config_should_support_explicit_basic_strategy() { + let config = indoc! { r#" + strategy: basic + username: foo + password: bar + "# + }; + + let config: HttpServerAuthConfig = serde_yaml::from_str(config).unwrap(); + + assert!(matches!(config, HttpServerAuthConfig::Basic { .. })); + if let HttpServerAuthConfig::Basic { username, password } = config { + assert_eq!(username, "foo"); + assert_eq!(password.inner(), "bar"); + } else { + unreachable!(); + } + } + + #[test] + fn config_should_support_custom_strategy() { + let config = indoc! { r#" + strategy: custom + source: "true" + "# + }; + + let config: HttpServerAuthConfig = serde_yaml::from_str(config).unwrap(); + + assert!(matches!(config, HttpServerAuthConfig::Custom { .. })); + if let HttpServerAuthConfig::Custom { source } = config { + assert_eq!(source, "true"); + } else { + unreachable!(); + } + } + #[test] fn build_basic_auth_should_always_work() { let basic_auth = HttpServerAuthConfig::Basic { From 79f06b560b46c44d21f59c59722ac9d83e56709f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Wed, 29 Jan 2025 18:03:56 +0100 Subject: [PATCH 08/14] Remove breaking change flag on changelog entry --- changelog.d/22236_custom_server_auth_strategy.breaking.md | 8 -------- changelog.d/22236_custom_server_auth_strategy.feature.md | 5 +++++ 2 files changed, 5 insertions(+), 8 deletions(-) delete mode 100644 changelog.d/22236_custom_server_auth_strategy.breaking.md create mode 100644 changelog.d/22236_custom_server_auth_strategy.feature.md diff --git a/changelog.d/22236_custom_server_auth_strategy.breaking.md b/changelog.d/22236_custom_server_auth_strategy.breaking.md deleted file mode 100644 index cc2cfe6e6efcf..0000000000000 --- a/changelog.d/22236_custom_server_auth_strategy.breaking.md +++ /dev/null @@ -1,8 +0,0 @@ -Custom authorization strategy is now supported for sources running -HTTP servers (`http_server` source, `prometheus` source, `datadog_agent`, etc.). - -Since there are now multiple authorization strategies, if you are using `auth` in any -of these supported components, you now also need to add `strategy: "basic"`, together with -`user` and `password`. - -authors: esensar diff --git a/changelog.d/22236_custom_server_auth_strategy.feature.md b/changelog.d/22236_custom_server_auth_strategy.feature.md new file mode 100644 index 0000000000000..b4b6dadd46405 --- /dev/null +++ b/changelog.d/22236_custom_server_auth_strategy.feature.md @@ -0,0 +1,5 @@ +Custom authorization strategy is now supported for sources running +HTTP servers (`http_server` source, `prometheus` source, `datadog_agent`, etc.). +If strategy is not explicitly defined, it defaults to `basic`, which is the current behavior. + +authors: esensar From cbb0061a47908939e78f8891240ca786111be69a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Wed, 29 Jan 2025 19:54:54 +0100 Subject: [PATCH 09/14] Add missing docs to `common/http/error` --- src/common/http/error.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/common/http/error.rs b/src/common/http/error.rs index da39e86cf1156..2edef0b948d99 100644 --- a/src/common/http/error.rs +++ b/src/common/http/error.rs @@ -1,8 +1,8 @@ -#![allow(missing_docs)] use std::{error::Error, fmt}; use serde::Serialize; +/// HTTP error, containing HTTP status code and a message #[derive(Serialize, Debug)] pub struct ErrorMessage { code: u16, @@ -16,6 +16,7 @@ pub struct ErrorMessage { feature = "sources-datadog_agent" ))] impl ErrorMessage { + /// Create a new `ErrorMessage` from HTTP status code and a message #[allow(unused)] // triggered by check-component-features pub fn new(code: http::StatusCode, message: String) -> Self { ErrorMessage { @@ -24,6 +25,7 @@ impl ErrorMessage { } } + /// Returns the HTTP status code #[allow(unused)] // triggered by check-component-features pub fn status_code(&self) -> http::StatusCode { http::StatusCode::from_u16(self.code).unwrap_or(http::StatusCode::INTERNAL_SERVER_ERROR) @@ -32,10 +34,12 @@ impl ErrorMessage { #[cfg(feature = "sources-utils-http-prelude")] impl ErrorMessage { + /// Returns the raw HTTP status code pub const fn code(&self) -> u16 { self.code } + /// Returns the error message pub fn message(&self) -> &str { self.message.as_str() } From 6b5c5bf0d6ee4856e9185d2a684169be7917864e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Wed, 29 Jan 2025 20:02:22 +0100 Subject: [PATCH 10/14] Make tests clearer by using `panic` instead of matching enum variant --- src/common/http/server_auth.rs | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/common/http/server_auth.rs b/src/common/http/server_auth.rs index bdeb416d16921..aa53b70285f8d 100644 --- a/src/common/http/server_auth.rs +++ b/src/common/http/server_auth.rs @@ -282,58 +282,53 @@ mod tests { #[test] fn config_should_default_to_basic() { - let config = indoc! { r#" + let config: HttpServerAuthConfig = serde_yaml::from_str(indoc! { r#" username: foo password: bar "# - }; - - let config: HttpServerAuthConfig = serde_yaml::from_str(config).unwrap(); + }) + .unwrap(); - assert!(matches!(config, HttpServerAuthConfig::Basic { .. })); if let HttpServerAuthConfig::Basic { username, password } = config { assert_eq!(username, "foo"); assert_eq!(password.inner(), "bar"); } else { - unreachable!(); + panic!("Expected HttpServerAuthConfig::Basic"); } } #[test] fn config_should_support_explicit_basic_strategy() { - let config = indoc! { r#" + let config: HttpServerAuthConfig = serde_yaml::from_str(indoc! { r#" strategy: basic username: foo password: bar "# - }; + }) + .unwrap(); - let config: HttpServerAuthConfig = serde_yaml::from_str(config).unwrap(); - - assert!(matches!(config, HttpServerAuthConfig::Basic { .. })); if let HttpServerAuthConfig::Basic { username, password } = config { assert_eq!(username, "foo"); assert_eq!(password.inner(), "bar"); } else { - unreachable!(); + panic!("Expected HttpServerAuthConfig::Basic"); } } #[test] fn config_should_support_custom_strategy() { - let config = indoc! { r#" + let config: HttpServerAuthConfig = serde_yaml::from_str(indoc! { r#" strategy: custom source: "true" "# - }; - - let config: HttpServerAuthConfig = serde_yaml::from_str(config).unwrap(); + }) + .unwrap(); assert!(matches!(config, HttpServerAuthConfig::Custom { .. })); if let HttpServerAuthConfig::Custom { source } = config { assert_eq!(source, "true"); } else { - unreachable!(); + panic!("Expected HttpServerAuthConfig::Custom"); } } From b4f88891e487bd0cf909d85d6bb0f559cf689629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Wed, 29 Jan 2025 20:02:52 +0100 Subject: [PATCH 11/14] Move out `VRL` auth handling into a separate function for readability --- src/common/http/server_auth.rs | 90 ++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/src/common/http/server_auth.rs b/src/common/http/server_auth.rs index aa53b70285f8d..1c84fbf96a79a 100644 --- a/src/common/http/server_auth.rs +++ b/src/common/http/server_auth.rs @@ -224,51 +224,57 @@ impl HttpServerAuthMatcher { )) } } - HttpServerAuthMatcher::Vrl { program } => { - let mut target = VrlTarget::new( - Event::Log(LogEvent::from_map( - ObjectMap::from([( - "headers".into(), - Value::Object( - headers - .iter() - .map(|(k, v)| { - ( - KeyString::from(k.to_string()), - Value::Bytes(Bytes::copy_from_slice(v.as_bytes())), - ) - }) - .collect::(), - ), - )]), - Default::default(), - )), - program.info(), - false, - ); - let timezone = TimeZone::default(); - - let result = Runtime::default().resolve(&mut target, program, &timezone); - match result.map_err(|e| { - warn!("Handling auth failed: {}", e); - ErrorMessage::new(StatusCode::UNAUTHORIZED, "Auth failed".to_owned()) - })? { - vrl::core::Value::Boolean(result) => { - if result { - Ok(()) - } else { - Err(ErrorMessage::new( - StatusCode::UNAUTHORIZED, - "Auth failed".to_owned(), - )) - } - } - _ => Err(ErrorMessage::new( + HttpServerAuthMatcher::Vrl { program } => self.handle_vrl_auth(headers, program), + } + } + + fn handle_vrl_auth( + &self, + headers: &HeaderMap, + program: &Program, + ) -> Result<(), ErrorMessage> { + let mut target = VrlTarget::new( + Event::Log(LogEvent::from_map( + ObjectMap::from([( + "headers".into(), + Value::Object( + headers + .iter() + .map(|(k, v)| { + ( + KeyString::from(k.to_string()), + Value::Bytes(Bytes::copy_from_slice(v.as_bytes())), + ) + }) + .collect::(), + ), + )]), + Default::default(), + )), + program.info(), + false, + ); + let timezone = TimeZone::default(); + + let result = Runtime::default().resolve(&mut target, program, &timezone); + match result.map_err(|e| { + warn!("Handling auth failed: {}", e); + ErrorMessage::new(StatusCode::UNAUTHORIZED, "Auth failed".to_owned()) + })? { + vrl::core::Value::Boolean(result) => { + if result { + Ok(()) + } else { + Err(ErrorMessage::new( StatusCode::UNAUTHORIZED, - "Invalid return value".to_owned(), - )), + "Auth failed".to_owned(), + )) } } + _ => Err(ErrorMessage::new( + StatusCode::UNAUTHORIZED, + "Invalid return value".to_owned(), + )), } } } From bf2a801c614fc970268c243e3f7e38839e5715ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Mon, 10 Feb 2025 17:19:43 +0100 Subject: [PATCH 12/14] Clean up changelog text Co-authored-by: Pavlos Rontidis --- changelog.d/22236_custom_server_auth_strategy.feature.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/changelog.d/22236_custom_server_auth_strategy.feature.md b/changelog.d/22236_custom_server_auth_strategy.feature.md index b4b6dadd46405..ed06c7c54f89f 100644 --- a/changelog.d/22236_custom_server_auth_strategy.feature.md +++ b/changelog.d/22236_custom_server_auth_strategy.feature.md @@ -1,5 +1,3 @@ -Custom authorization strategy is now supported for sources running -HTTP servers (`http_server` source, `prometheus` source, `datadog_agent`, etc.). -If strategy is not explicitly defined, it defaults to `basic`, which is the current behavior. +Sources running HTTP servers (`http_server` source, `prometheus` source, `datadog_agent`, etc.) now support a new `custom` authorization strategy . If a strategy is not explicitly defined, it defaults to `basic`, which is the current behavior. authors: esensar From 8e20c376d6dcb886c55551e7f3728cf1beb0ce60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Mon, 10 Feb 2025 17:25:59 +0100 Subject: [PATCH 13/14] Move test only functions in `server_auth` to `tests` module --- src/common/http/server_auth.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/common/http/server_auth.rs b/src/common/http/server_auth.rs index 1c84fbf96a79a..ff0111f0b0652 100644 --- a/src/common/http/server_auth.rs +++ b/src/common/http/server_auth.rs @@ -192,18 +192,6 @@ pub enum HttpServerAuthMatcher { } impl HttpServerAuthMatcher { - #[cfg(test)] - fn auth_header(self) -> (HeaderValue, &'static str) { - match self { - HttpServerAuthMatcher::AuthHeader(header_value, error_message) => { - (header_value, error_message) - } - HttpServerAuthMatcher::Vrl { .. } => { - panic!("Expected HttpServerAuthMatcher::AuthHeader") - } - } - } - /// Compares passed headers to the matcher pub fn handle_auth(&self, headers: &HeaderMap) -> Result<(), ErrorMessage> { match self { @@ -286,6 +274,19 @@ mod tests { use super::*; + impl HttpServerAuthMatcher { + fn auth_header(self) -> (HeaderValue, &'static str) { + match self { + HttpServerAuthMatcher::AuthHeader(header_value, error_message) => { + (header_value, error_message) + } + HttpServerAuthMatcher::Vrl { .. } => { + panic!("Expected HttpServerAuthMatcher::AuthHeader") + } + } + } + } + #[test] fn config_should_default_to_basic() { let config: HttpServerAuthConfig = serde_yaml::from_str(indoc! { r#" From 60d5c191126780c1c37fd7d501e481f4ad5892f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Mon, 10 Feb 2025 19:51:16 +0100 Subject: [PATCH 14/14] Return serde tag for correct docs generation --- src/common/http/server_auth.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/http/server_auth.rs b/src/common/http/server_auth.rs index ff0111f0b0652..ea10ea0a25f59 100644 --- a/src/common/http/server_auth.rs +++ b/src/common/http/server_auth.rs @@ -32,6 +32,7 @@ use super::ErrorMessage; #[configurable_component(no_deser)] #[derive(Clone, Debug, Eq, PartialEq)] #[configurable(metadata(docs::enum_tag_description = "The authentication strategy to use."))] +#[serde(tag = "strategy", rename_all = "snake_case")] pub enum HttpServerAuthConfig { /// Basic authentication. ///