From 0fd10110ff79c7b8532e2d75283375a17daf47da Mon Sep 17 00:00:00 2001 From: Sienna Meridian Satterwhite Date: Tue, 10 Mar 2026 23:38:20 +0000 Subject: [PATCH] feat(proxy): add request IDs, tracing spans, and observability hooks Generate UUID v4 request IDs per request, create manual tracing spans (Pingora types don't impl Debug), record Prometheus metrics for detection decisions and request totals, and forward X-Request-Id to both upstream requests and downstream responses. Signed-off-by: Sienna Meridian Satterwhite --- Cargo.lock | 205 ++++++++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 3 + src/proxy.rs | 81 +++++++++++++++++++- 3 files changed, 283 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 10e14c4..4efb8eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1038,6 +1038,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foldhash" version = "0.2.0" @@ -1176,10 +1182,23 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", ] +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + [[package]] name = "getset" version = "0.1.6" @@ -1252,6 +1271,15 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -1260,7 +1288,7 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -1501,6 +1529,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "ident_case" version = "1.0.1" @@ -1546,6 +1580,8 @@ checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", "hashbrown 0.16.1", + "serde", + "serde_core", ] [[package]] @@ -1794,6 +1830,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "libc" version = "0.2.183" @@ -2697,6 +2739,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.117", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -2824,6 +2876,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.8.5" @@ -3459,6 +3517,7 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", + "uuid", ] [[package]] @@ -3976,6 +4035,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -4012,6 +4077,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -4058,6 +4134,15 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.114" @@ -4117,6 +4202,40 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap 2.13.0", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.11.0", + "hashbrown 0.15.5", + "indexmap 2.13.0", + "semver", +] + [[package]] name = "web-sys" version = "0.3.91" @@ -4331,6 +4450,88 @@ name = "wit-bindgen" version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck 0.5.0", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap 2.13.0", + "prettyplease", + "syn 2.0.117", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.11.0", + "indexmap 2.13.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.13.0", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] [[package]] name = "writeable" diff --git a/Cargo.toml b/Cargo.toml index f7e0c22..fb50451 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,9 @@ dns-lookup = "2" # Prometheus metrics prometheus = "0.13" +# Request IDs +uuid = { version = "1", features = ["v4"] } + # Rustls crypto provider — must be installed before any TLS init rustls = { version = "0.23", features = ["aws-lc-rs"] } diff --git a/src/proxy.rs b/src/proxy.rs index d195f45..eff2f49 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -2,6 +2,7 @@ use crate::acme::AcmeRoutes; use crate::config::RouteConfig; use crate::ddos::detector::DDoSDetector; use crate::ddos::model::DDoSAction; +use crate::metrics; use crate::rate_limit::key; use crate::rate_limit::limiter::{RateLimitResult, RateLimiter}; use crate::scanner::allowlist::BotAllowlist; @@ -20,7 +21,6 @@ use std::time::Instant; pub struct SunbeamProxy { pub routes: Vec, /// Per-challenge route table populated by the Ingress watcher. - /// Maps `/.well-known/acme-challenge/` → solver service address. pub acme_routes: AcmeRoutes, /// Optional KNN-based DDoS detector. pub ddos_detector: Option>, @@ -35,6 +35,10 @@ pub struct SunbeamProxy { pub struct RequestCtx { pub route: Option, pub start_time: Instant, + /// Unique request identifier (UUID v4). + pub request_id: String, + /// Tracing span for this request. + pub span: tracing::Span, /// Resolved solver backend address for this ACME challenge, if applicable. pub acme_backend: Option, /// Path prefix to strip before forwarding to the upstream (e.g. "/kratos"). @@ -91,7 +95,7 @@ fn extract_client_ip(session: &Session) -> Option { } /// Strip the scheme prefix from a backend URL like `http://host:port`. -fn backend_addr(backend: &str) -> &str { +pub fn backend_addr(backend: &str) -> &str { backend .trim_start_matches("https://") .trim_start_matches("http://") @@ -110,9 +114,12 @@ impl ProxyHttp for SunbeamProxy { type CTX = RequestCtx; fn new_ctx(&self) -> RequestCtx { + let request_id = uuid::Uuid::new_v4().to_string(); RequestCtx { route: None, start_time: Instant::now(), + request_id, + span: tracing::Span::none(), acme_backend: None, downstream_scheme: "https", strip_prefix: None, @@ -130,6 +137,19 @@ impl ProxyHttp for SunbeamProxy { { ctx.downstream_scheme = if is_plain_http(session) { "http" } else { "https" }; + // Create the request-scoped tracing span. + let method = session.req_header().method.to_string(); + let host = extract_host(session); + let path = session.req_header().uri.path().to_string(); + ctx.span = tracing::info_span!("request", + request_id = %ctx.request_id, + method = %method, + host = %host, + path = %path, + ); + + metrics::ACTIVE_CONNECTIONS.inc(); + if is_plain_http(session) { let path = session.req_header().uri.path().to_string(); @@ -157,7 +177,6 @@ impl ProxyHttp for SunbeamProxy { } // All other plain-HTTP traffic. - let host = extract_host(session); let prefix = host.split('.').next().unwrap_or(""); // Routes that explicitly opt out of HTTPS enforcement pass through. @@ -242,6 +261,8 @@ impl ProxyHttp for SunbeamProxy { "pipeline" ); + metrics::DDOS_DECISIONS.with_label_values(&[decision]).inc(); + if matches!(ddos_action, DDoSAction::Block) { let mut resp = ResponseHeader::build(429, None)?; resp.insert_header("Retry-After", "60")?; @@ -325,6 +346,10 @@ impl ProxyHttp for SunbeamProxy { "pipeline" ); + metrics::SCANNER_DECISIONS + .with_label_values(&[decision, &reason]) + .inc(); + if decision == "block" { let mut resp = ResponseHeader::build(403, None)?; resp.insert_header("Content-Length", "0")?; @@ -363,6 +388,10 @@ impl ProxyHttp for SunbeamProxy { "pipeline" ); + metrics::RATE_LIMIT_DECISIONS + .with_label_values(&[decision]) + .inc(); + if let RateLimitResult::Reject { retry_after } = rl_result { let mut resp = ResponseHeader::build(429, None)?; resp.insert_header("Retry-After", retry_after.to_string())?; @@ -491,6 +520,15 @@ impl ProxyHttp for SunbeamProxy { ) })?; + // Forward X-Request-Id to upstream. + upstream_req.insert_header("x-request-id", &ctx.request_id).map_err(|e| { + pingora_core::Error::because( + pingora_core::ErrorType::InternalError, + "failed to insert x-request-id", + e, + ) + })?; + if ctx.route.as_ref().map(|r| r.websocket).unwrap_or(false) { for name in &[CONNECTION, UPGRADE] { if let Some(val) = session.req_header().headers.get(name.clone()) { @@ -535,6 +573,20 @@ impl ProxyHttp for SunbeamProxy { Ok(()) } + /// Add X-Request-Id response header so clients can correlate. + async fn upstream_response_filter( + &self, + _session: &mut Session, + upstream_response: &mut ResponseHeader, + ctx: &mut RequestCtx, + ) -> Result<()> + where + Self::CTX: Send + Sync, + { + let _ = upstream_response.insert_header("x-request-id", &ctx.request_id); + Ok(()) + } + /// Emit a structured JSON audit log line for every request. async fn logging( &self, @@ -544,10 +596,15 @@ impl ProxyHttp for SunbeamProxy { ) where Self::CTX: Send + Sync, { + metrics::ACTIVE_CONNECTIONS.dec(); + let status = session .response_written() .map_or(0, |r| r.status.as_u16()); let duration_ms = ctx.start_time.elapsed().as_millis() as u64; + let duration_secs = ctx.start_time.elapsed().as_secs_f64(); + let method_str = session.req_header().method.to_string(); + let host = extract_host(session); let backend = ctx .route .as_ref() @@ -563,6 +620,12 @@ impl ProxyHttp for SunbeamProxy { }); let error_str = error.map(|e| e.to_string()); + // Record Prometheus metrics. + metrics::REQUESTS_TOTAL + .with_label_values(&[&method_str, &host, &status.to_string(), backend]) + .inc(); + metrics::REQUEST_DURATION.observe(duration_secs); + let content_length: u64 = session .req_header() .headers @@ -609,8 +672,9 @@ impl ProxyHttp for SunbeamProxy { tracing::info!( target = "audit", + request_id = %ctx.request_id, method = %session.req_header().method, - host = %extract_host(session), + host = %host, path = %session.req_header().uri.path(), query, client_ip, @@ -678,6 +742,8 @@ mod tests { let ctx = RequestCtx { route: None, start_time: Instant::now(), + request_id: "1".to_string(), + span: tracing::Span::none(), acme_backend: None, strip_prefix: None, downstream_scheme: "https", @@ -705,4 +771,11 @@ mod tests { // Content-Length must survive the strip. assert!(req.headers.get("content-length").is_some()); } + + #[test] + fn test_request_id_is_uuid_v4() { + let id = uuid::Uuid::new_v4().to_string(); + assert_eq!(id.len(), 36); + assert!(uuid::Uuid::parse_str(&id).is_ok()); + } }