feat(proxy): add request IDs, tracing spans, and observability hooks

Generate UUID v4 request IDs per request, create manual tracing spans
(Pingora types don't impl Debug), record Prometheus metrics for
detection decisions and request totals, and forward X-Request-Id to
both upstream requests and downstream responses.

Signed-off-by: Sienna Meridian Satterwhite <sienna@sunbeam.pt>

Cargo.lock

@@ -1038,6 +1038,12 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foldhash"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
+
 [[package]]
 name = "foldhash"
 version = "0.2.0"
@@ -1176,10 +1182,23 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
 dependencies = [
  "cfg-if",
  "libc",
- "r-efi",
+ "r-efi 5.3.0",
  "wasip2",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi 6.0.0",
+ "wasip2",
+ "wasip3",
+]
+
 [[package]]
 name = "getset"
 version = "0.1.6"
@@ -1252,6 +1271,15 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
+[[package]]
+name = "hashbrown"
+version = "0.15.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
+dependencies = [
+ "foldhash 0.1.5",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.16.1"
@@ -1260,7 +1288,7 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
 dependencies = [
  "allocator-api2",
  "equivalent",
- "foldhash",
+ "foldhash 0.2.0",
 ]
 
 [[package]]
@@ -1501,6 +1529,12 @@ dependencies = [
  "zerovec",
 ]
 
+[[package]]
+name = "id-arena"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
+
 [[package]]
 name = "ident_case"
 version = "1.0.1"
@@ -1546,6 +1580,8 @@ checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
 dependencies = [
  "equivalent",
  "hashbrown 0.16.1",
+ "serde",
+ "serde_core",
 ]
 
 [[package]]
@@ -1794,6 +1830,12 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
+[[package]]
+name = "leb128fmt"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
+
 [[package]]
 name = "libc"
 version = "0.2.183"
@@ -2697,6 +2739,16 @@ dependencies = [
  "zerocopy",
 ]
 
+[[package]]
+name = "prettyplease"
+version = "0.2.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
+dependencies = [
+ "proc-macro2",
+ "syn 2.0.117",
+]
+
 [[package]]
 name = "proc-macro-error"
 version = "1.0.4"
@@ -2824,6 +2876,12 @@ version = "5.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
 
+[[package]]
+name = "r-efi"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
+
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -3459,6 +3517,7 @@ dependencies = [
  "tracing",
  "tracing-opentelemetry",
  "tracing-subscriber",
+ "uuid",
 ]
 
 [[package]]
@@ -3976,6 +4035,12 @@ version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
 
+[[package]]
+name = "unicode-xid"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
+
 [[package]]
 name = "unsafe-libyaml"
 version = "0.2.11"
@@ -4012,6 +4077,17 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
+[[package]]
+name = "uuid"
+version = "1.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37"
+dependencies = [
+ "getrandom 0.4.2",
+ "js-sys",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "valuable"
 version = "0.1.1"
@@ -4058,6 +4134,15 @@ dependencies = [
  "wit-bindgen",
 ]
 
+[[package]]
+name = "wasip3"
+version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
+dependencies = [
+ "wit-bindgen",
+]
+
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.114"
@@ -4117,6 +4202,40 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "wasm-encoder"
+version = "0.244.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"
+dependencies = [
+ "leb128fmt",
+ "wasmparser",
+]
+
+[[package]]
+name = "wasm-metadata"
+version = "0.244.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
+dependencies = [
+ "anyhow",
+ "indexmap 2.13.0",
+ "wasm-encoder",
+ "wasmparser",
+]
+
+[[package]]
+name = "wasmparser"
+version = "0.244.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
+dependencies = [
+ "bitflags 2.11.0",
+ "hashbrown 0.15.5",
+ "indexmap 2.13.0",
+ "semver",
+]
+
 [[package]]
 name = "web-sys"
 version = "0.3.91"
@@ -4331,6 +4450,88 @@ name = "wit-bindgen"
 version = "0.51.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
+dependencies = [
+ "wit-bindgen-rust-macro",
+]
+
+[[package]]
+name = "wit-bindgen-core"
+version = "0.51.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"
+dependencies = [
+ "anyhow",
+ "heck 0.5.0",
+ "wit-parser",
+]
+
+[[package]]
+name = "wit-bindgen-rust"
+version = "0.51.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
+dependencies = [
+ "anyhow",
+ "heck 0.5.0",
+ "indexmap 2.13.0",
+ "prettyplease",
+ "syn 2.0.117",
+ "wasm-metadata",
+ "wit-bindgen-core",
+ "wit-component",
+]
+
+[[package]]
+name = "wit-bindgen-rust-macro"
+version = "0.51.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"
+dependencies = [
+ "anyhow",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+ "wit-bindgen-core",
+ "wit-bindgen-rust",
+]
+
+[[package]]
+name = "wit-component"
+version = "0.244.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
+dependencies = [
+ "anyhow",
+ "bitflags 2.11.0",
+ "indexmap 2.13.0",
+ "log",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "wasm-encoder",
+ "wasm-metadata",
+ "wasmparser",
+ "wit-parser",
+]
+
+[[package]]
+name = "wit-parser"
+version = "0.244.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
+dependencies = [
+ "anyhow",
+ "id-arena",
+ "indexmap 2.13.0",
+ "log",
+ "semver",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "unicode-xid",
+ "wasmparser",
+]
 
 [[package]]
 name = "writeable"

Cargo.toml

@@ -52,6 +52,9 @@ dns-lookup  = "2"
 # Prometheus metrics
 prometheus  = "0.13"
 
+# Request IDs
+uuid = { version = "1", features = ["v4"] }
+
 # Rustls crypto provider — must be installed before any TLS init
 rustls      = { version = "0.23", features = ["aws-lc-rs"] }
 

src/proxy.rs

@@ -2,6 +2,7 @@ use crate::acme::AcmeRoutes;
 use crate::config::RouteConfig;
 use crate::ddos::detector::DDoSDetector;
 use crate::ddos::model::DDoSAction;
+use crate::metrics;
 use crate::rate_limit::key;
 use crate::rate_limit::limiter::{RateLimitResult, RateLimiter};
 use crate::scanner::allowlist::BotAllowlist;
@@ -20,7 +21,6 @@ use std::time::Instant;
 pub struct SunbeamProxy {
     pub routes: Vec<RouteConfig>,
     /// Per-challenge route table populated by the Ingress watcher.
-    /// Maps `/.well-known/acme-challenge/<token>` → solver service address.
     pub acme_routes: AcmeRoutes,
     /// Optional KNN-based DDoS detector.
     pub ddos_detector: Option<Arc<DDoSDetector>>,
@@ -35,6 +35,10 @@ pub struct SunbeamProxy {
 pub struct RequestCtx {
     pub route: Option<RouteConfig>,
     pub start_time: Instant,
+    /// Unique request identifier (UUID v4).
+    pub request_id: String,
+    /// Tracing span for this request.
+    pub span: tracing::Span,
     /// Resolved solver backend address for this ACME challenge, if applicable.
     pub acme_backend: Option<String>,
     /// Path prefix to strip before forwarding to the upstream (e.g. "/kratos").
@@ -91,7 +95,7 @@ fn extract_client_ip(session: &Session) -> Option<IpAddr> {
 }
 
 /// Strip the scheme prefix from a backend URL like `http://host:port`.
-fn backend_addr(backend: &str) -> &str {
+pub fn backend_addr(backend: &str) -> &str {
     backend
         .trim_start_matches("https://")
         .trim_start_matches("http://")
@@ -110,9 +114,12 @@ impl ProxyHttp for SunbeamProxy {
     type CTX = RequestCtx;
 
     fn new_ctx(&self) -> RequestCtx {
+        let request_id = uuid::Uuid::new_v4().to_string();
         RequestCtx {
             route: None,
             start_time: Instant::now(),
+            request_id,
+            span: tracing::Span::none(),
             acme_backend: None,
             downstream_scheme: "https",
             strip_prefix: None,
@@ -130,6 +137,19 @@ impl ProxyHttp for SunbeamProxy {
     {
         ctx.downstream_scheme = if is_plain_http(session) { "http" } else { "https" };
 
+        // Create the request-scoped tracing span.
+        let method = session.req_header().method.to_string();
+        let host = extract_host(session);
+        let path = session.req_header().uri.path().to_string();
+        ctx.span = tracing::info_span!("request",
+            request_id = %ctx.request_id,
+            method = %method,
+            host = %host,
+            path = %path,
+        );
+
+        metrics::ACTIVE_CONNECTIONS.inc();
+
         if is_plain_http(session) {
             let path = session.req_header().uri.path().to_string();
 
@@ -157,7 +177,6 @@ impl ProxyHttp for SunbeamProxy {
             }
 
             // All other plain-HTTP traffic.
-            let host = extract_host(session);
             let prefix = host.split('.').next().unwrap_or("");
 
             // Routes that explicitly opt out of HTTPS enforcement pass through.
@@ -242,6 +261,8 @@ impl ProxyHttp for SunbeamProxy {
                     "pipeline"
                 );
 
+                metrics::DDOS_DECISIONS.with_label_values(&[decision]).inc();
+
                 if matches!(ddos_action, DDoSAction::Block) {
                     let mut resp = ResponseHeader::build(429, None)?;
                     resp.insert_header("Retry-After", "60")?;
@@ -325,6 +346,10 @@ impl ProxyHttp for SunbeamProxy {
                 "pipeline"
             );
 
+            metrics::SCANNER_DECISIONS
+                .with_label_values(&[decision, &reason])
+                .inc();
+
             if decision == "block" {
                 let mut resp = ResponseHeader::build(403, None)?;
                 resp.insert_header("Content-Length", "0")?;
@@ -363,6 +388,10 @@ impl ProxyHttp for SunbeamProxy {
                     "pipeline"
                 );
 
+                metrics::RATE_LIMIT_DECISIONS
+                    .with_label_values(&[decision])
+                    .inc();
+
                 if let RateLimitResult::Reject { retry_after } = rl_result {
                     let mut resp = ResponseHeader::build(429, None)?;
                     resp.insert_header("Retry-After", retry_after.to_string())?;
@@ -491,6 +520,15 @@ impl ProxyHttp for SunbeamProxy {
                 )
             })?;
 
+        // Forward X-Request-Id to upstream.
+        upstream_req.insert_header("x-request-id", &ctx.request_id).map_err(|e| {
+            pingora_core::Error::because(
+                pingora_core::ErrorType::InternalError,
+                "failed to insert x-request-id",
+                e,
+            )
+        })?;
+
         if ctx.route.as_ref().map(|r| r.websocket).unwrap_or(false) {
             for name in &[CONNECTION, UPGRADE] {
                 if let Some(val) = session.req_header().headers.get(name.clone()) {
@@ -535,6 +573,20 @@ impl ProxyHttp for SunbeamProxy {
         Ok(())
     }
 
+    /// Add X-Request-Id response header so clients can correlate.
+    async fn upstream_response_filter(
+        &self,
+        _session: &mut Session,
+        upstream_response: &mut ResponseHeader,
+        ctx: &mut RequestCtx,
+    ) -> Result<()>
+    where
+        Self::CTX: Send + Sync,
+    {
+        let _ = upstream_response.insert_header("x-request-id", &ctx.request_id);
+        Ok(())
+    }
+
     /// Emit a structured JSON audit log line for every request.
     async fn logging(
         &self,
@@ -544,10 +596,15 @@ impl ProxyHttp for SunbeamProxy {
     ) where
         Self::CTX: Send + Sync,
     {
+        metrics::ACTIVE_CONNECTIONS.dec();
+
         let status = session
             .response_written()
             .map_or(0, |r| r.status.as_u16());
         let duration_ms = ctx.start_time.elapsed().as_millis() as u64;
+        let duration_secs = ctx.start_time.elapsed().as_secs_f64();
+        let method_str = session.req_header().method.to_string();
+        let host = extract_host(session);
         let backend = ctx
             .route
             .as_ref()
@@ -563,6 +620,12 @@ impl ProxyHttp for SunbeamProxy {
             });
         let error_str = error.map(|e| e.to_string());
 
+        // Record Prometheus metrics.
+        metrics::REQUESTS_TOTAL
+            .with_label_values(&[&method_str, &host, &status.to_string(), backend])
+            .inc();
+        metrics::REQUEST_DURATION.observe(duration_secs);
+
         let content_length: u64 = session
             .req_header()
             .headers
@@ -609,8 +672,9 @@ impl ProxyHttp for SunbeamProxy {
 
         tracing::info!(
             target = "audit",
+            request_id = %ctx.request_id,
             method  = %session.req_header().method,
-            host    = %extract_host(session),
+            host    = %host,
             path    = %session.req_header().uri.path(),
             query,
             client_ip,
@@ -678,6 +742,8 @@ mod tests {
         let ctx = RequestCtx {
             route: None,
             start_time: Instant::now(),
+            request_id: "1".to_string(),
+            span: tracing::Span::none(),
             acme_backend: None,
             strip_prefix: None,
             downstream_scheme: "https",
@@ -705,4 +771,11 @@ mod tests {
         // Content-Length must survive the strip.
         assert!(req.headers.get("content-length").is_some());
     }
+
+    #[test]
+    fn test_request_id_is_uuid_v4() {
+        let id = uuid::Uuid::new_v4().to_string();
+        assert_eq!(id.len(), 36);
+        assert!(uuid::Uuid::parse_str(&id).is_ok());
+    }
 }