feat(monitoring): wire up full LGTM observability stack

- Prometheus: discover ServiceMonitors/PodMonitors in all namespaces,
  enable remote write receiver for Tempo metrics generator
- Tempo: enable metrics generator (service-graphs + span-metrics)
  with remote write to Prometheus
- Loki: add Grafana Alloy DaemonSet to ship container logs
- Grafana: enable dashboard sidecar, add Pingora/Loki/Tempo/OpenBao
  dashboards, add stable UIDs and cross-linking between datasources
  (Loki↔Tempo derived fields, traces→logs, traces→metrics, service map)
- Linkerd: enable proxy tracing to Alloy OTLP collector, point
  linkerd-viz at existing Prometheus instead of deploying its own
- Pingora: add OTLP rollout plan (endpoint commented out until proxy
  telemetry panic fix is deployed and Alloy is verified healthy)
This commit is contained in:
2026-03-21 17:36:54 +00:00
parent 5f923d14f9
commit d3943c9a84
9 changed files with 523 additions and 0 deletions

View File

@@ -21,6 +21,13 @@ data:
key_path = "/etc/tls/tls.key"
[telemetry]
# Rollout plan for OTLP tracing:
# 1. Deploy proxy build that includes the graceful telemetry init
# (proxy/src/telemetry.rs — no longer panics on exporter failure)
# 2. Verify Alloy is running:
# kubectl -n monitoring get pods -l app.kubernetes.io/name=alloy
# 3. Uncomment the line below:
# otlp_endpoint = "http://alloy.monitoring.svc.cluster.local:4318"
otlp_endpoint = ""
metrics_port = 9090