feat: add PrometheusRule alerts for all services
28 alert rules across 9 PrometheusRule files covering infrastructure (Longhorn, cert-manager), data (PostgreSQL, OpenBao, OpenSearch), storage (SeaweedFS), devtools (Gitea), identity (Hydra, Kratos), media (LiveKit), and mesh (Linkerd golden signals for all services). Severity routing: critical alerts fire to Matrix + email, warnings to Matrix only (AlertManager config updated in separate commit).
This commit is contained in:
37
base/data/opensearch-alertrules.yaml
Normal file
37
base/data/opensearch-alertrules.yaml
Normal file
@@ -0,0 +1,37 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: opensearch-alerts
|
||||
namespace: data
|
||||
labels:
|
||||
role: alert-rules
|
||||
spec:
|
||||
groups:
|
||||
- name: opensearch
|
||||
rules:
|
||||
- alert: OpenSearchClusterRed
|
||||
expr: opensearch_cluster_health_status{color="red"} == 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "OpenSearch cluster health is RED"
|
||||
description: "OpenSearch cluster {{ $labels.cluster }} health status is red."
|
||||
|
||||
- alert: OpenSearchClusterYellow
|
||||
expr: opensearch_cluster_health_status{color="yellow"} == 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "OpenSearch cluster health is YELLOW"
|
||||
description: "OpenSearch cluster {{ $labels.cluster }} health status is yellow."
|
||||
|
||||
- alert: OpenSearchHeapHigh
|
||||
expr: (opensearch_jvm_mem_heap_used_bytes / opensearch_jvm_mem_heap_max_bytes) > 0.85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "OpenSearch JVM heap usage is high"
|
||||
description: "OpenSearch node {{ $labels.node }} in {{ $labels.namespace }} heap usage is above 85%."
|
||||
Reference in New Issue
Block a user