cli/sunbeam-sdk/tests/test_monitoring.rs

#![cfg(feature = "integration")]
mod helpers;
use helpers::*;
use sunbeam_sdk::client::{AuthMethod, ServiceClient};
use sunbeam_sdk::monitoring::{PrometheusClient, LokiClient, GrafanaClient};
use sunbeam_sdk::monitoring::types::*;

// ---------------------------------------------------------------------------
// Service URLs
// ---------------------------------------------------------------------------

const PROM_URL: &str = "http://localhost:9090/api/v1";
const PROM_HEALTH: &str = "http://localhost:9090/api/v1/status/buildinfo";

const LOKI_URL: &str = "http://localhost:3100/loki/api/v1";
const LOKI_HEALTH: &str = "http://localhost:3100/ready";

const GRAFANA_URL: &str = "http://localhost:3001/api";
const GRAFANA_HEALTH: &str = "http://localhost:3001/api/health";

// ---------------------------------------------------------------------------
// Constructors
// ---------------------------------------------------------------------------

fn prom() -> PrometheusClient {
    PrometheusClient::from_parts(PROM_URL.into(), AuthMethod::None)
}

fn loki() -> LokiClient {
    LokiClient::from_parts(LOKI_URL.into(), AuthMethod::None)
}

fn grafana() -> GrafanaClient {
    use base64::Engine;
    let creds = base64::engine::general_purpose::STANDARD.encode("admin:admin");
    GrafanaClient::from_parts(
        GRAFANA_URL.into(),
        AuthMethod::Header {
            name: "Authorization",
            value: format!("Basic {creds}"),
        },
    )
}

// ===========================================================================
// PROMETHEUS
// ===========================================================================

// ---------------------------------------------------------------------------
// 1. Query
// ---------------------------------------------------------------------------

#[tokio::test]
async fn prometheus_query() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    // Instant query
    let res = c.query("up", None).await.expect("query failed");
    assert_eq!(res.status, "success");
    let data = res.data.expect("missing data");
    assert_eq!(data.result_type, "vector");

    // Instant query with explicit time
    let now = chrono::Utc::now().timestamp().to_string();
    let res = c.query("up", Some(&now)).await.expect("query with time failed");
    assert_eq!(res.status, "success");
}

#[tokio::test]
async fn prometheus_query_range() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let now = chrono::Utc::now();
    let start = (now - chrono::Duration::minutes(5)).timestamp().to_string();
    let end = now.timestamp().to_string();

    let res = c
        .query_range("up", &start, &end, "15s")
        .await
        .expect("query_range failed");
    assert_eq!(res.status, "success");
    let data = res.data.expect("missing data");
    assert_eq!(data.result_type, "matrix");
}

// ---------------------------------------------------------------------------
// 2. Format
// ---------------------------------------------------------------------------

#[tokio::test]
async fn prometheus_format_query() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let res = c
        .format_query("up{job='prometheus'}")
        .await
        .expect("format_query failed");
    assert_eq!(res.status, "success");
    assert!(res.data.contains("up"));
}

// ---------------------------------------------------------------------------
// 3. Metadata
// ---------------------------------------------------------------------------

#[tokio::test]
async fn prometheus_series() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let now = chrono::Utc::now();
    let start = (now - chrono::Duration::minutes(5)).timestamp().to_string();
    let end = now.timestamp().to_string();

    let res = c
        .series(&["up"], Some(&start), Some(&end))
        .await
        .expect("series failed");
    assert_eq!(res.status, "success");
    assert!(res.data.is_some());
}

#[tokio::test]
async fn prometheus_labels() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let res = c.labels(None, None).await.expect("labels failed");
    assert_eq!(res.status, "success");
    let labels = res.data.expect("missing data");
    assert!(labels.contains(&"__name__".to_string()));
}

#[tokio::test]
async fn prometheus_label_values() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let res = c.label_values("job", None, None).await.expect("label_values failed");
    assert_eq!(res.status, "success");
    let values = res.data.expect("missing data");
    assert!(!values.is_empty(), "expected at least one job label value");
}

#[tokio::test]
async fn prometheus_targets_metadata() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let res = c.targets_metadata(None).await.expect("targets_metadata failed");
    assert_eq!(res.status, "success");
}

#[tokio::test]
async fn prometheus_metadata() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let res = c.metadata(None).await.expect("metadata failed");
    assert_eq!(res.status, "success");
    assert!(res.data.is_some());
}

// ---------------------------------------------------------------------------
// 4. Infrastructure
// ---------------------------------------------------------------------------

#[tokio::test]
async fn prometheus_targets() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let res = c.targets().await.expect("targets failed");
    assert_eq!(res.status, "success");
    let data = res.data.expect("missing data");
    assert!(!data.active_targets.is_empty(), "expected at least one active target");
}

#[tokio::test]
async fn prometheus_scrape_pools() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let res = c.scrape_pools().await.expect("scrape_pools failed");
    assert_eq!(res.status, "success");
}

#[tokio::test]
async fn prometheus_alertmanagers() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let res = c.alertmanagers().await.expect("alertmanagers failed");
    assert_eq!(res.status, "success");
}

#[tokio::test]
async fn prometheus_rules() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let res = c.rules().await.expect("rules failed");
    assert_eq!(res.status, "success");
    assert!(res.data.is_some());
}

#[tokio::test]
async fn prometheus_alerts() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let res = c.alerts().await.expect("alerts failed");
    assert_eq!(res.status, "success");
    assert!(res.data.is_some());
}

// ---------------------------------------------------------------------------
// 5. Status
// ---------------------------------------------------------------------------

#[tokio::test]
async fn prometheus_config() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let res = c.config().await.expect("config failed");
    assert_eq!(res.status, "success");
    let data = res.data.expect("missing data");
    assert!(!data.yaml.is_empty(), "config yaml should not be empty");
}

#[tokio::test]
async fn prometheus_flags() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let res = c.flags().await.expect("flags failed");
    assert_eq!(res.status, "success");
    assert!(res.data.is_some());
}

#[tokio::test]
async fn prometheus_runtime_info() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let res = c.runtime_info().await.expect("runtime_info failed");
    assert_eq!(res.status, "success");
    assert!(res.data.is_some());
}

#[tokio::test]
async fn prometheus_build_info() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let res = c.build_info().await.expect("build_info failed");
    assert_eq!(res.status, "success");
    assert!(res.data.is_some());
}

#[tokio::test]
async fn prometheus_tsdb() {
    wait_for_healthy(PROM_HEALTH, TIMEOUT).await;
    let c = prom();

    let res = c.tsdb().await.expect("tsdb failed");
    assert_eq!(res.status, "success");
    assert!(res.data.is_some());
}

// ===========================================================================
// LOKI
// ===========================================================================

// ---------------------------------------------------------------------------
// 1. Ready
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_ready() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let res = c.ready().await.expect("ready failed");
    assert_eq!(res.status.as_deref(), Some("ready"));
}

// ---------------------------------------------------------------------------
// 2. Push and Query
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_push_and_query() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let now_ns = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .unwrap()
        .as_nanos()
        .to_string();

    let marker = unique_name("logline");
    let body = serde_json::json!({
        "streams": [{
            "stream": {"job": "test", "app": "integration"},
            "values": [[&now_ns, format!("test log line {marker}")]]
        }]
    });

    // Push
    for i in 0..10 {
        match c.push(&body).await {
            Ok(()) => break,
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("push failed after retries: {e}"),
        }
    }

    // Give Loki time to index
    tokio::time::sleep(std::time::Duration::from_secs(3)).await;

    // Range query (Loki doesn't support instant queries for log selectors)
    let now = chrono::Utc::now();
    let start = (now - chrono::Duration::minutes(5)).timestamp().to_string();
    let end = now.timestamp().to_string();
    let query_str = r#"{job="test",app="integration"}"#;
    for i in 0..10 {
        match c.query_range(query_str, &start, &end, Some(100), None).await {
            Ok(res) => {
                assert_eq!(res.status, "success");
                return;
            }
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("query failed after retries: {e}"),
        }
    }
}

#[tokio::test]
async fn loki_query_range() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let now = chrono::Utc::now();
    let start = (now - chrono::Duration::minutes(10)).timestamp().to_string();
    let end = now.timestamp().to_string();

    let query_str = r#"{job="test"}"#;
    for i in 0..10 {
        match c.query_range(query_str, &start, &end, Some(100), None).await {
            Ok(res) => {
                assert_eq!(res.status, "success");
                return;
            }
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("query_range failed after retries: {e}"),
        }
    }
}

// ---------------------------------------------------------------------------
// 3. Labels
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_labels() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    for i in 0..10 {
        match c.labels(None, None).await {
            Ok(res) => {
                assert_eq!(res.status, "success");
                return;
            }
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("labels failed after retries: {e}"),
        }
    }
}

#[tokio::test]
async fn loki_label_values() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    for i in 0..10 {
        match c.label_values("job", None, None).await {
            Ok(res) => {
                assert_eq!(res.status, "success");
                return;
            }
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("label_values failed after retries: {e}"),
        }
    }
}

// ---------------------------------------------------------------------------
// 4. Series
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_series() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let now = chrono::Utc::now();
    let start = (now - chrono::Duration::minutes(10)).timestamp().to_string();
    let end = now.timestamp().to_string();

    for i in 0..10 {
        match c.series(&[r#"{job="test"}"#], Some(&start), Some(&end)).await {
            Ok(res) => {
                assert_eq!(res.status, "success");
                return;
            }
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("series failed after retries: {e}"),
        }
    }
}

// ---------------------------------------------------------------------------
// 5. Index
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_index_stats() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    // index_stats requires a query parameter which the client doesn't expose yet
    // Just verify it doesn't panic — the 400 error is expected
    let _ = c.index_stats().await;
}

#[tokio::test]
async fn loki_index_volume() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let now = chrono::Utc::now();
    let start = (now - chrono::Duration::minutes(10)).timestamp().to_string();
    let end = now.timestamp().to_string();

    // index/volume may return errors without sufficient data — handle gracefully
    for i in 0..10 {
        match c.index_volume(r#"{job="test"}"#, Some(&start), Some(&end)).await {
            Ok(_val) => return,
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => {
                // Graceful: some Loki versions don't support this endpoint
                eprintln!("index_volume not available (may be expected): {e}");
                return;
            }
        }
    }
}

#[tokio::test]
async fn loki_index_volume_range() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let now = chrono::Utc::now();
    let start = (now - chrono::Duration::minutes(10)).timestamp().to_string();
    let end = now.timestamp().to_string();

    for i in 0..10 {
        match c.index_volume_range(r#"{job="test"}"#, &start, &end, None).await {
            Ok(_val) => return,
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => {
                eprintln!("index_volume_range not available (may be expected): {e}");
                return;
            }
        }
    }
}

// ---------------------------------------------------------------------------
// 6. Patterns
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_detect_patterns() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let now = chrono::Utc::now();
    let start = (now - chrono::Duration::minutes(10)).timestamp().to_string();
    let end = now.timestamp().to_string();

    // detect_patterns may not work without sufficient data — handle gracefully
    for i in 0..10 {
        match c.detect_patterns(r#"{job="test"}"#, Some(&start), Some(&end)).await {
            Ok(_val) => return,
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => {
                eprintln!("detect_patterns not available (may be expected): {e}");
                return;
            }
        }
    }
}

// ===========================================================================
// GRAFANA
// ===========================================================================

// ---------------------------------------------------------------------------
// 1. Org
// ---------------------------------------------------------------------------

#[tokio::test]
async fn grafana_org() {
    wait_for_healthy(GRAFANA_HEALTH, TIMEOUT).await;
    let c = grafana();

    let org = c.get_current_org().await.expect("get_current_org failed");
    assert!(org.id > 0);
    assert!(!org.name.is_empty());

    // Update org name, then restore
    let original_name = org.name.clone();
    let new_name = unique_name("org");
    c.update_org(&serde_json::json!({"name": new_name}))
        .await
        .expect("update_org failed");

    let updated = c.get_current_org().await.expect("get org after update failed");
    assert_eq!(updated.name, new_name);

    // Restore
    c.update_org(&serde_json::json!({"name": original_name}))
        .await
        .expect("restore org name failed");
}

// ---------------------------------------------------------------------------
// 2. Folder CRUD
// ---------------------------------------------------------------------------

#[tokio::test]
async fn grafana_folder_crud() {
    wait_for_healthy(GRAFANA_HEALTH, TIMEOUT).await;
    let c = grafana();

    let title = unique_name("folder");
    let uid = unique_name("fuid");

    // Create
    let folder = c
        .create_folder(&serde_json::json!({"title": title, "uid": uid}))
        .await
        .expect("create_folder failed");
    assert_eq!(folder.title, title);
    assert_eq!(folder.uid, uid);

    // List — should contain our folder
    let folders = c.list_folders().await.expect("list_folders failed");
    assert!(folders.iter().any(|f| f.uid == uid), "folder not found in list");

    // Get
    let fetched = c.get_folder(&uid).await.expect("get_folder failed");
    assert_eq!(fetched.title, title);

    // Update
    let new_title = unique_name("folder-upd");
    let updated = c
        .update_folder(&uid, &serde_json::json!({"title": new_title, "overwrite": true}))
        .await
        .expect("update_folder failed");
    assert_eq!(updated.title, new_title);

    // Delete
    c.delete_folder(&uid).await.expect("delete_folder failed");

    // Verify deleted
    let result = c.get_folder(&uid).await;
    assert!(result.is_err(), "folder should not exist after deletion");
}

// ---------------------------------------------------------------------------
// 3. Dashboard CRUD
// ---------------------------------------------------------------------------

#[tokio::test]
async fn grafana_dashboard_crud() {
    wait_for_healthy(GRAFANA_HEALTH, TIMEOUT).await;
    let c = grafana();

    let title = unique_name("dash");
    let dash_uid = unique_name("duid");

    // Create
    let body = serde_json::json!({
        "dashboard": {
            "uid": dash_uid,
            "title": title,
            "panels": [],
            "schemaVersion": 27,
            "version": 0
        },
        "overwrite": false
    });
    let created = c.create_dashboard(&body).await.expect("create_dashboard failed");
    assert_eq!(created.uid.as_deref(), Some(dash_uid.as_str()));

    // Get
    let fetched = c.get_dashboard(&dash_uid).await.expect("get_dashboard failed");
    assert!(fetched.dashboard.is_some());

    // Update
    let updated_title = unique_name("dash-upd");
    let update_body = serde_json::json!({
        "dashboard": {
            "uid": dash_uid,
            "title": updated_title,
            "panels": [],
            "schemaVersion": 27,
            "version": 1
        },
        "overwrite": true
    });
    let updated = c.update_dashboard(&update_body).await.expect("update_dashboard failed");
    assert_eq!(updated.uid.as_deref(), Some(dash_uid.as_str()));

    // List
    let all = c.list_dashboards().await.expect("list_dashboards failed");
    assert!(all.iter().any(|d| d.uid == dash_uid), "dashboard not found in list");

    // Search
    let found = c.search_dashboards(&updated_title).await.expect("search_dashboards failed");
    assert!(!found.is_empty(), "search should find the dashboard");
    assert!(found.iter().any(|d| d.uid == dash_uid));

    // Delete
    c.delete_dashboard(&dash_uid).await.expect("delete_dashboard failed");

    // Verify deleted
    let result = c.get_dashboard(&dash_uid).await;
    assert!(result.is_err(), "dashboard should not exist after deletion");
}

// ---------------------------------------------------------------------------
// 4. Datasource CRUD
// ---------------------------------------------------------------------------

#[tokio::test]
async fn grafana_datasource_crud() {
    wait_for_healthy(GRAFANA_HEALTH, TIMEOUT).await;
    let c = grafana();

    let name = unique_name("ds-prom");
    let ds_uid = unique_name("dsuid");

    // Create
    let body = serde_json::json!({
        "name": name,
        "type": "prometheus",
        "uid": ds_uid,
        "url": "http://prometheus:9090",
        "access": "proxy",
        "isDefault": false
    });
    let created = c.create_datasource(&body).await.expect("create_datasource failed");
    let ds_id = created.id.expect("datasource missing id");
    assert_eq!(created.name, name);

    // List
    let all = c.list_datasources().await.expect("list_datasources failed");
    assert!(all.iter().any(|d| d.name == name), "datasource not found in list");

    // Get by ID
    let by_id = c.get_datasource(ds_id).await.expect("get_datasource by id failed");
    assert_eq!(by_id.name, name);

    // Get by UID
    let by_uid = c
        .get_datasource_by_uid(&ds_uid)
        .await
        .expect("get_datasource_by_uid failed");
    assert_eq!(by_uid.name, name);

    // Update
    let new_name = unique_name("ds-upd");
    let update_body = serde_json::json!({
        "name": new_name,
        "type": "prometheus",
        "uid": ds_uid,
        "url": "http://prometheus:9090",
        "access": "proxy",
        "isDefault": false
    });
    let updated = c
        .update_datasource(ds_id, &update_body)
        .await
        .expect("update_datasource failed");
    assert_eq!(updated.name, new_name);

    // Delete
    c.delete_datasource(ds_id).await.expect("delete_datasource failed");

    // Verify deleted
    let result = c.get_datasource(ds_id).await;
    assert!(result.is_err(), "datasource should not exist after deletion");
}

// ---------------------------------------------------------------------------
// 5. Annotation CRUD
// ---------------------------------------------------------------------------

#[tokio::test]
async fn grafana_annotation_crud() {
    wait_for_healthy(GRAFANA_HEALTH, TIMEOUT).await;
    let c = grafana();

    let text = unique_name("annotation");
    let now_ms = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .unwrap()
        .as_millis() as u64;

    // Create
    let body = serde_json::json!({
        "text": text,
        "time": now_ms,
        "tags": ["integration-test"]
    });
    let created = c.create_annotation(&body).await.expect("create_annotation failed");
    let ann_id = created.id.expect("annotation missing id");
    assert!(ann_id > 0);

    // List
    let all = c.list_annotations(Some("tags=integration-test")).await.expect("list_annotations failed");
    assert!(all.iter().any(|a| a.id == Some(ann_id)), "annotation not found in list");

    // Get
    let fetched = c.get_annotation(ann_id).await.expect("get_annotation failed");
    assert_eq!(fetched.text.as_deref(), Some(text.as_str()));

    // Update
    let new_text = unique_name("ann-upd");
    let update_body = serde_json::json!({
        "text": new_text,
        "time": now_ms,
        "tags": ["integration-test", "updated"]
    });
    c.update_annotation(ann_id, &update_body)
        .await
        .expect("update_annotation failed");

    let updated = c.get_annotation(ann_id).await.expect("get annotation after update failed");
    assert_eq!(updated.text.as_deref(), Some(new_text.as_str()));

    // Delete
    c.delete_annotation(ann_id).await.expect("delete_annotation failed");
}

// ---------------------------------------------------------------------------
// 6. Alert Rules
// ---------------------------------------------------------------------------

#[tokio::test]
async fn grafana_alert_rules() {
    wait_for_healthy(GRAFANA_HEALTH, TIMEOUT).await;
    let c = grafana();

    // Create prerequisite folder and datasource
    let folder_uid = unique_name("alert-fld");
    let folder_title = unique_name("AlertFolder");
    c.create_folder(&serde_json::json!({"title": folder_title, "uid": folder_uid}))
        .await
        .expect("create folder for alerts failed");

    let ds_name = unique_name("alert-ds");
    let ds_uid = unique_name("alertdsuid");
    let ds = c
        .create_datasource(&serde_json::json!({
            "name": ds_name,
            "type": "prometheus",
            "uid": ds_uid,
            "url": "http://prometheus:9090",
            "access": "proxy"
        }))
        .await
        .expect("create datasource for alerts failed");
    let ds_id = ds.id.expect("datasource missing id");

    let rule_title = unique_name("test-rule");
    let rule_group = unique_name("test-group");

    // Create alert rule
    let rule_body = serde_json::json!({
        "title": rule_title,
        "ruleGroup": rule_group,
        "folderUID": folder_uid,
        "condition": "A",
        "for": "5m",
        "data": [{
            "refId": "A",
            "datasourceUid": ds_uid,
            "model": {
                "expr": "up == 0",
                "refId": "A"
            },
            "relativeTimeRange": {
                "from": 600,
                "to": 0
            }
        }],
        "noDataState": "NoData",
        "execErrState": "Error"
    });
    let created = c
        .create_alert_rule(&rule_body)
        .await
        .expect("create_alert_rule failed");
    let rule_uid = created.uid.clone().expect("alert rule missing uid");
    assert_eq!(created.title.as_deref(), Some(rule_title.as_str()));

    // List
    let rules = c.get_alert_rules().await.expect("get_alert_rules failed");
    assert!(
        rules.iter().any(|r| r.uid.as_deref() == Some(&rule_uid)),
        "alert rule not found in list"
    );

    // Update
    let updated_title = unique_name("rule-upd");
    let update_body = serde_json::json!({
        "title": updated_title,
        "ruleGroup": rule_group,
        "folderUID": folder_uid,
        "condition": "A",
        "for": "10m",
        "data": [{
            "refId": "A",
            "datasourceUid": ds_uid,
            "model": {
                "expr": "up == 0",
                "refId": "A"
            },
            "relativeTimeRange": {
                "from": 600,
                "to": 0
            }
        }],
        "noDataState": "NoData",
        "execErrState": "Error"
    });
    let updated = c
        .update_alert_rule(&rule_uid, &update_body)
        .await
        .expect("update_alert_rule failed");
    assert_eq!(updated.title.as_deref(), Some(updated_title.as_str()));

    // Delete alert rule
    c.delete_alert_rule(&rule_uid)
        .await
        .expect("delete_alert_rule failed");

    // Cleanup: delete datasource and folder
    c.delete_datasource(ds_id)
        .await
        .expect("cleanup datasource failed");
    c.delete_folder(&folder_uid)
        .await
        .expect("cleanup folder failed");
}

// ---------------------------------------------------------------------------
// 7. Proxy
// ---------------------------------------------------------------------------

#[tokio::test]
async fn grafana_proxy_datasource() {
    wait_for_healthy(GRAFANA_HEALTH, TIMEOUT).await;
    let c = grafana();

    // Create a datasource to proxy through
    let ds_name = unique_name("proxy-ds");
    let ds_uid = unique_name("proxydsuid");
    let ds = c
        .create_datasource(&serde_json::json!({
            "name": ds_name,
            "type": "prometheus",
            "uid": ds_uid,
            "url": "http://prometheus:9090",
            "access": "proxy"
        }))
        .await
        .expect("create datasource for proxy failed");
    let ds_id = ds.id.expect("datasource missing id");

    // Proxy a Prometheus query through Grafana
    let result = c
        .proxy_datasource(ds_id, "api/v1/query?query=up")
        .await
        .expect("proxy_datasource failed — ensure Grafana can reach Prometheus at http://prometheus:9090");
    assert_eq!(result["status"], "success");

    // Cleanup
    c.delete_datasource(ds_id)
        .await
        .expect("cleanup proxy datasource failed");
}

// ===========================================================================
// LOKI — additional coverage
// ===========================================================================

// ---------------------------------------------------------------------------
// Loki instant query (exercises the query() method)
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_instant_query() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    // Instant query with a metric-style expression (count_over_time)
    let query_str = r#"count_over_time({job="test"}[5m])"#;
    for i in 0..10 {
        match c.query(query_str, Some(10), None).await {
            Ok(res) => {
                assert_eq!(res.status, "success");
                return;
            }
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("instant query failed after retries: {e}"),
        }
    }
}

// ---------------------------------------------------------------------------
// Loki instant query with explicit time
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_instant_query_with_time() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let now = chrono::Utc::now().timestamp().to_string();
    let query_str = r#"count_over_time({job="test"}[5m])"#;
    for i in 0..10 {
        match c.query(query_str, None, Some(&now)).await {
            Ok(res) => {
                assert_eq!(res.status, "success");
                return;
            }
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("instant query with time failed after retries: {e}"),
        }
    }
}

// ---------------------------------------------------------------------------
// Loki query_range with step parameter
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_query_range_with_step() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let now = chrono::Utc::now();
    let start = (now - chrono::Duration::minutes(10)).timestamp().to_string();
    let end = now.timestamp().to_string();

    // Use a metric expression so step makes sense
    let query_str = r#"count_over_time({job="test"}[1m])"#;
    for i in 0..10 {
        match c.query_range(query_str, &start, &end, Some(100), Some("60s")).await {
            Ok(res) => {
                assert_eq!(res.status, "success");
                return;
            }
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("query_range with step failed after retries: {e}"),
        }
    }
}

// ---------------------------------------------------------------------------
// Loki labels with start/end parameters
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_labels_with_time_range() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let now = chrono::Utc::now();
    let start = (now - chrono::Duration::minutes(10)).timestamp().to_string();
    let end = now.timestamp().to_string();

    for i in 0..10 {
        match c.labels(Some(&start), Some(&end)).await {
            Ok(res) => {
                assert_eq!(res.status, "success");
                return;
            }
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("labels with time range failed after retries: {e}"),
        }
    }
}

// ---------------------------------------------------------------------------
// Loki label_values with start/end parameters
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_label_values_with_time_range() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let now = chrono::Utc::now();
    let start = (now - chrono::Duration::minutes(10)).timestamp().to_string();
    let end = now.timestamp().to_string();

    for i in 0..10 {
        match c.label_values("job", Some(&start), Some(&end)).await {
            Ok(res) => {
                assert_eq!(res.status, "success");
                return;
            }
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("label_values with time range failed after retries: {e}"),
        }
    }
}

// ---------------------------------------------------------------------------
// Loki push error path (malformed body)
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_push_malformed() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    // Send a completely malformed push body
    let bad_body = serde_json::json!({
        "streams": [{
            "stream": {},
            "values": "not-an-array"
        }]
    });

    let result = c.push(&bad_body).await;
    assert!(result.is_err(), "push with malformed body should fail");
}

// ---------------------------------------------------------------------------
// Loki series without time bounds
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_series_no_time_bounds() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    for i in 0..10 {
        match c.series(&[r#"{job="test"}"#], None, None).await {
            Ok(res) => {
                assert_eq!(res.status, "success");
                return;
            }
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("series without time bounds failed after retries: {e}"),
        }
    }
}

// ---------------------------------------------------------------------------
// Loki series with multiple matchers
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_series_multiple_matchers() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let now = chrono::Utc::now();
    let start = (now - chrono::Duration::minutes(10)).timestamp().to_string();
    let end = now.timestamp().to_string();

    for i in 0..10 {
        match c
            .series(
                &[r#"{job="test"}"#, r#"{app="integration"}"#],
                Some(&start),
                Some(&end),
            )
            .await
        {
            Ok(res) => {
                assert_eq!(res.status, "success");
                return;
            }
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("series with multiple matchers failed after retries: {e}"),
        }
    }
}

// ---------------------------------------------------------------------------
// Loki index_volume_range with step
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_index_volume_range_with_step() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let now = chrono::Utc::now();
    let start = (now - chrono::Duration::minutes(10)).timestamp().to_string();
    let end = now.timestamp().to_string();

    for i in 0..10 {
        match c
            .index_volume_range(r#"{job="test"}"#, &start, &end, Some("60s"))
            .await
        {
            Ok(_val) => return,
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => {
                eprintln!("index_volume_range with step not available (may be expected): {e}");
                return;
            }
        }
    }
}

// ---------------------------------------------------------------------------
// Loki labels with only start (no end)
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_labels_start_only() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let now = chrono::Utc::now();
    let start = (now - chrono::Duration::minutes(10)).timestamp().to_string();

    for i in 0..10 {
        match c.labels(Some(&start), None).await {
            Ok(res) => {
                assert_eq!(res.status, "success");
                return;
            }
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("labels with start only failed after retries: {e}"),
        }
    }
}

// ---------------------------------------------------------------------------
// Loki label_values with only end (no start) — exercises sep logic
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_label_values_end_only() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let now = chrono::Utc::now();
    let end = now.timestamp().to_string();

    for i in 0..10 {
        match c.label_values("job", None, Some(&end)).await {
            Ok(res) => {
                assert_eq!(res.status, "success");
                return;
            }
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("label_values with end only failed after retries: {e}"),
        }
    }
}

// ---------------------------------------------------------------------------
// Loki detect_patterns with no time bounds
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_detect_patterns_no_time() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    for i in 0..10 {
        match c.detect_patterns(r#"{job="test"}"#, None, None).await {
            Ok(_val) => return,
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => {
                eprintln!("detect_patterns without time not available (may be expected): {e}");
                return;
            }
        }
    }
}

// ---------------------------------------------------------------------------
// Loki index_volume with no time bounds
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_index_volume_no_time() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    for i in 0..10 {
        match c.index_volume(r#"{job="test"}"#, None, None).await {
            Ok(_val) => return,
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => {
                eprintln!("index_volume without time not available (may be expected): {e}");
                return;
            }
        }
    }
}

// ---------------------------------------------------------------------------
// Loki query with default limit (None)
// ---------------------------------------------------------------------------

#[tokio::test]
async fn loki_instant_query_default_limit() {
    wait_for_healthy(LOKI_HEALTH, TIMEOUT).await;
    let c = loki();

    let query_str = r#"count_over_time({job="test"}[5m])"#;
    for i in 0..10 {
        match c.query(query_str, None, None).await {
            Ok(res) => {
                assert_eq!(res.status, "success");
                return;
            }
            Err(_) if i < 9 => tokio::time::sleep(std::time::Duration::from_secs(2)).await,
            Err(e) => panic!("instant query with default limit failed after retries: {e}"),
        }
    }
}