AkurAI-Monitor / tests/alert_e2e.rs

Latest change 6cc95bf768ad770440c1359ef6ba6878e92edd74 - Test alert lifecycle against a throwaway Monitor, not the live one by Ólafur Búi Ólafsson

//! End-to-end alert lifecycle against a throwaway Monitor.
//!
//! Synthetic metrics must never reach the live Monitor (every open alert
//! there is a CRIT on the desktop HUD), so this starts the built binary on a
//! free loopback port against its own scratch database on the
//! `MONITOR_TEST_DB_URL` server, with a cleared environment, and drops both
//! afterwards, even on failure.
//!
//! It waits on the real 60s evaluation tick, so it is ignored by default:
//! `MONITOR_TEST_DB_URL=... cargo test --test alert_e2e -- --ignored`

use mysql::prelude::Queryable;
use mysql::{Opts, OptsBuilder, Pool, PooledConn};
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};

const HOST_NAME: &str = "e2e-synthetic-host";
const METRIC: &str = "e2e.synthetic.value";

struct Scratch {
    admin: Pool,
    name: String,
    url: String,
    child: Option<Child>,
    alert_log: std::path::PathBuf,
}

impl Scratch {
    fn new(server_url: &str) -> Scratch {
        let opts = Opts::from_url(server_url).expect("invalid MONITOR_TEST_DB_URL");
        let admin = Pool::new(OptsBuilder::from_opts(opts)).expect("connect test MariaDB");
        let name = format!("akurai_test_e2e_{}", std::process::id());
        admin
            .get_conn()
            .unwrap()
            .query_drop(format!("CREATE DATABASE `{name}`"))
            .expect("create scratch database");
        let base = server_url.rsplit_once('/').expect("db url has a path").0;
        let alert_log =
            std::path::Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("{name}-alerts.log"));
        Scratch {
            admin,
            url: format!("{base}/{name}"),
            name,
            child: None,
            alert_log,
        }
    }

    fn conn(&self) -> PooledConn {
        let opts = Opts::from_url(&self.url).unwrap();
        Pool::new(OptsBuilder::from_opts(opts))
            .unwrap()
            .get_conn()
            .unwrap()
    }

    fn start_monitor(&mut self) {
        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let listen = listener.local_addr().unwrap().to_string();
        drop(listener);
        let child = Command::new(env!("CARGO_BIN_EXE_rust-monitor"))
            .env_clear()
            .env("PATH", std::env::var("PATH").unwrap_or_default())
            .env("MONITOR_DB_URL", &self.url)
            .env("MONITOR_LISTEN", listen)
            .env("MONITOR_HOST_NAME", HOST_NAME)
            .env(
                "MONITOR_STATIC_DIR",
                concat!(env!("CARGO_MANIFEST_DIR"), "/ui"),
            )
            .env("MONITOR_ALERT_LOG", &self.alert_log)
            .env("MONITOR_LOG_FILES", "")
            .env("MONITOR_JOURNAL_MATCH", "e2e-no-such-unit")
            .env("MONITOR_ALERT_STALE_SECS", "1")
            .env("RUST_LOG", "rust_monitor=warn")
            .stdout(Stdio::null())
            .stderr(Stdio::inherit())
            .spawn()
            .expect("start rust-monitor");
        self.child = Some(child);
    }
}

impl Drop for Scratch {
    fn drop(&mut self) {
        if let Some(child) = self.child.as_mut() {
            child.kill().ok();
            child.wait().ok();
        }
        if let Ok(mut conn) = self.admin.get_conn() {
            conn.query_drop(format!("DROP DATABASE IF EXISTS `{}`", self.name))
                .ok();
        }
        std::fs::remove_file(&self.alert_log).ok();
    }
}

fn now() -> i64 {
    chrono::Utc::now().timestamp()
}

fn wait_for<T>(what: &str, timeout: Duration, mut probe: impl FnMut() -> Option<T>) -> T {
    let deadline = Instant::now() + timeout;
    loop {
        if let Some(value) = probe() {
            return value;
        }
        assert!(Instant::now() < deadline, "timed out waiting for {what}");
        std::thread::sleep(Duration::from_secs(2));
    }
}

#[test]
#[ignore = "waits on the 60s evaluation tick; run with --ignored"]
fn synthetic_alert_triggers_then_resolves_once_its_metric_goes_silent() {
    let Ok(server_url) = std::env::var("MONITOR_TEST_DB_URL") else {
        eprintln!("skipping: MONITOR_TEST_DB_URL is not set");
        return;
    };
    let mut scratch = Scratch::new(&server_url);
    scratch.start_monitor();
    let mut conn = scratch.conn();

    let host_id: i64 = wait_for("monitor startup", Duration::from_secs(30), || {
        conn.exec_first("SELECT id FROM hosts WHERE name = ?", (HOST_NAME,))
            .ok()
            .flatten()
    });
    conn.exec_drop(
        "INSERT INTO alert_rules (host_id, name, metric_name, operator, threshold, duration_secs)
         VALUES (?, 'E2E Synthetic', ?, 'gt', 1, 60)",
        (host_id, METRIC),
    )
    .unwrap();
    let rule_id = conn.last_insert_id() as i64;

    let event_id: i64 = wait_for("alert to trigger", Duration::from_secs(150), || {
        conn.exec_drop(
            "INSERT INTO metrics (host_id, name, value, ts) VALUES (?, ?, 5, ?)",
            (host_id, METRIC, now()),
        )
        .unwrap();
        conn.exec_first(
            "SELECT id FROM alert_events WHERE rule_id = ? AND resolved_at IS NULL",
            (rule_id,),
        )
        .unwrap()
    });

    // Stop reporting: the rule window (60s) bounds the stale window, so the
    // alert must resolve within about two evaluation ticks.
    let resolved_at: i64 = wait_for("stale alert to resolve", Duration::from_secs(200), || {
        conn.exec_first::<Option<i64>, _, _>(
            "SELECT resolved_at FROM alert_events WHERE id = ?",
            (event_id,),
        )
        .unwrap()
        .flatten()
    });
    assert!(resolved_at > 0);
    let log = std::fs::read_to_string(&scratch.alert_log).unwrap_or_default();
    assert!(
        log.contains(&format!("metric=\"{METRIC}\" reason=stale")),
        "alert log records the stale resolution:\n{log}"
    );
}