AkurAI-Mail / scripts/ops/antispam-metrics.sh

Latest change e0753e49bfea7ef672aabf8df9a3a27b8bdd9dc7 - Recover EC2-only ops scripts, checks and live divergent copies by Ólafur Búi Ólafsson

#!/usr/bin/env bash
# scripts/ops/antispam-metrics.sh — daily metrics + alerting for the
# anti-spam sweep (akurai-mail-antispam.service/.timer, runs every 10 min).
#
# Does NOT re-run the sweep. Reads its journal output (one JSON line per run,
# already emitted by `akurai-mail-server anti-spam-scan`), appends a durable
# JSONL series, and fires three independent alerts. Run by
# antispam-metrics.timer every 5 min.
set -euo pipefail

: "${ANTISPAM_UNIT:=akurai-mail-antispam.service}"
: "${ANTISPAM_STATE_DIR:=/var/lib/akurai-mail-ops/antispam-metrics}"
: "${ANTISPAM_ALERT_TO:=postmaster@olibuijr.com}"
: "${ANTISPAM_WINDOW_SEC:=3600}"        # trailing window for silent-failure check
: "${ANTISPAM_BASELINE_N:=20}"          # records used for move-ratio baseline
: "${ANTISPAM_SPIKE_FACTOR:=3}"         # alert if ratio > baseline * this
: "${ANTISPAM_MIN_MOVED_FOR_SPIKE:=5}"  # ignore spike alert on tiny counts
: "${JOURNAL_CMD:=journalctl -a -u $ANTISPAM_UNIT -o json --no-pager}"
: "${DRY_RUN:=0}"

DAILY_LOG="$ANTISPAM_STATE_DIR/daily.jsonl"
LAST_TS_FILE="$ANTISPAM_STATE_DIR/last_ts"
ALERT_STATE="$ANTISPAM_STATE_DIR/alerts.state"
mkdir -p "$ANTISPAM_STATE_DIR"
touch "$DAILY_LOG" "$ALERT_STATE"

now=$(date +%s)
last_ts=$(cat "$LAST_TS_FILE" 2>/dev/null || echo $((now - ANTISPAM_WINDOW_SEC)))

send_mail() {
    local subject="$1" body="$2"
    if [ "$DRY_RUN" = "1" ]; then
        echo "[DRY_RUN] Subject: $subject"; echo "$body"; return 0
    fi
    { echo "Subject: $subject"; echo "To: $ANTISPAM_ALERT_TO"; echo; echo "$body"; } | /usr/sbin/sendmail -t
}

alert_once() {
    # dedupe one alert kind per calendar day
    local kind="$1" subject="$2" body="$3"
    local today key last
    today=$(date +%F)
    key="${today}:${kind}"
    last=$(grep "^${kind}=" "$ALERT_STATE" 2>/dev/null | tail -1 | cut -d= -f2 || true)
    if [ "$last" != "$key" ]; then
        send_mail "$subject" "$body"
        # replace/append the state line for this kind
        grep -v "^${kind}=" "$ALERT_STATE" > "$ALERT_STATE.tmp" 2>/dev/null || true
        echo "${kind}=${key}" >> "$ALERT_STATE.tmp"
        mv "$ALERT_STATE.tmp" "$ALERT_STATE"
    fi
}

# --- 1. pull new journal entries since last_ts, split scan results vs errors ---
entries_json=$(eval "$JOURNAL_CMD --since '@$last_ts'" 2>/dev/null) && journal_ok=1 || journal_ok=0

new_records=0
exec_failures=$(printf '%s\n' "$entries_json" | jq -r 'select(.MESSAGE and (.MESSAGE | test("203/EXEC|Failed with result"))) | .MESSAGE' | wc -l)

while IFS= read -r line; do
    [ -z "$line" ] && continue
    msg=$(printf '%s' "$line" | jq -r '.MESSAGE // empty')
    ts=$(printf '%s' "$line" | jq -r '.__REALTIME_TIMESTAMP // empty')
    [ -z "$msg" ] && continue
    scanned=$(printf '%s' "$msg" | jq -r '.scanned // empty' 2>/dev/null || true)
    [ -z "$scanned" ] && continue
    moved=$(printf '%s' "$msg" | jq -r '.moved // 0')
    ok=$(printf '%s' "$msg" | jq -r '.ok // false')
    rec_ts=$(( ts / 1000000 ))
    ratio=$(awk -v s="$scanned" -v m="$moved" 'BEGIN { print (s>0)? m/s : 0 }')
    # per-mailbox Junk counts: cumulative totals from antiSpam.mailboxes[],
    # dropped down to non-zero entries only (most mailboxes have 0 spam).
    mailboxes=$(printf '%s' "$msg" | jq -c '[(.antiSpam.mailboxes // [])[] | select(.spam > 0)]')
    jq -nc --argjson ts "$rec_ts" --argjson scanned "$scanned" --argjson moved "$moved" \
        --argjson ratio "$ratio" --argjson ok "$ok" --argjson mailboxes "$mailboxes" \
        '{ts:$ts, scanned:$scanned, moved:$moved, ratio:$ratio, ok:$ok, mailboxes:$mailboxes}' >> "$DAILY_LOG"
    new_records=$((new_records + 1))
done < <(printf '%s\n' "$entries_json" | jq -c 'select(.MESSAGE and (.MESSAGE | test("\"ok\": true")))')

if [ "$journal_ok" = "1" ]; then
    echo "$now" > "$LAST_TS_FILE"
fi

# --- 2. sweep-error alert ---
if [ "$exec_failures" -gt 0 ]; then
    alert_once sweep-errors \
        "[mail VM] anti-spam sweep failed ${exec_failures}x since last check" \
        "Unit: $ANTISPAM_UNIT
Failures observed: $exec_failures
Window: since $(date -d @"$last_ts") to $(date -d @"$now")

Run: journalctl -u $ANTISPAM_UNIT --since '@${last_ts}' | grep -E '203/EXEC|Failed with result'"
fi

# --- 3. silent-failure alert: zero scans across the trailing window ---
window_start=$((now - ANTISPAM_WINDOW_SEC))
total_scanned=$(awk -v w="$window_start" -F'"scanned":' 'BEGIN{sum=0} {print}' "$DAILY_LOG" | \
    jq -s --argjson w "$window_start" '[.[] | select(.ts >= $w)] | map(.scanned) | add // 0')
record_count=$(jq -s --argjson w "$window_start" '[.[] | select(.ts >= $w)] | length' "$DAILY_LOG")
if [ "$record_count" -gt 0 ] && [ "$total_scanned" -eq 0 ]; then
    alert_once zero-scans \
        "[mail VM] anti-spam sweep scanned 0 messages in the last $((ANTISPAM_WINDOW_SEC / 60))m" \
        "Records in window: $record_count, total scanned: 0.
This usually means dovecot/maildir access is broken even though the sweep
unit exits 0 — check: journalctl -u $ANTISPAM_UNIT --since '-${ANTISPAM_WINDOW_SEC}s'"
fi

# --- 4. move-ratio spike vs trailing baseline ---
latest=$(tail -1 "$DAILY_LOG" 2>/dev/null || echo "")
if [ -n "$latest" ]; then
    cur_ratio=$(printf '%s' "$latest" | jq -r '.ratio')
    cur_moved=$(printf '%s' "$latest" | jq -r '.moved')
    baseline=$(tail -n $((ANTISPAM_BASELINE_N + 1)) "$DAILY_LOG" | head -n -1 | \
        jq -s 'if length>0 then (map(.ratio) | add / length) else 0 end')
    threshold=$(awk -v b="$baseline" -v f="$ANTISPAM_SPIKE_FACTOR" 'BEGIN { print b*f }')
    is_spike=$(awk -v c="$cur_ratio" -v t="$threshold" -v m="$cur_moved" -v minm="$ANTISPAM_MIN_MOVED_FOR_SPIKE" \
        'BEGIN { print (m>=minm && t>0 && c>t) ? 1 : 0 }')
    if [ "$is_spike" = "1" ]; then
        alert_once move-ratio-spike \
            "[mail VM] anti-spam move-ratio spike: ${cur_ratio} vs baseline ${baseline}" \
            "Latest run: moved=$cur_moved ratio=$cur_ratio
Trailing baseline (last $ANTISPAM_BASELINE_N runs): $baseline
Spike threshold (baseline x ${ANTISPAM_SPIKE_FACTOR}): $threshold"
    fi
fi

echo "antispam-metrics: appended $new_records record(s), $exec_failures failure(s) in journal window"