AkurAI-Monitor / scripts/titan-monitor-agent

Latest change 8bade644cef8ddaa358cc88eb6017c8e64d5a6e4 - AKURAI-MONITOR-7 monitor processes, docker, systemd; fix #errors alert delivery by Ólafur Búi Ólafsson

#!/usr/bin/env bash
# Collection agent for the Titan workstation host.
#
# Samples the same metric namespace rust-monitor's own in-process collector
# uses (src/collectors.rs) and pushes each point to the central monitor's
# management API, attributed to this host's registered host_id. Run as a
# systemd oneshot on a timer (see titan-monitor-agent.service/.timer);
# each invocation is a single sample-and-push cycle.
#
# Config (env, normally supplied via /etc/titan-monitor-agent.env):
#   MONITOR_BASE_URL   default https://monitor.olibuijr.com
#   MONITOR_API_TOKEN  required — management bearer token (PassVault entry 25)
#   MONITOR_HOST_ID    required — this host's id in the `hosts` table
set -euo pipefail
# shellcheck source=scripts/metric-contract.sh
source "$(dirname "${BASH_SOURCE[0]}")/metric-contract.sh"

BASE_URL="${MONITOR_BASE_URL:-https://monitor.olibuijr.com}"
TOKEN="${MONITOR_API_TOKEN:?MONITOR_API_TOKEN is required}"
HOST_ID="${MONITOR_HOST_ID:?MONITOR_HOST_ID is required}"

push() {
  local name="$1" value="$2"
  curl -sf -m 5 -X POST "$BASE_URL/api/manage/metrics" \
    -H "Authorization: Bearer $TOKEN" \
    -H "Content-Type: application/json" \
    -d "{\"host_id\":${HOST_ID},\"name\":\"${name}\",\"value\":${value}}" \
    >/dev/null || echo "titan-monitor-agent: failed to push ${name}" >&2
}

# --- cpu.usage: percent busy over a 1s /proc/stat sample window ---
read_cpu_ticks() {
  local line idle total field
  read -r line < /proc/stat
  # line: cpu user nice system idle iowait irq softirq steal guest guest_nice
  local -a f=($line)
  idle="${f[4]}"
  total=0
  for field in "${f[@]:1}"; do total=$((total + field)); done
  echo "$idle $total"
}
read -r idle1 total1 <<<"$(read_cpu_ticks)"
sleep 1
read -r idle2 total2 <<<"$(read_cpu_ticks)"
idle_delta=$((idle2 - idle1))
total_delta=$((total2 - total1))
if [ "$total_delta" -gt 0 ]; then
  cpu_usage=$(awk -v i="$idle_delta" -v t="$total_delta" 'BEGIN { printf "%.1f", (1 - i / t) * 100 }')
  push "$METRIC_CPU_USAGE" "$cpu_usage"
fi

# --- memory ---
mem_total_kb=$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)
mem_available_kb=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)
mem_used_kb=$((mem_total_kb - mem_available_kb))
mem_used_pct=$(awk -v u="$mem_used_kb" -v t="$mem_total_kb" 'BEGIN { printf "%.1f", (t > 0) ? (u / t * 100) : 0 }')
push "$METRIC_MEM_TOTAL" "$mem_total_kb"
push "$METRIC_MEM_USED" "$mem_used_kb"
push "$METRIC_MEM_AVAILABLE" "$mem_available_kb"
push "$METRIC_MEM_USED_PCT" "$mem_used_pct"

# --- disk (root filesystem) ---
read -r _ disk_total_kb disk_used_kb _ disk_used_pct_raw _ < <(df -kP / | tail -n1)
disk_used_pct="${disk_used_pct_raw%\%}"
disk_total_gb=$(awk -v k="$disk_total_kb" 'BEGIN { printf "%.1f", k / 1048576 }')
disk_used_gb=$(awk -v k="$disk_used_kb" 'BEGIN { printf "%.1f", k / 1048576 }')
push "$METRIC_DISK_ROOT_TOTAL" "$disk_total_gb"
push "$METRIC_DISK_ROOT_USED" "$disk_used_gb"
push "$METRIC_DISK_ROOT_USED_PCT" "$disk_used_pct"

# --- load average ---
read -r load1 load5 load15 _ < /proc/loadavg
push "$METRIC_LOAD_1M" "$load1"
push "$METRIC_LOAD_5M" "$load5"
push "$METRIC_LOAD_15M" "$load15"

# --- uptime ---
read -r uptime_seconds _ < /proc/uptime
push "$METRIC_UPTIME" "${uptime_seconds%.*}"

# --- network, per physical-looking interface (delta over a 1s window) ---
read_net() {
  while IFS=: read -r iface stats; do
    iface="${iface// /}"
    [ "$iface" = "lo" ] && continue
    [[ "$iface" == veth* || "$iface" == docker* || "$iface" == br-* ]] && continue
    read -r rx _ _ _ _ _ _ _ tx _ <<<"$stats"
    printf '%s %s %s\n' "$iface" "$rx" "$tx"
  done < <(tail -n +3 /proc/net/dev)
}
declare -A rx1 tx1 rx2 tx2
while read -r iface rx tx; do
  rx1["$iface"]="$rx"
  tx1["$iface"]="$tx"
done < <(read_net)
sleep 1
while read -r iface rx tx; do
  rx2["$iface"]="$rx"
  tx2["$iface"]="$tx"
done < <(read_net)
for iface in "${!rx2[@]}"; do
  [ -n "${rx1[$iface]:-}" ] || continue
  rx_delta=$((rx2[$iface] - rx1[$iface]))
  tx_delta=$((tx2[$iface] - tx1[$iface]))
  [ "$rx_delta" -lt 0 ] && rx_delta=0
  [ "$tx_delta" -lt 0 ] && tx_delta=0
  push "net.${iface}.${METRIC_NET_RX_SUFFIX}" "$rx_delta"
  push "net.${iface}.${METRIC_NET_TX_SUFFIX}" "$tx_delta"
done

# --- per-process memory: aggregate RSS by command name (same aggregation
# and top-N-by-RSS ruling as src/collectors.rs::collect_processes), so the
# metric namespace matches regardless of which host reported it ---
sanitize_segment() {
  local cleaned
  cleaned=$(printf '%s' "$1" | tr -c 'A-Za-z0-9_-' '_' | sed 's/^_*//;s/_*$//')
  printf '%s' "${cleaned:-unknown}"
}

PROC_TOP_N="${MONITOR_PROC_TOP_N:-15}"
if command -v ps >/dev/null 2>&1; then
  while IFS=$'\t' read -r comm rss_sum proc_count; do
    [ -n "$comm" ] || continue
    name=$(sanitize_segment "$comm")
    push "proc.${name}.mem_kb" "$rss_sum"
    push "proc.${name}.count" "$proc_count"
  done < <(ps -eo comm=,rss= 2>/dev/null | awk '
    {
      rss = $NF
      $NF = ""
      sub(/ +$/, "")
      comm = $0
      gsub(/^ +/, "", comm)
      sums[comm] += rss
      counts[comm]++
    }
    END { for (c in sums) printf "%s\t%d\t%d\n", c, sums[c], counts[c] }
  ' | sort -t$'\t' -k2,2 -rn | head -n "$PROC_TOP_N")
fi

# --- systemd services: allowlisted units (memory/state/restarts) plus a
# host-wide failed-unit count that is always emitted regardless of the
# allowlist, so "any service failed" alerting needs no configuration ---
if command -v systemctl >/dev/null 2>&1; then
  IFS=',' read -ra systemd_prefixes <<<"${MONITOR_SYSTEMD_MATCH:-${MONITOR_JOURNAL_MATCH:-akurai-,golfsetrid,rust-monitor}}"
  failed_count=0
  matched_units=()
  while IFS= read -r line; do
    unit=$(awk '{print $1}' <<<"$line")
    [[ "$unit" == *.service ]] || continue
    active=$(awk '{print $3}' <<<"$line")
    [ "$active" = "failed" ] && failed_count=$((failed_count + 1))
    base="${unit%.service}"
    for prefix in "${systemd_prefixes[@]}"; do
      [ -n "$prefix" ] || continue
      if [[ "$base" == "$prefix"* ]]; then
        matched_units+=("$unit")
        break
      fi
    done
  done < <(systemctl list-units --type=service --all --no-legend --plain --no-pager 2>/dev/null || true)
  push "$METRIC_SYSTEMD_FAILED" "$failed_count"

  for unit in "${matched_units[@]:-}"; do
    [ -n "$unit" ] || continue
    mem_current=""
    active_state=""
    restarts=""
    while IFS='=' read -r key value; do
      case "$key" in
      MemoryCurrent) mem_current="$value" ;;
      ActiveState) active_state="$value" ;;
      NRestarts) restarts="$value" ;;
      esac
    done < <(systemctl show "$unit" -p MemoryCurrent,ActiveState,NRestarts 2>/dev/null || true)
    name=$(sanitize_segment "${unit%.service}")
    if [ "$active_state" = "active" ]; then
      push "systemd.${name}.active" 1
    else
      push "systemd.${name}.active" 0
    fi
    # MemoryCurrent reads back as the u64 sentinel ("[not set]") when memory
    # accounting is disabled for the unit; skip rather than push a bogus
    # multi-exabyte sample.
    if [[ "$mem_current" =~ ^[0-9]+$ ]] && [ "$mem_current" != "18446744073709551615" ]; then
      push "systemd.${name}.mem_kb" "$((mem_current / 1024))"
    fi
    if [[ "$restarts" =~ ^[0-9]+$ ]]; then
      push "systemd.${name}.restarts" "$restarts"
    fi
  done
fi

# --- docker containers: memory usage of every running container. No-ops
# cleanly when docker is absent or the socket is unreachable — optional
# per-host dependency, not a requirement ---
if command -v docker >/dev/null 2>&1; then
  running=0
  while IFS=$'\t' read -r raw_name mem_field; do
    [ -n "$raw_name" ] || continue
    used=$(awk -F'/' '{print $1}' <<<"$mem_field" | xargs)
    used_kb=$(awk -v raw="$used" '
      BEGIN {
        n = raw + 0
        unit = raw
        gsub(/^[0-9.]+/, "", unit)
        if (unit == "B") kb = n / 1024
        else if (unit == "KiB" || unit == "kB") kb = n
        else if (unit == "MiB" || unit == "MB") kb = n * 1024
        else if (unit == "GiB" || unit == "GB") kb = n * 1024 * 1024
        else if (unit == "TiB" || unit == "TB") kb = n * 1024 * 1024 * 1024
        else { print ""; next }
        printf "%.0f", kb
      }')
    running=$((running + 1))
    if [ -n "$used_kb" ]; then
      push "docker.$(sanitize_segment "$raw_name").mem_kb" "$used_kb"
    fi
  done < <(docker stats --no-stream --format '{{.Name}}\t{{.MemUsage}}' 2>/dev/null || true)
  push "$METRIC_DOCKER_RUNNING" "$running"
fi