Menu
Public repository / main
AkurAI-EC2 / bin/akurai-ec2
Latest change 4fe4eb7c1983febeca1ce0b127f5eed7640d5be0 - release: refuse manifests marked retired = "<reason>" by Ólafur Búi Ólafsson
#!/usr/bin/env bash
# akurai-ec2 — reusable ops for the AkurAI AWS platform box (mail.olibuijr.com).
#
# One place for the things we kept re-deriving: SSH, file shipping, deploying a
# single-binary app behind nginx+systemd, TLS, and status. Designed to oneshot
# future deploys instead of guessing.
#
# akurai-ec2 ssh '<cmd>' run a command on the box
# akurai-ec2 ship <local> <remote> scp a file up (creates parent dir)
# akurai-ec2 status services, ports, nginx, disk
# akurai-ec2 processes [pattern] list the running AkurAI fleet: unit,
# akurai-ec2 ps [pattern] state, PID, memory, uptime, port.
# Optional pattern overrides the fleet
# match (an egrep alternation).
# akurai-ec2 deploy-binary <name> <bin> <port> [appdir]
# push a single binary, scaffold/serve it
# via systemd on 127.0.0.1:<port>.
# Backs up the previous binary to
# <name>.prev, health-gates the new one
# (must answer HTTP on :<port>), and
# auto-rolls-back if it doesn't.
# akurai-ec2 migrate-bunfork-state [source-ssh]
# stop the source service, securely move
# encrypted state to EC2, migrate, and
# readiness-gate; source stays stopped.
# akurai-ec2 retire-bunfork-source [source-ssh] [domain]
# after public readiness, disable/remove
# the source unit and archive its runtime.
# akurai-ec2 verify-bunfork <bundle> verify bundle inventory locally
# akurai-ec2 dns <list|get|upsert|delete|verify> ...
# manage authoritative EC2 DNS zones
# (default) or legacy 1984 Hosting;
# mutations require --yes.
# akurai-ec2 workspace <request|resume|approve-delete|status|list> ...
# durable, fenced lifecycle control; source
# operations stay pending until supported.
# akurai-ec2 retire-app <name> --domain <fqdn> [--inventory]
# inventory an app without changing it.
# akurai-ec2 retire-app <name> --domain <fqdn> --apply --yes
# [--delete-data] [--delete-user]
# [--delete-certificate] [--delete-dns]
# remove bounded EC2 app resources and
# verify their absence.
# akurai-ec2 deploy-bunfork <bundle> [domain] [port]
# atomically deploy a verified Bunfork
# native bundle, persistent encrypted
# state, hardened systemd, nginx, and TLS.
# akurai-ec2 bunfork-backup <install|run|verify> [retention]
# install/run/verify encrypted Bunfork
# backups and the persistent daily timer.
# akurai-ec2 tasks-backup <install|run|verify> [retention]
# install/run/verify encrypted hourly
# AkurAI-Tasks backups and recovery.
# akurai-ec2 workbench-backup <install|run|verify> [retention]
# encrypt Workbench snapshots on Titan,
# store them on EC2, and verify restores.
# akurai-ec2 bunfork-monitor <install|run|verify> [domain] [warn-days] [recipient]
# install/run/verify persistent HTTPS
# readiness and TLS-expiry monitoring.
# akurai-ec2 nginx-proxy <domain> <port> reverse-proxy vhost + reload; re-asserts
# HTTPS if a cert already exists (so a
# redeploy never drops TLS)
# akurai-ec2 tls <domain> obtain/renew Let's Encrypt cert (certbot)
# akurai-ec2 cert-status [domain] [--warn-days N] [--json]
# strict public TLS, hostname, and expiry check
# akurai-ec2 release [patch|minor|major] unified release engine — reads the
# repo's akurai-deploy.toml and runs the
# whole release (gate → bump VERSION +
# Cargo.toml → hybrid changelog → tag +
# sync to AkurAI Build → deploy-binary + nginx + tls).
# Replaces each project's bespoke deploy.sh.
# [publish|ec2] legacy no-bump aliases
# [--mode update|publish|release] canonical release mode selector
# [--bump patch|minor|major] canonical bump selector for release mode
# [--dry-run] preview bump + changelog, ship nothing
# [--changelog "text"] one-line changelog (skip the git draft)
# [--yes] [-C <dir>] accept draft unattended / run in <dir>
# manifest key retired = "<reason>" refuse every release mode for an app retired from EC2
# akurai-ec2 rollback <name> restore <name>.prev (the previous binary)
# akurai-ec2 logs <service> journalctl -u <service> -n 80
# akurai-ec2 mail-autoconfig verify [domain...]
# probe email client auto-setup for a domain:
# Mozilla autoconfig (GET) + MS autodiscover
# (POST, must not 405) + TLS host-match + SRV.
# Defaults to olibuijr.com + golfsetridak.is.
# akurai-ec2 ports [list] registry vs live: every port, who owns
# it, and whether it's bound (the index)
# akurai-ec2 ports free [n] next n free port(s) in the app pool
# (8094-8199). Claim it in the registry
# BEFORE deploy-binary to avoid collisions.
# akurai-ec2 ports check <port> is <port> free? who holds it (registry+live)?
# akurai-ec2 ports drift flag collisions, crash-loops, and nginx
# vhosts pointing at the wrong upstream.
# akurai-ec2 allow-ssh [ip] oneshot: authorize SSH (port 22) from an
# IP (default: this host's current public
# IP) on the box SG, THEN verify SSH — one
# command. Idempotent. The SG locks 22 to
# /32s; a dynamic-IP change or a new host
# (e.g. a Titan CI worker) times out SSH
# until re-added. Also runs AUTOMATICALLY:
# ssh_run self-heals on a connection
# timeout (authorize + retry).
# akurai-ec2 health [--all] fan-out health check across all registered
# apps in AKURAI_APPS.toml
# akurai-ec2 search <pattern> cross-repo search across all AkurAI Rust src
# akurai-ec2 repos status git status across all AkurAI repos
# akurai-ec2 validate-config [path] validate an akurai-deploy.toml schema
# akurai-ec2 secret-sync SERVICE KEY ENV_FILE
# read a secret from stdin, atomically
# update the protected env, and restart.
# akurai-ec2 mariadb <status|health|create-db|create-user|sql|dump|logs|install>
# operate the dockerized MariaDB stack
# (prod :3306, dev :3307) via on-box
# akurai-mariadbctl.
# akurai-ec2 provision-oidc-client NAME REDIRECT ENV_PREFIX EMAIL FOLDER [REMOTE_ENV_FILE]
# create IDP client, store credentials in
# PassVault, and merge a protected app env.
# akurai-ec2 release propagate build + deploy Framework binary to all
# shared-binary apps (in AKURAI_APPS.toml)
set -euo pipefail
# Resolve the *account* home from passwd, not $HOME: Paperclip agent sessions run
# with a sandboxed $HOME, so keying off it hides the ssh/aws config and breaks deploys.
ACCOUNT_HOME="$(getent passwd "$(id -un)" | cut -d: -f6)"
ACCOUNT_HOME="${ACCOUNT_HOME:-$HOME}"
SSH_HOST="${AKURAI_EC2_SSH:-akurai-ec2}" # ~/.ssh/config alias
SSH_CONFIG="${AKURAI_EC2_SSH_CONFIG:-$ACCOUNT_HOME/.ssh/config}"
PUBLIC_IP="3.94.46.219"
# Intended port assignments (SSOT) — see the _VMPORTS skill. `ports` reads this.
PORTS_REGISTRY="${AKURAI_PORTS_REGISTRY:-$ACCOUNT_HOME/.config/akurai-ec2/ports.registry}"
APP_POOL_LO=8094
APP_POOL_HI=8199
SCRIPT_SOURCE="${BASH_SOURCE[0]}"
if command -v readlink >/dev/null 2>&1; then
resolved_script="$(readlink -f "$SCRIPT_SOURCE" 2>/dev/null || true)"
[ -n "$resolved_script" ] && SCRIPT_SOURCE="$resolved_script"
fi
SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_SOURCE")" && pwd)"
MANIFEST_HELPER="$SCRIPT_DIR/../libexec/akurai-ec2-manifest.py"
DNS_HELPER="$SCRIPT_DIR/../libexec/akurai-ec2-dns.py"
RETIRE_APP_HELPER="$SCRIPT_DIR/../libexec/akurai-ec2-retire-app.py"
WORKSPACE_HELPER="$SCRIPT_DIR/../libexec/akurai-ec2-workspace.py"
[ -x "$WORKSPACE_HELPER" ] || WORKSPACE_HELPER="$SCRIPT_DIR/akurai-ec2-workspace.py"
declare -Ag RELEASE_MANIFEST=()
RELEASE_MANIFEST_KIND="flat"
say() { printf '\033[1;34m▸ %s\033[0m\n' "$*"; }
die() { printf '\033[1;31m✗ %s\033[0m\n' "$*" >&2; exit 1; }
ssh_cmd() { command ssh -F "$SSH_CONFIG" "$@"; }
scp_cmd() { command scp -F "$SSH_CONFIG" "$@"; }
backup_dispatch() {
local kind="$1"
local action="${2:-verify}"
local retention="$3"
local maximum="$4"
case "$action" in
install|run|verify) ;;
*) die "$kind backup action must be install, run, or verify" ;;
esac
[[ "$retention" =~ ^[0-9]+$ ]] && [ "$retention" -ge 2 ] && [ "$retention" -le "$maximum" ] || \
die "$kind backup retention must be an integer from 2 through $maximum"
BACKUP_ACTION="$action"
BACKUP_RETENTION="$retention"
}
# Authorize SSH (port 22) from an IP (default: current public IP) on the box's
# security group. Óli's home IP is dynamic, so a drift times out SSH; this re-adds
# it. Returns 0 on success/already-allowed, 1 on any failure (caller decides).
ensure_my_ip() {
local ip region sg out
if [ -z "${AWS_PROFILE:-}" ] && [ -z "${AWS_ACCESS_KEY_ID:-}" ] && \
grep -q '^\[profile akurai-ec2\]$' "${AWS_CONFIG_FILE:-$ACCOUNT_HOME/.aws/config}" 2>/dev/null; then
export AWS_CONFIG_FILE="${AWS_CONFIG_FILE:-$ACCOUNT_HOME/.aws/config}"
export AWS_PROFILE=akurai-ec2
fi
ip="${1:-}"
[ -z "$ip" ] && ip=$(curl -s --max-time 8 https://api.ipify.org 2>/dev/null || true)
[ -n "$ip" ] || { printf '\033[1;31m✗ could not determine your public IP\033[0m\n' >&2; return 1; }
command -v aws >/dev/null 2>&1 || { printf '\033[1;31m✗ aws CLI not available\033[0m\n' >&2; return 1; }
region="${AWS_DEFAULT_REGION:-us-east-1}"
sg=$(AWS_DEFAULT_REGION="$region" aws ec2 describe-instances \
--filters "Name=ip-address,Values=$PUBLIC_IP" \
--query "Reservations[].Instances[].SecurityGroups[].GroupId" --output text 2>/dev/null || true)
[ -n "$sg" ] || { printf '\033[1;31m✗ could not find the security group for %s (aws creds / region?)\033[0m\n' "$PUBLIC_IP" >&2; return 1; }
if out=$(AWS_DEFAULT_REGION="$region" aws ec2 authorize-security-group-ingress \
--group-id "$sg" \
--ip-permissions "IpProtocol=tcp,FromPort=22,ToPort=22,IpRanges=[{CidrIp=$ip/32,Description=\"akurai-ec2 allow-ssh\"}]" 2>&1); then
say "SSH allowed from $ip/32 on $sg"
elif printf '%s' "$out" | grep -q Duplicate; then
say "already allowed ($ip)"
else
printf '\033[1;31m✗ %s\033[0m\n' "$out" >&2; return 1
fi
return 0
}
# SSH wrapper with an auto-recovery preflight: if the connection times out (the
# signature of a dynamic-IP drift locking us out of the SG), authorize the current
# IP once and retry. Every command path (ship/status/deploy/…) inherits this.
AKURAI_EC2_SSH_RECOVERED=0
ssh_run() {
local err rc=0
err="$(mktemp)"
ssh_cmd -o ConnectTimeout=15 "$SSH_HOST" "$@" 2>"$err" || rc=$?
cat "$err" >&2
if [ "$rc" -ne 0 ] && [ "$AKURAI_EC2_SSH_RECOVERED" = 0 ] && grep -qiE 'timed out' "$err"; then
AKURAI_EC2_SSH_RECOVERED=1
say "SSH timed out — auto-authorizing your current IP on the security group…"
if ensure_my_ip; then
rm -f "$err"
ssh_cmd -o ConnectTimeout=15 "$SSH_HOST" "$@"
return $?
fi
fi
rm -f "$err"
return "$rc"
}
release_load_manifest_flat() {
local manifest_path="$1" ln t k v
RELEASE_MANIFEST=()
RELEASE_MANIFEST_KIND="flat"
while IFS= read -r ln; do
t="$(printf '%s' "$ln" | sed -E 's/^[[:space:]]+//')"
case "$t" in \#*|"") continue ;; esac
case "$ln" in *=*) ;; *) continue ;; esac
k="$(printf '%s' "${ln%%=*}" | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//')"
v="$(printf '%s' "${ln#*=}" | sed -E 's/[[:space:]]+#.*$//; s/^[[:space:]]+//; s/[[:space:]]+$//')"
v="${v%\"}"; v="${v#\"}"
RELEASE_MANIFEST["$k"]="$v"
done < "$manifest_path"
}
release_load_manifest_toml() {
local manifest_path="$1" helper_output key value
local -A manifest_tmp=()
[ -f "$MANIFEST_HELPER" ] || die "manifest helper missing: $MANIFEST_HELPER"
command -v python3 >/dev/null 2>&1 || die "python3 is required for nested release manifests"
if ! helper_output="$(python3 "$MANIFEST_HELPER" "$manifest_path")"; then
die "manifest helper failed: $manifest_path"
fi
while IFS=$'\t' read -r key value; do
[ -n "$key" ] || continue
manifest_tmp["$key"]="$value"
done <<< "$helper_output"
RELEASE_MANIFEST=()
RELEASE_MANIFEST_KIND="toml"
for key in "${!manifest_tmp[@]}"; do
RELEASE_MANIFEST["$key"]="${manifest_tmp["$key"]}"
done
}
release_load_manifest() {
local manifest_path="$1"
if grep -qE '^[[:space:]]*\[' "$manifest_path"; then
release_load_manifest_toml "$manifest_path"
else
release_load_manifest_flat "$manifest_path"
fi
}
mf_get() {
local key
for key in "$@"; do
if [[ -v RELEASE_MANIFEST["$key"] ]]; then
printf '%s\n' "${RELEASE_MANIFEST["$key"]}"
return 0
fi
done
return 1
}
mf_get_bool() {
local default_value="$1" value key_hint
shift
key_hint="${1:-value}"
value="$(mf_get "$@" || true)"
if [ -z "$value" ]; then
printf '%s\n' "$default_value"
return 0
fi
case "${value,,}" in
1|true|yes|on) printf 'true\n' ;;
0|false|no|off) printf 'false\n' ;;
*) die "manifest: expected boolean for '$key_hint', got '$value'" ;;
esac
}
mf_require() {
local label="$1" value
shift
value="$(mf_get "$@" || true)"
[ -n "$value" ] || die "manifest: '$label' required"
printf '%s\n' "$value"
}
mf_get_string_array_lines() {
local label="$1" value decoded
shift
value="$(mf_get "$@" || true)"
[ -n "$value" ] || return 0
if [[ "$value" != \[* ]]; then
printf '%s\n' "$value"
return 0
fi
command -v python3 >/dev/null 2>&1 || die "python3 is required for array manifest fields"
if ! decoded="$(VALUE="$value" python3 - "$label" <<'PY'
import json
import os
import sys
label = sys.argv[1]
raw = os.environ["VALUE"]
try:
data = json.loads(raw)
except json.JSONDecodeError as exc:
print(f"manifest: invalid array for '{label}': {exc}", file=sys.stderr)
raise SystemExit(1)
if not isinstance(data, list):
print(f"manifest: expected array for '{label}'", file=sys.stderr)
raise SystemExit(1)
for idx, item in enumerate(data):
if not isinstance(item, str):
print(
f"manifest: '{label}' array item {idx + 1} must be a string",
file=sys.stderr,
)
raise SystemExit(1)
if any(ch in item for ch in ("\0", "\r", "\n")):
print(
f"manifest: '{label}' array item {idx + 1} contains unsupported control characters",
file=sys.stderr,
)
raise SystemExit(1)
print(item)
PY
)"; then
die "manifest: invalid array field '$label'"
fi
printf '%s' "$decoded"
}
verify_bunfork_bundle() {
local bundle="$1"
python3 - "$bundle" <<'PY'
import hashlib, json, pathlib, sys
root = pathlib.Path(sys.argv[1])
manifest = json.loads((root / "bunfork.json").read_text())
if manifest.get("schema") != "bunfork-deployment-v2":
raise SystemExit("invalid Bunfork deployment schema")
expected = set(manifest.get("files", {})) | {"bunfork.json"}
actual = set()
for path in root.rglob("*"):
rel = path.relative_to(root).as_posix()
if path.is_symlink() or (path.exists() and not (path.is_file() or path.is_dir())):
raise SystemExit(f"unsafe bundle entry: {rel}")
if path.is_file():
actual.add(rel)
if actual != expected:
raise SystemExit(f"bundle inventory mismatch: missing={sorted(expected-actual)} extra={sorted(actual-expected)}")
for rel, record in manifest["files"].items():
data = (root / rel).read_bytes()
if len(data) != record["bytes"] or hashlib.sha256(data).hexdigest() != record["sha256"]:
raise SystemExit(f"bundle digest mismatch: {rel}")
PY
}
cmd="${1:-help}"; shift || true
case "$cmd" in
workspace)
[ -x "$WORKSPACE_HELPER" ] || die "workspace helper not found: $WORKSPACE_HELPER"
AKURAI_EC2_CLI="${AKURAI_EC2_CLI:-$SCRIPT_SOURCE}" exec python3 "$WORKSPACE_HELPER" "$@"
;;
dns)
[ -x "$DNS_HELPER" ] || die "DNS helper not found: $DNS_HELPER"
exec python3 "$DNS_HELPER" "$@"
;;
retire-app)
[ -x "$RETIRE_APP_HELPER" ] || die "retire-app helper not found: $RETIRE_APP_HELPER"
AKURAI_EC2_CLI="${AKURAI_EC2_CLI:-$SCRIPT_SOURCE}" exec python3 "$RETIRE_APP_HELPER" "$@"
;;
esac
case "$cmd" in
ssh)
ssh_run "$@"
;;
ship)
[ $# -eq 2 ] || die "usage: akurai-ec2 ship <local> <remote>"
local_file="$1"; remote="$2"
ssh_run "mkdir -p \"\$(dirname '$remote')\""
scp_cmd -o ConnectTimeout=15 "$local_file" "$SSH_HOST:$remote"
say "shipped $local_file → $remote"
;;
status)
ssh_run 'bash -s' <<'EOF'
echo "=== app services ==="
systemctl list-units --type=service --state=running | grep -iE 'akurai|golfset|mail|idp|bunfork' || true
echo "=== listening ports ==="
sudo ss -tlnp 2>/dev/null | grep -E 'LISTEN' | awk '{print $4}' | sort -u | tail -20
echo "=== nginx vhosts ==="
ls /etc/nginx/sites-enabled/
echo "=== disk ==="
df -h / | tail -1
EOF
;;
deploy-binary)
# deploy-binary <name> <local-bin> <port> [appdir-to-upload] [--json]
[ $# -ge 3 ] || die "usage: akurai-ec2 deploy-binary <name> <local-bin> <port> [appdir] [--json]"
name="$1"; bin="$2"; port="$3"; shift 3
appdir=""; emit_json=0
while [ $# -gt 0 ]; do
case "$1" in
--json) emit_json=1 ;;
*) [ -z "$appdir" ] || die "deploy-binary: unexpected argument '$1'"; appdir="$1" ;;
esac
shift
done
[ -f "$bin" ] || die "deploy-binary: binary not found: $bin"
deploy_timestamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
binary_sha256="$(sha256sum "$bin" | cut -d' ' -f1)"
asset_sha256=""
if [ -n "$appdir" ]; then
[ -d "$appdir" ] || die "deploy-binary: app directory not found: $appdir"
asset_sha256="$(python3 - "$appdir" <<'PY'
import hashlib
import pathlib
import sys
root = pathlib.Path(sys.argv[1])
digest = hashlib.sha256()
for path in sorted(item for item in root.rglob("*") if item.is_file()):
digest.update(path.relative_to(root).as_posix().encode())
digest.update(b"\0")
digest.update(path.read_bytes())
print(digest.hexdigest())
PY
)"
fi
base="/opt/$name"
previous_backup="$(ssh_run "[ -f $base/bin/$name ] && printf true || printf false")"
say "deploying '$name' on port $port"
ssh_run "sudo mkdir -p $base/bin && sudo chown -R ubuntu:ubuntu $base"
# Stop the service first — a running binary is busy and can't be overwritten.
ssh_run "sudo systemctl stop $name.service 2>/dev/null || true"
# Back up the currently-deployed binary before overwriting it — the rollback
# target if the new build turns out unhealthy (see the health-gate below).
ssh_run "[ -f $base/bin/$name ] && cp -p $base/bin/$name $base/bin/$name.prev || true"
scp_cmd -o ConnectTimeout=15 "$bin" "$SSH_HOST:$base/bin/$name"
ssh_run "chmod +x $base/bin/$name"
if [ -n "$appdir" ]; then
say "uploading app dir $appdir"
( cd "$appdir" && tar czf - . ) | ssh_run "mkdir -p $base/app && tar xzf - -C $base/app"
fi
say "installing systemd unit $name.service"
# Optional overrides (set by `akurai-ec2 release` from the app manifest) let
# apps that DON'T follow the Framework `serve --dir frontend` convention keep
# their own ExecStart / EnvironmentFile / data dirs. All unset → the default
# Framework unit below, so existing callers are unchanged.
# AKURAI_EXEC_START custom ExecStart; tokens {bin}→$base/bin/$name,
# {port}→$port, {app}→$base/app are expanded
# AKURAI_ENV_FILE systemd EnvironmentFile= (must already exist on the box)
# AKURAI_DATA_DIR space-separated dirs to create + chown ubuntu before start
# AKURAI_USER systemd User= (default ubuntu) — for apps with a dedicated
# service account; data dirs are chowned to it
di_user="${AKURAI_USER:-ubuntu}"
di_exec="${AKURAI_EXEC_START:-$base/bin/$name serve --dir $base/app/frontend --host 127.0.0.1 --port $port --name $name}"
di_exec="${di_exec//\{bin\}/$base/bin/$name}"; di_exec="${di_exec//\{port\}/$port}"; di_exec="${di_exec//\{app\}/$base/app}"
di_envline=""; if [ -n "${AKURAI_ENV_FILE:-}" ]; then di_envline="EnvironmentFile=${AKURAI_ENV_FILE}"; fi
# AKURAI_HARDEN=1 append the standard sandboxing block (NoNewPrivileges,
# PrivateTmp, ProtectSystem=full, ReadWritePaths=$base/app
# plus any AKURAI_DATA_DIR entries) — opt-in since it can
# break apps that write outside those paths.
di_hardenline=""
if [ "${AKURAI_HARDEN:-0}" = 1 ] || [ "${AKURAI_HARDEN:-0}" = true ]; then
di_rwpaths="$base/app"
if [ -n "${AKURAI_DATA_DIR:-}" ]; then
while IFS= read -r d || [ -n "$d" ]; do [ -n "$d" ] && di_rwpaths="$di_rwpaths $d"; done <<< "$AKURAI_DATA_DIR"
fi
di_hardenline="NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=full
ReadWritePaths=$di_rwpaths"
fi
if [ -n "${AKURAI_DATA_DIR:-}" ]; then
while IFS= read -r d || [ -n "$d" ]; do
[ -n "$d" ] || continue
ssh_run "sudo mkdir -p '$d' && sudo chown $di_user:$di_user '$d'" </dev/null
done <<< "$AKURAI_DATA_DIR"
fi
# Always ensure the systemd WorkingDirectory ($base/app) exists and is owned
# by the service user. Single-binary apps upload no appdir, so without this
# the unit fails at CHDIR (status=200/CHDIR) and crash-loops every deploy.
ssh_run "sudo mkdir -p $base/app && sudo chown -R $di_user:$di_user $base/app"
ssh_run "sudo tee /etc/systemd/system/$name.service >/dev/null" <<EOF
[Unit]
Description=$name (AkurAI single-binary app)
After=network.target
[Service]
Type=simple
User=$di_user
WorkingDirectory=$base/app
${di_envline}
ExecStart=$di_exec
Restart=on-failure
RestartSec=2
${di_hardenline}
[Install]
WantedBy=multi-user.target
EOF
ssh_run "sudo systemctl daemon-reload && sudo systemctl enable --now $name.service && sleep 1 && systemctl is-active $name.service"
# Health-gate: confirm the app actually answers HTTP on its port — 'active'
# alone can't tell a serving app from one that crashes on first request. Any
# HTTP status means it's serving; 000 = connection refused / crashed.
health_ok=0; code=000
for _ in 1 2 3 4 5 6; do
code=$(ssh_run "curl -s -o /dev/null -w '%{http_code}' --max-time 4 http://127.0.0.1:$port/ 2>/dev/null || echo 000")
[ "$code" != "000" ] && { health_ok=1; break; }
sleep 2
done
if [ "$health_ok" = 1 ]; then
say "health: $name answering on :$port (HTTP $code)"
else
say "$name failed its health check on :$port — auto-rolling back"
if ssh_run "[ -f $base/bin/$name.prev ]"; then
ssh_run "sudo systemctl stop $name.service; cp -p $base/bin/$name.prev $base/bin/$name; sudo systemctl start $name.service; sleep 2; systemctl is-active $name.service"
die "rolled back $name to the previous binary (the new build was unhealthy on :$port)"
else
die "$name is unhealthy on :$port and there is no .prev backup to roll back to — investigate"
fi
fi
say "deployed: $name on 127.0.0.1:$port"
if [ "$emit_json" = 1 ]; then
DEPLOY_NAME="$name" DEPLOY_PORT="$port" DEPLOY_TIMESTAMP="$deploy_timestamp" \
DEPLOY_BINARY_SHA="$binary_sha256" DEPLOY_ASSET_SHA="$asset_sha256" \
DEPLOY_PREVIOUS="$previous_backup" DEPLOY_HEALTH="$code" python3 - <<'PY'
import json
import os
print(json.dumps({
"app": os.environ["DEPLOY_NAME"],
"port": int(os.environ["DEPLOY_PORT"]),
"deployed_at": os.environ["DEPLOY_TIMESTAMP"],
"binary_sha256": os.environ["DEPLOY_BINARY_SHA"],
"asset_tree_sha256": os.environ["DEPLOY_ASSET_SHA"] or None,
"previous_binary_backed_up": os.environ["DEPLOY_PREVIOUS"] == "true",
"systemd_state": "active",
"loopback_http_status": int(os.environ["DEPLOY_HEALTH"]),
"ok": True,
}, separators=(",", ":")))
PY
fi
;;
verify-bunfork)
[ $# -eq 1 ] || die "usage: akurai-ec2 verify-bunfork <bundle>"
bundle="$(readlink -m "$1")"
[ -d "$bundle" ] || die "bundle directory not found: $bundle"
[ -f "$bundle/bunfork.json" ] || die "bundle manifest missing: $bundle/bunfork.json"
verify_bunfork_bundle "$bundle"
say "verified Bunfork bundle: $bundle"
;;
migrate-bunfork-state)
[ $# -le 1 ] || die "usage: akurai-ec2 migrate-bunfork-state [source-ssh]"
source_host="${1:-titan}"
archive="$(mktemp --suffix=.tgz)"
chmod 0600 "$archive"
migrated=0
cleanup_migration() {
rm -f "$archive"
if [ "$migrated" != 1 ]; then
ssh_cmd -o ConnectTimeout=10 "$source_host" 'systemctl --user start bunfork.service' >/dev/null 2>&1 || true
fi
}
trap cleanup_migration EXIT
say "stopping source Bunfork on $source_host for a consistent encrypted-state transfer"
ssh_cmd -o BatchMode=yes -o ConnectTimeout=15 "$source_host" \
'set -eu; systemctl --user stop bunfork.service; test -s "$HOME/.local/share/bunfork/bunfork.db"; test -s "$HOME/.config/bunfork/db.key"; test -s "$HOME/.config/bunfork/api.token"; tar -C "$HOME" -czf - .local/share/bunfork/bunfork.db .config/bunfork/db.key .config/bunfork/api.token' \
>"$archive"
ssh_run true
remote_archive="/tmp/bunfork-state-$(date -u +%Y%m%dT%H%M%SZ).tgz"
scp_cmd -o ConnectTimeout=15 "$archive" "$SSH_HOST:$remote_archive"
ssh_run "bash -s -- '$remote_archive'" <<'REMOTE'
set -euo pipefail
archive="$1"
stage="$(mktemp -d)"
backup="/var/backups/bunfork/pre-migration-$(date -u +%Y%m%dT%H%M%SZ)"
cleanup() { rm -rf "$stage"; rm -f "$archive"; }
trap cleanup EXIT
mkdir -p "$stage"
tar -xzf "$archive" -C "$stage"
sudo install -d -m 0700 "$backup"
for path in /var/lib/bunfork/bunfork.db /etc/bunfork/db.key /etc/bunfork/api.token; do
if sudo test -f "$path"; then sudo cp -a "$path" "$backup/"; fi
done
sudo systemctl stop bunfork.service
sudo install -o bunfork -g bunfork -m 0600 "$stage/.local/share/bunfork/bunfork.db" /var/lib/bunfork/bunfork.db
sudo install -o bunfork -g bunfork -m 0600 "$stage/.config/bunfork/db.key" /etc/bunfork/db.key
sudo install -o bunfork -g bunfork -m 0600 "$stage/.config/bunfork/api.token" /etc/bunfork/api.token
if ! sudo -u bunfork /opt/bunfork/current/bunfork --database /var/lib/bunfork/bunfork.db --key-file /etc/bunfork/db.key migrate || \
! sudo systemctl start bunfork.service; then
sudo systemctl stop bunfork.service || true
for name in bunfork.db db.key api.token; do
if sudo test -f "$backup/$name"; then
case "$name" in
bunfork.db) target=/var/lib/bunfork/bunfork.db ;;
*) target=/etc/bunfork/$name ;;
esac
sudo cp -a "$backup/$name" "$target"
fi
done
sudo systemctl start bunfork.service || true
echo "state migration failed; restored EC2 pre-migration state" >&2
exit 1
fi
for _ in 1 2 3 4 5 6 7 8 9 10; do
curl -fsS --max-time 4 http://127.0.0.1:3100/_bunfork/ready >/dev/null 2>&1 && exit 0
sleep 2
done
echo "migrated Bunfork did not become ready" >&2
exit 1
REMOTE
curl -fsS --max-time 20 https://bunfork.olibuijr.com/_bunfork/ready >/dev/null || die "public Bunfork readiness failed after state migration"
migrated=1
say "migrated encrypted Bunfork state; source service remains stopped pending retirement"
;;
retire-bunfork-source)
[ $# -le 2 ] || die "usage: akurai-ec2 retire-bunfork-source [source-ssh] [domain]"
source_host="${1:-titan}"
domain="${2:-bunfork.olibuijr.com}"
curl -fsS --max-time 20 "https://$domain/_bunfork/ready" >/dev/null || die "refuse to retire source: public Bunfork is not ready"
say "retiring source Bunfork service on $source_host"
ssh_cmd -o BatchMode=yes -o ConnectTimeout=15 "$source_host" 'bash -s' <<'REMOTE'
set -euo pipefail
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
systemctl --user disable --now bunfork.service 2>/dev/null || true
rm -f "$HOME/.config/systemd/user/bunfork.service"
systemctl --user daemon-reload
if [ -d "$HOME/.local/lib/bunfork" ]; then
mv "$HOME/.local/lib/bunfork" "$HOME/.local/lib/bunfork.retired-$stamp"
fi
if systemctl --user list-unit-files bunfork.service --no-legend 2>/dev/null | grep -q bunfork; then
echo "source Bunfork unit still exists" >&2
exit 1
fi
printf 'retired_runtime=%s\n' "$HOME/.local/lib/bunfork.retired-$stamp"
REMOTE
say "source Bunfork service removed; encrypted source data/config retained for rollback"
;;
bunfork-backup)
[ $# -le 2 ] || die "usage: akurai-ec2 bunfork-backup <install|run|verify> [retention]"
backup_dispatch "bunfork" "${1:-verify}" "${2:-14}" 365
action="$BACKUP_ACTION"
retention="$BACKUP_RETENTION"
if [ "$action" = install ]; then
say "installing encrypted Bunfork backup service and timer"
ssh_run "bash -s -- '$retention'" <<'REMOTE'
set -euo pipefail
retention="$1"
sudo install -d -o bunfork -g bunfork -m 0700 /var/backups/bunfork
sudo install -d -o root -g root -m 0755 /usr/local/libexec
sudo tee /usr/local/libexec/bunfork-backup >/dev/null <<'SCRIPT'
#!/usr/bin/env bash
set -euo pipefail
umask 0077
retention="${BUNFORK_BACKUP_RETENTION:-14}"
[[ "$retention" =~ ^[0-9]+$ ]] && [ "$retention" -ge 2 ] && [ "$retention" -le 365 ]
directory=/var/backups/bunfork
stamp="$(date -u +%Y%m%dT%H%M%S)-$(date +%N)"
destination="$directory/bunfork-$stamp.db"
/opt/bunfork/current/bunfork \
--database /var/lib/bunfork/bunfork.db \
--key-file /etc/bunfork/db.key \
backup --out "$destination"
mapfile -t backups < <(find "$directory" -maxdepth 1 -type f -name 'bunfork-*.db' -printf '%f\n' | LC_ALL=C sort)
excess=$((${#backups[@]} - retention))
if [ "$excess" -gt 0 ]; then
for ((index = 0; index < excess; index++)); do
rm -- "$directory/${backups[$index]}"
done
fi
SCRIPT
sudo chown root:root /usr/local/libexec/bunfork-backup
sudo chmod 0755 /usr/local/libexec/bunfork-backup
sudo tee /etc/systemd/system/bunfork-backup.service >/dev/null <<UNIT
[Unit]
Description=Create a verified encrypted Bunfork database backup
After=bunfork.service
Requires=bunfork.service
[Service]
Type=oneshot
User=bunfork
Group=bunfork
Environment=BUNFORK_BACKUP_RETENTION=$retention
ExecStart=/usr/local/libexec/bunfork-backup
UMask=0077
NoNewPrivileges=true
PrivateDevices=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadOnlyPaths=/opt/bunfork /etc/bunfork
ReadWritePaths=/var/backups/bunfork /var/lib/bunfork
CapabilityBoundingSet=
LockPersonality=true
MemoryDenyWriteExecute=true
RestrictSUIDSGID=true
RestrictAddressFamilies=AF_UNIX
[Install]
WantedBy=multi-user.target
UNIT
sudo tee /etc/systemd/system/bunfork-backup.timer >/dev/null <<'UNIT'
[Unit]
Description=Daily encrypted Bunfork database backup
[Timer]
OnCalendar=*-*-* 03:15:00 UTC
RandomizedDelaySec=15m
Persistent=true
Unit=bunfork-backup.service
[Install]
WantedBy=timers.target
UNIT
sudo systemctl daemon-reload
sudo systemctl enable --now bunfork-backup.timer
sudo systemctl start bunfork-backup.service
REMOTE
elif [ "$action" = run ]; then
say "running an encrypted Bunfork backup"
ssh_run "sudo systemctl start bunfork-backup.service"
fi
say "verifying encrypted Bunfork backup and recovery path"
ssh_run 'bash -s' <<'REMOTE'
set -euo pipefail
sudo systemctl is-enabled --quiet bunfork-backup.timer
sudo systemctl is-active --quiet bunfork-backup.timer
sudo systemctl show bunfork-backup.service -p Result --value | grep -qx success
latest="$(sudo find /var/backups/bunfork -maxdepth 1 -type f -name 'bunfork-*.db' -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-)"
[ -n "$latest" ] && sudo test -s "$latest"
[ "$(sudo stat -c %a "$latest")" = 600 ]
[ "$(sudo stat -c %U:%G "$latest")" = bunfork:bunfork ]
header="$(sudo od -An -N16 -tx1 "$latest" | tr -d ' \n')"
[ "$header" != 53514c69746520666f726d6174203300 ]
scratch="$(sudo -u bunfork mktemp -d /var/backups/bunfork/.verify-XXXXXXXX)"
cleanup() { sudo rm -rf "$scratch"; }
trap cleanup EXIT
sudo -u bunfork install -m 0600 "$latest" "$scratch/backup.db"
sudo -u bunfork install -m 0600 "$latest" "$scratch/restored.db"
sudo -u bunfork /opt/bunfork/current/bunfork \
--database "$scratch/restored.db" \
--key-file /etc/bunfork/db.key \
restore "$scratch/backup.db" --yes >/dev/null
sudo -u bunfork /opt/bunfork/current/bunfork \
--database "$scratch/restored.db" \
--key-file /etc/bunfork/db.key migrate >/dev/null
count="$(sudo find /var/backups/bunfork -maxdepth 1 -type f -name 'bunfork-*.db' | wc -l)"
mtime="$(sudo stat -c %y "$latest" | cut -d. -f1)Z"
printf 'timer=active latest=%s backups=%s recovery=verified\n' "$mtime" "$count"
REMOTE
;;
tasks-backup)
[ $# -le 2 ] || die "usage: akurai-ec2 tasks-backup <install|run|verify> [retention]"
backup_dispatch "AkurAI-Tasks" "${1:-verify}" "${2:-48}" 720
action="$BACKUP_ACTION"
retention="$BACKUP_RETENTION"
if [ "$action" = install ]; then
say "installing encrypted AkurAI-Tasks backup service and hourly timer"
ssh_run "bash -s -- '$retention'" <<'REMOTE'
set -euo pipefail
retention="$1"
recipient="akurai-tasks-backup@olibuijr.com"
sudo install -d -o root -g root -m 0700 /etc/akurai-tasks-backup/gnupg
sudo install -d -o root -g root -m 0711 /var/backups/akurai-tasks
sudo install -d -o ubuntu -g root -m 2770 /var/backups/akurai-tasks/staging
sudo find /var/backups/akurai-tasks/staging -maxdepth 1 -type f \
\( -name 'akurai-tasks-*.db' -o -name 'akurai-tasks-*.db.manifest.json' \) -delete
sudo install -d -o root -g root -m 0755 /usr/local/libexec
if ! sudo gpg --batch --homedir /etc/akurai-tasks-backup/gnupg --with-colons --list-keys "$recipient" 2>/dev/null | grep -q '^pub:'; then
sudo gpg --batch --homedir /etc/akurai-tasks-backup/gnupg --passphrase '' \
--quick-generate-key "$recipient" rsa3072 encrypt 0
fi
fingerprint="$(sudo gpg --batch --homedir /etc/akurai-tasks-backup/gnupg --with-colons --fingerprint "$recipient" | awk -F: '$1 == "fpr" { print $10; exit }')"
[[ "$fingerprint" =~ ^[A-F0-9]{40}$ ]]
printf '%s\n' "$fingerprint" | sudo tee /etc/akurai-tasks-backup/recipient.fingerprint >/dev/null
sudo chmod 0600 /etc/akurai-tasks-backup/recipient.fingerprint
while IFS= read -r archive; do
listing="$(mktemp)"
if sudo gpg --batch --homedir /etc/akurai-tasks-backup/gnupg --decrypt "$archive" 2>/dev/null |
sudo tar -tf - >"$listing" 2>/dev/null &&
[ "$(grep -Ec '^akurai-tasks-[0-9]+-[0-9]+\.db$' "$listing")" -eq 1 ] &&
[ "$(grep -Ec '^akurai-tasks-[0-9]+-[0-9]+\.db\.manifest\.json$' "$listing")" -eq 1 ]; then
:
else
sudo rm -f -- "$archive"
fi
rm -f "$listing"
done < <(sudo find /var/backups/akurai-tasks -maxdepth 1 -type f -name 'akurai-tasks-*.tar.gpg' -print)
sudo tee /usr/local/libexec/akurai-tasks-backup >/dev/null <<'SCRIPT'
#!/usr/bin/env bash
set -euo pipefail
umask 0077
retention="${AKURAI_TASKS_BACKUP_RETENTION:-48}"
[[ "$retention" =~ ^[0-9]+$ ]] && [ "$retention" -ge 2 ] && [ "$retention" -le 720 ]
set -a
source /etc/akurai-tasks/akurai-tasks.env
set +a
: "${AKURAI_TASKS_MCP_TOKEN:?AKURAI_TASKS_MCP_TOKEN is required}"
directory=/var/backups/akurai-tasks
staging="$directory/staging"
config="$(mktemp)"
response="$(mktemp)"
backup=""
incoming=""
cleanup() {
rm -f "$config" "$response"
case "$backup" in
"$staging"/akurai-tasks-*.db) rm -f -- "$backup" "$backup.manifest.json" ;;
esac
case "$incoming" in
"$directory"/.incoming-*.tar.gpg) rm -f -- "$incoming" ;;
esac
}
trap cleanup EXIT
printf 'header = "Authorization: Bearer %s"\n' "$AKURAI_TASKS_MCP_TOKEN" >"$config"
curl --fail --silent --show-error --config "$config" \
--header 'Content-Type: application/json' \
--data '{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"system_backup","arguments":{}}}' \
http://127.0.0.1:8108/mcp >"$response"
backup="$(python3 - "$response" <<'PY'
import json
import sys
document = json.load(open(sys.argv[1], encoding="utf-8"))
if "error" in document:
raise SystemExit(document["error"].get("message", "backup RPC failed"))
result = json.loads(document["result"]["content"][0]["text"])
print(result["backup"])
PY
)"
case "$backup" in "$staging"/akurai-tasks-*.db) ;; *) echo "unexpected backup path" >&2; exit 1 ;; esac
[ -s "$backup" ] && [ -s "$backup.manifest.json" ]
name="$(basename "$backup")"
stamp="$(date -u +%Y%m%dT%H%M%S)-$(date +%N)"
destination="$directory/akurai-tasks-$stamp.tar.gpg"
incoming="$directory/.incoming-$stamp.tar.gpg"
fingerprint="$(</etc/akurai-tasks-backup/recipient.fingerprint)"
tar -C "$staging" -cf - "$name" "$name.manifest.json" |
gpg --batch --yes --trust-model always --homedir /etc/akurai-tasks-backup/gnupg \
--recipient "${fingerprint}!" --output "$incoming" --encrypt
[ -s "$incoming" ]
mv -f -- "$incoming" "$destination"
incoming=""
rm -f -- "$backup" "$backup.manifest.json"
backup=""
mapfile -t backups < <(find "$directory" -maxdepth 1 -type f -name 'akurai-tasks-*.tar.gpg' -printf '%f\n' | LC_ALL=C sort)
excess=$((${#backups[@]} - retention))
if [ "$excess" -gt 0 ]; then
for ((index = 0; index < excess; index++)); do
rm -- "$directory/${backups[$index]}"
done
fi
SCRIPT
sudo chown root:root /usr/local/libexec/akurai-tasks-backup
sudo chmod 0755 /usr/local/libexec/akurai-tasks-backup
sudo tee /etc/systemd/system/akurai-tasks-backup.service >/dev/null <<UNIT
[Unit]
Description=Create a verified encrypted AkurAI-Tasks database backup
After=akurai-tasks.service
Requires=akurai-tasks.service
[Service]
Type=oneshot
User=root
Group=root
Environment=AKURAI_TASKS_BACKUP_RETENTION=$retention
ExecStart=/usr/local/libexec/akurai-tasks-backup
UMask=0077
NoNewPrivileges=true
PrivateDevices=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadOnlyPaths=/opt/akurai-tasks /etc/akurai-tasks
ReadWritePaths=/var/backups/akurai-tasks /etc/akurai-tasks-backup/gnupg
CapabilityBoundingSet=
LockPersonality=true
MemoryDenyWriteExecute=true
RestrictSUIDSGID=true
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
[Install]
WantedBy=multi-user.target
UNIT
sudo tee /etc/systemd/system/akurai-tasks-backup.timer >/dev/null <<'UNIT'
[Unit]
Description=Hourly encrypted AkurAI-Tasks database backup
[Timer]
OnCalendar=hourly
RandomizedDelaySec=5m
Persistent=true
Unit=akurai-tasks-backup.service
[Install]
WantedBy=timers.target
UNIT
sudo systemctl daemon-reload
sudo systemctl enable --now akurai-tasks-backup.timer
sudo systemctl start akurai-tasks-backup.service
REMOTE
elif [ "$action" = run ]; then
say "running an encrypted AkurAI-Tasks backup"
ssh_run "sudo systemctl start akurai-tasks-backup.service"
fi
say "verifying encrypted AkurAI-Tasks backup and isolated recovery"
ssh_run 'bash -s' <<'REMOTE'
set -euo pipefail
sudo systemctl is-enabled --quiet akurai-tasks-backup.timer
sudo systemctl is-active --quiet akurai-tasks-backup.timer
sudo systemctl show akurai-tasks-backup.service -p Result --value | grep -qx success
latest="$(sudo find /var/backups/akurai-tasks -maxdepth 1 -type f -name 'akurai-tasks-*.tar.gpg' -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-)"
[ -n "$latest" ] && sudo test -s "$latest"
[ "$(sudo stat -c %a "$latest")" = 600 ]
[ "$(sudo stat -c %U:%G "$latest")" = root:root ]
header="$(sudo od -An -N16 -tx1 "$latest" | tr -d ' \n')"
[ "$header" != 414b555241495f42545245455f563100 ]
scratch="$(sudo mktemp -d /var/backups/akurai-tasks/.verify-XXXXXXXX)"
cleanup() { sudo rm -rf "$scratch"; }
trap cleanup EXIT
sudo gpg --batch --homedir /etc/akurai-tasks-backup/gnupg --decrypt "$latest" |
sudo tar -xf - -C "$scratch"
backup="$(sudo find "$scratch" -maxdepth 1 -type f -name 'akurai-tasks-*.db' -print -quit)"
[ -n "$backup" ] && sudo test -s "$backup" && sudo test -s "$backup.manifest.json"
expected="$(sudo python3 - "$backup.manifest.json" <<'PY'
import json
import sys
print(json.load(open(sys.argv[1], encoding="utf-8"))["sha256"])
PY
)"
actual="$(sudo sha256sum "$backup" | cut -d' ' -f1)"
[ "$actual" = "$expected" ]
sudo /opt/akurai-tasks/bin/akurai-tasks --db "$scratch/restored.db" restore "$backup" >/dev/null
sudo /opt/akurai-tasks/bin/akurai-tasks --db "$scratch/restored.db" doctor >/dev/null
count="$(sudo find /var/backups/akurai-tasks -maxdepth 1 -type f -name 'akurai-tasks-*.tar.gpg' | wc -l)"
mtime="$(sudo stat -c %y "$latest" | cut -d. -f1)Z"
printf 'timer=active latest=%s backups=%s encryption=gpg recovery=verified\n' "$mtime" "$count"
REMOTE
;;
workbench-backup)
[ $# -le 2 ] || die "usage: akurai-ec2 workbench-backup <install|run|verify> [retention]"
backup_dispatch "Workbench" "${1:-verify}" "${2:-14}" 365
action="$BACKUP_ACTION"
retention="$BACKUP_RETENTION"
workbench_ctl="${OMP_WORKBENCH_CTL:-$HOME/.local/bin/titan-omp-workbenchctl}"
offsite_home="${OMP_WORKBENCH_OFFSITE_HOME:-$HOME/.local/share/omp-workbench/offsite}"
offsite_gnupg="$offsite_home/gnupg"
fingerprint_file="$offsite_home/recipient.fingerprint"
recipient="titan-omp-workbench-offsite@olibuijr.com"
service_path="$HOME/.config/systemd/user/omp-workbench-offsite-backup.service"
timer_path="$HOME/.config/systemd/user/omp-workbench-offsite-backup.timer"
if [ "$action" = install ]; then
say "installing EC2 encryption identity and backup destination"
printf -v recipient_q '%q' "$recipient"
public_key="$(ssh_run "bash -s -- $recipient_q" <<'REMOTE'
set -euo pipefail
recipient="$1"
sudo install -d -o root -g root -m 0700 /etc/omp-workbench-backup/gnupg
sudo install -d -o root -g root -m 0700 /var/backups/omp-workbench
if ! sudo gpg --batch --homedir /etc/omp-workbench-backup/gnupg --with-colons --list-keys "$recipient" 2>/dev/null | grep -q '^pub:'; then
# Unattended recovery intentionally uses a root-only keyring without a
# passphrase; storing an automation passphrase beside it adds no host boundary.
sudo gpg --batch --homedir /etc/omp-workbench-backup/gnupg --passphrase '' \
--quick-generate-key "$recipient" rsa3072 encrypt 0
fi
sudo gpg --batch --homedir /etc/omp-workbench-backup/gnupg --armor --export "$recipient"
REMOTE
)"
[[ "$public_key" == *"BEGIN PGP PUBLIC KEY BLOCK"* ]] || die "EC2 did not return a valid backup public key"
install -d -m 0700 "$offsite_home" "$offsite_gnupg"
public_key_file="$offsite_home/recipient.asc"
public_key_tmp="$(mktemp "$offsite_home/.recipient-XXXXXXXX.asc")"
printf '%s\n' "$public_key" >"$public_key_tmp"
chmod 0600 "$public_key_tmp"
mv -f "$public_key_tmp" "$public_key_file"
gpg --batch --homedir "$offsite_gnupg" --import "$public_key_file" >/dev/null 2>&1 || die "failed to import EC2 backup public key"
fingerprint="$(gpg --batch --homedir "$offsite_gnupg" --with-colons --fingerprint "$recipient" | awk -F: '$1 == "fpr" { print $10; exit }')"
[[ "$fingerprint" =~ ^[A-F0-9]{40}$ ]] || die "invalid EC2 backup key fingerprint"
printf '%s:6:\n' "$fingerprint" | gpg --batch --homedir "$offsite_gnupg" --import-ownertrust >/dev/null 2>&1
fingerprint_tmp="$(mktemp "$offsite_home/.fingerprint-XXXXXXXX")"
printf '%s\n' "$fingerprint" >"$fingerprint_tmp"
chmod 0600 "$fingerprint_tmp"
mv -f "$fingerprint_tmp" "$fingerprint_file"
unit_dir="$(dirname "$service_path")"
install -d -m 0700 "$unit_dir"
service_tmp="$(mktemp "$unit_dir/.omp-workbench-offsite-backup-service-XXXXXXXX")"
timer_tmp="$(mktemp "$unit_dir/.omp-workbench-offsite-backup-timer-XXXXXXXX")"
cleanup_units() { rm -f "$service_tmp" "$timer_tmp"; }
trap cleanup_units EXIT
cat >"$service_tmp" <<UNIT
[Unit]
Description=Create and send an encrypted Workbench backup to AkurAI EC2
After=network-online.target omp-workbench.service
Wants=network-online.target
[Service]
Type=oneshot
ExecStart=$SCRIPT_SOURCE workbench-backup run $retention
UMask=0077
NoNewPrivileges=true
[Install]
WantedBy=default.target
UNIT
cat >"$timer_tmp" <<'UNIT'
[Unit]
Description=Daily off-machine Workbench backup
[Timer]
OnCalendar=*-*-* 04:15:00 UTC
RandomizedDelaySec=15m
Persistent=true
Unit=omp-workbench-offsite-backup.service
[Install]
WantedBy=timers.target
UNIT
chmod 0600 "$service_tmp" "$timer_tmp"
mv -f "$service_tmp" "$service_path"
mv -f "$timer_tmp" "$timer_path"
trap - EXIT
systemctl --user daemon-reload
systemctl --user enable --now omp-workbench-offsite-backup.timer
systemctl --user start omp-workbench-offsite-backup.service
elif [ "$action" = run ]; then
[ -x "$workbench_ctl" ] || die "Workbench controller not found: $workbench_ctl"
[ -d "$offsite_gnupg" ] || die "offsite encryption identity is not installed"
say "creating and encrypting a Workbench snapshot"
snapshot="$("$workbench_ctl" backup)"
case "$snapshot" in
"$HOME"/.local/share/omp-workbench/backups/workbench-*.sqlite) ;;
*) die "Workbench controller returned an unexpected snapshot path" ;;
esac
[ -r "$fingerprint_file" ] || die "backup recipient fingerprint is not installed"
fingerprint="$(<"$fingerprint_file")"
[[ "$fingerprint" =~ ^[A-F0-9]{40}$ ]] || die "invalid backup recipient fingerprint"
ciphertext="$(mktemp "$offsite_home/.outgoing-XXXXXXXX.sqlite.gpg")"
remote_tmp=""
cleanup_upload() {
rm -f "$ciphertext"
if [ -n "$remote_tmp" ]; then
printf -v remote_tmp_q '%q' "$remote_tmp"
ssh_cmd -o ConnectTimeout=15 "$SSH_HOST" "rm -f -- $remote_tmp_q" >/dev/null 2>&1 || true
fi
}
trap cleanup_upload EXIT
gpg --batch --yes --homedir "$offsite_gnupg" \
--recipient "${fingerprint}!" --output "$ciphertext" --encrypt "$snapshot"
[ -s "$ciphertext" ] || die "encrypted backup is empty"
header="$(od -An -N16 -tx1 "$ciphertext" | tr -d ' \n')"
[ "$header" != 53514c69746520666f726d6174203300 ] || die "backup was not encrypted"
name="workbench-$(date -u +%Y%m%dT%H%M%S)-$(date +%N).sqlite.gpg"
remote_tmp="$(ssh_run 'find /tmp -maxdepth 1 -user "$USER" -type f -name "omp-workbench-backup-*.gpg" -mmin +1440 -delete; mktemp /tmp/omp-workbench-backup-XXXXXXXX.gpg')"
scp_cmd -q -o ConnectTimeout=15 "$ciphertext" "$SSH_HOST:$remote_tmp"
printf -v remote_tmp_q '%q' "$remote_tmp"
printf -v name_q '%q' "$name"
ssh_run "bash -s -- $remote_tmp_q $name_q $retention" <<'REMOTE'
set -euo pipefail
source_path="$1"
name="$2"
retention="$3"
case "$source_path" in /tmp/omp-workbench-backup-*.gpg) ;; *) exit 1 ;; esac
[[ "$name" =~ ^workbench-[0-9]{8}T[0-9]{6}-[0-9]+\.sqlite\.gpg$ ]]
[[ "$retention" =~ ^[0-9]+$ ]] && [ "$retention" -ge 2 ] && [ "$retention" -le 365 ]
directory=/var/backups/omp-workbench
sudo install -d -o root -g root -m 0700 "$directory"
scratch="$(sudo mktemp -d "$directory/.verify-XXXXXXXX")"
incoming="$directory/.incoming-$name"
cleanup() {
rm -f "$source_path"
sudo rm -rf "$scratch" "$incoming"
}
trap cleanup EXIT
sudo gpg --batch --homedir /etc/omp-workbench-backup/gnupg \
--output "$scratch/restored.sqlite" --decrypt "$source_path" >/dev/null 2>&1
[ "$(sudo sqlite3 "$scratch/restored.sqlite" 'PRAGMA quick_check;')" = ok ]
sudo install -o root -g root -m 0600 "$source_path" "$incoming"
sudo mv -f "$incoming" "$directory/$name"
mapfile -t backups < <(sudo find "$directory" -maxdepth 1 -type f -name 'workbench-*.sqlite.gpg' -printf '%f\n' | LC_ALL=C sort)
excess=$((${#backups[@]} - retention))
if [ "$excess" -gt 0 ]; then
for ((index = 0; index < excess; index++)); do
sudo rm -- "$directory/${backups[$index]}"
done
fi
REMOTE
remote_tmp=""
trap - EXIT
cleanup_upload
fi
say "verifying off-machine Workbench backup and isolated restore"
if [ "$action" != run ]; then
systemctl --user is-enabled --quiet omp-workbench-offsite-backup.timer
systemctl --user is-active --quiet omp-workbench-offsite-backup.timer
systemctl --user show omp-workbench-offsite-backup.service -p Result --value | grep -qx success
fi
ssh_run "bash -s -- $retention" <<'REMOTE'
set -euo pipefail
retention="$1"
[[ "$retention" =~ ^[0-9]+$ ]] && [ "$retention" -ge 2 ] && [ "$retention" -le 365 ]
[ "$(sudo stat -c %a /etc/omp-workbench-backup/gnupg)" = 700 ]
[ "$(sudo stat -c %U:%G /etc/omp-workbench-backup/gnupg)" = root:root ]
sudo gpg --batch --homedir /etc/omp-workbench-backup/gnupg --with-colons --list-secret-keys \
titan-omp-workbench-offsite@olibuijr.com 2>/dev/null | grep -q '^sec:'
latest="$(sudo find /var/backups/omp-workbench -maxdepth 1 -type f -name 'workbench-*.sqlite.gpg' -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-)"
[ -n "$latest" ] && sudo test -s "$latest"
scratch="$(sudo mktemp -d /var/backups/omp-workbench/.verify-XXXXXXXX)"
cleanup() { sudo rm -rf "$scratch"; }
trap cleanup EXIT
sudo install -o root -g root -m 0600 "$latest" "$scratch/candidate.sqlite.gpg"
[ "$(sudo stat -c %a "$scratch/candidate.sqlite.gpg")" = 600 ]
[ "$(sudo stat -c %U:%G "$scratch/candidate.sqlite.gpg")" = root:root ]
header="$(sudo od -An -N16 -tx1 "$scratch/candidate.sqlite.gpg" | tr -d ' \n')"
[ "$header" != 53514c69746520666f726d6174203300 ]
sudo gpg --batch --homedir /etc/omp-workbench-backup/gnupg \
--output "$scratch/restored.sqlite" --decrypt "$scratch/candidate.sqlite.gpg" >/dev/null 2>&1
[ "$(sudo sqlite3 "$scratch/restored.sqlite" 'PRAGMA quick_check;')" = ok ]
count="$(sudo find /var/backups/omp-workbench -maxdepth 1 -type f -name 'workbench-*.sqlite.gpg' | wc -l)"
[ "$count" -le "$retention" ]
mtime="$(sudo stat -c %y "$latest" | cut -d. -f1)Z"
printf 'timer=active latest=%s backups=%s encryption=gpg recovery=verified\n' "$mtime" "$count"
REMOTE
;;
bunfork-monitor)
action="${1:-verify}"
domain="${2:-bunfork.olibuijr.com}"
warn_days="${3:-30}"
recipient="${4:-olibuijr@olibuijr.com}"
[ $# -le 4 ] || die "usage: akurai-ec2 bunfork-monitor <install|run|verify> [domain] [warn-days] [recipient]"
case "$action" in install|run|verify) ;; *) die "monitor action must be install, run, or verify" ;; esac
[[ "$domain" =~ ^[A-Za-z0-9.-]+$ ]] || die "invalid domain: $domain"
[[ "$warn_days" =~ ^[0-9]+$ ]] && [ "$warn_days" -ge 1 ] && [ "$warn_days" -le 3650 ] || \
die "TLS warning threshold must be from 1 through 3650 days"
[[ "$recipient" =~ ^[A-Za-z0-9.!#$%\&\'*+/=?^_\`{|}~-]+@[A-Za-z0-9.-]+$ ]] || die "invalid alert recipient"
if [ "$action" = install ]; then
say "installing Bunfork readiness and TLS monitor"
ssh_run "bash -s -- '$domain' '$warn_days' '$recipient'" <<'REMOTE'
set -euo pipefail
domain="$1"; warn_days="$2"; recipient="$3"
sudo install -d -o root -g root -m 0755 /usr/local/libexec
sudo install -d -o bunfork -g bunfork -m 0700 /var/lib/bunfork-monitor
sudo tee /usr/local/libexec/bunfork-monitor >/dev/null <<'SCRIPT'
#!/usr/bin/env bash
set -euo pipefail
domain="${BUNFORK_MONITOR_DOMAIN:?}"
warn_days="${BUNFORK_TLS_WARN_DAYS:?}"
curl --fail --silent --show-error --output /dev/null --max-time 15 \
"https://$domain/_bunfork/ready"
python3 - "$domain" "$warn_days" <<'PY'
import datetime, socket, ssl, sys
host, warning_days = sys.argv[1], int(sys.argv[2])
context = ssl.create_default_context()
with socket.create_connection((host, 443), timeout=10) as raw:
with context.wrap_socket(raw, server_hostname=host) as tls:
certificate = tls.getpeercert()
expiry = datetime.datetime.strptime(
certificate["notAfter"], "%b %d %H:%M:%S %Y %Z"
).replace(tzinfo=datetime.timezone.utc)
remaining = int((expiry - datetime.datetime.now(datetime.timezone.utc)).total_seconds() // 86400)
if remaining < warning_days:
raise SystemExit(f"TLS certificate has {remaining} days remaining; threshold is {warning_days}")
print(f"https_ready=true tls_hostname_verified=true tls_days_remaining={remaining}")
PY
rm -f /var/lib/bunfork-monitor/last-alert
SCRIPT
sudo tee /usr/local/libexec/bunfork-monitor-alert >/dev/null <<'SCRIPT'
#!/usr/bin/env bash
set -euo pipefail
umask 0077
unit="${1:-bunfork-monitor.service}"
recipient="${BUNFORK_MONITOR_RECIPIENT:?}"
state=/var/lib/bunfork-monitor/last-alert
now="$(date +%s)"
last=0
[ ! -e "$state" ] || last="$(stat -c %Y "$state")"
if [ $((now - last)) -lt 21600 ]; then
exit 0
fi
{
printf 'To: %s\n' "$recipient"
printf 'From: Bunfork monitor <root@mail.olibuijr.com>\n'
printf 'Subject: Bunfork production monitor failed\n'
printf 'Content-Type: text/plain; charset=UTF-8\n\n'
printf 'Bunfork production readiness or TLS verification failed.\n'
printf 'Unit: %s\n' "$unit"
printf 'Host: bunfork.olibuijr.com\n'
printf 'Inspect with: akurai-ec2 logs bunfork-monitor\n'
} | /usr/sbin/sendmail -t
touch "$state"
SCRIPT
sudo chown root:root /usr/local/libexec/bunfork-monitor /usr/local/libexec/bunfork-monitor-alert
sudo chmod 0755 /usr/local/libexec/bunfork-monitor /usr/local/libexec/bunfork-monitor-alert
sudo tee /etc/systemd/system/bunfork-monitor.service >/dev/null <<UNIT
[Unit]
Description=Verify Bunfork HTTPS readiness and TLS lifetime
After=network-online.target
Wants=network-online.target
OnFailure=bunfork-monitor-alert@%n.service
[Service]
Type=oneshot
User=bunfork
Group=bunfork
Environment=BUNFORK_MONITOR_DOMAIN=$domain
Environment=BUNFORK_TLS_WARN_DAYS=$warn_days
ExecStart=/usr/local/libexec/bunfork-monitor
UMask=0077
NoNewPrivileges=true
PrivateDevices=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/var/lib/bunfork-monitor
CapabilityBoundingSet=
LockPersonality=true
MemoryDenyWriteExecute=true
RestrictSUIDSGID=true
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
[Install]
WantedBy=multi-user.target
UNIT
sudo tee /etc/systemd/system/bunfork-monitor-alert@.service >/dev/null <<UNIT
[Unit]
Description=Send bounded Bunfork monitor failure alert for %i
[Service]
Type=oneshot
User=bunfork
Group=bunfork
Environment=BUNFORK_MONITOR_RECIPIENT=$recipient
ExecStart=/usr/local/libexec/bunfork-monitor-alert %i
UMask=0077
NoNewPrivileges=true
PrivateDevices=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/var/lib/bunfork-monitor
CapabilityBoundingSet=
LockPersonality=true
MemoryDenyWriteExecute=true
RestrictSUIDSGID=true
RestrictAddressFamilies=AF_UNIX
UNIT
sudo tee /etc/systemd/system/bunfork-monitor.timer >/dev/null <<'UNIT'
[Unit]
Description=Frequent Bunfork production readiness and TLS check
[Timer]
OnBootSec=2m
OnUnitActiveSec=15m
RandomizedDelaySec=1m
Persistent=true
Unit=bunfork-monitor.service
[Install]
WantedBy=timers.target
UNIT
sudo systemctl daemon-reload
sudo systemd-analyze verify /etc/systemd/system/bunfork-monitor.service \
/etc/systemd/system/bunfork-monitor-alert@.service \
/etc/systemd/system/bunfork-monitor.timer
sudo systemctl enable --now bunfork-monitor.timer
sudo systemctl start bunfork-monitor.service
REMOTE
elif [ "$action" = run ]; then
say "running Bunfork production monitor"
ssh_run "sudo systemctl start bunfork-monitor.service"
fi
say "verifying Bunfork production monitor"
ssh_run "bash -s -- '$domain' '$warn_days'" <<'REMOTE'
set -euo pipefail
domain="$1"; warn_days="$2"
sudo systemctl is-enabled --quiet bunfork-monitor.timer
sudo systemctl is-active --quiet bunfork-monitor.timer
sudo systemctl show bunfork-monitor.service -p Result --value | grep -qx success
sudo systemd-analyze verify /etc/systemd/system/bunfork-monitor.service \
/etc/systemd/system/bunfork-monitor-alert@.service \
/etc/systemd/system/bunfork-monitor.timer
[ "$(sudo systemctl show bunfork-monitor.service -p ProtectSystem --value)" = strict ]
[ "$(sudo systemctl show bunfork-monitor.service -p NoNewPrivileges --value)" = yes ]
[ "$(sudo systemctl show bunfork-monitor.service -p User --value)" = bunfork ]
sudo systemctl show bunfork-monitor.service -p OnFailure --value | \
grep -qx bunfork-monitor-alert@bunfork-monitor.service.service
sudo systemctl show bunfork-monitor.timer -p TimersMonotonic --value | \
grep -q 'OnUnitActiveUSec=15min'
sudo -u bunfork env BUNFORK_MONITOR_DOMAIN="$domain" BUNFORK_TLS_WARN_DAYS="$warn_days" \
/usr/local/libexec/bunfork-monitor
if sudo -u bunfork env BUNFORK_MONITOR_DOMAIN="$domain" BUNFORK_TLS_WARN_DAYS=3650 \
/usr/local/libexec/bunfork-monitor >/dev/null 2>&1; then
echo "TLS threshold failure test unexpectedly passed" >&2
exit 1
fi
sudo -u bunfork env BUNFORK_MONITOR_DOMAIN="$domain" BUNFORK_TLS_WARN_DAYS="$warn_days" \
/usr/local/libexec/bunfork-monitor >/dev/null
next="$(systemctl show bunfork-monitor.timer -p NextElapseUSecMonotonic --value)"
[ -n "$next" ]
printf 'timer=active next=%s readiness=verified tls_threshold_days=%s\n' "$next" "$warn_days"
REMOTE
;;
deploy-bunfork)
[ $# -ge 1 ] && [ $# -le 3 ] || die "usage: akurai-ec2 deploy-bunfork <bundle> [domain] [port]"
bundle="$(readlink -m "$1")"
domain="${2:-bunfork.olibuijr.com}"
port="${3:-3100}"
[ -d "$bundle" ] || die "bundle directory not found: $bundle"
[ -x "$bundle/bunfork" ] || die "bundle binary is missing or not executable: $bundle/bunfork"
[ -f "$bundle/bunfork.json" ] || die "bundle manifest missing: $bundle/bunfork.json"
[[ "$domain" =~ ^[A-Za-z0-9.-]+$ ]] || die "invalid domain: $domain"
[[ "$port" =~ ^[0-9]+$ ]] && [ "$port" -ge 1024 ] && [ "$port" -le 65535 ] || die "invalid port: $port"
verify_bunfork_bundle "$bundle"
release_id="$(date -u +%Y%m%dT%H%M%SZ)-$(sha256sum "$bundle/bunfork.json" | cut -c1-12)"
archive="$(mktemp --suffix=.tgz)"
trap 'rm -f "$archive"' EXIT
tar -C "$bundle" -czf "$archive" .
say "preflighting EC2 SSH"
ssh_run true
remote_archive="/tmp/bunfork-$release_id.tgz"
scp_cmd -o ConnectTimeout=15 "$archive" "$SSH_HOST:$remote_archive"
rm -f "$archive"
trap - EXIT
say "installing Bunfork release $release_id"
ssh_run "bash -s -- '$release_id' '$port' '$domain' '$remote_archive'" <<'REMOTE'
set -euo pipefail
release_id="$1"; port="$2"; domain="$3"; archive="$4"
base=/opt/bunfork
release="$base/releases/$release_id"
previous="$(readlink -f "$base/current" 2>/dev/null || true)"
sudo install -d -m 0755 "$base/releases" "$release"
sudo tar -xzf "$archive" -C "$release"
rm -f "$archive"
sudo python3 - "$release" <<'PY'
import hashlib, json, os, pathlib, stat, sys
root = pathlib.Path(sys.argv[1])
manifest = json.loads((root / "bunfork.json").read_text())
if manifest.get("schema") != "bunfork-deployment-v2":
raise SystemExit("invalid Bunfork deployment schema")
expected = set(manifest.get("files", {})) | {"bunfork.json"}
actual = set()
for path in root.rglob("*"):
rel = path.relative_to(root).as_posix()
if path.is_symlink() or (path.exists() and not (path.is_file() or path.is_dir())):
raise SystemExit(f"unsafe bundle entry: {rel}")
if path.is_file():
actual.add(rel)
if actual != expected:
raise SystemExit(f"bundle inventory mismatch: missing={sorted(expected-actual)} extra={sorted(actual-expected)}")
for rel, record in manifest["files"].items():
path = root / rel
data = path.read_bytes()
if len(data) != record["bytes"] or hashlib.sha256(data).hexdigest() != record["sha256"]:
raise SystemExit(f"bundle digest mismatch: {rel}")
PY
sudo chmod 0755 "$release/bunfork"
sudo find "$release" -type d -exec chmod 0755 {} +
sudo find "$release" -type f ! -name bunfork -exec chmod 0644 {} +
sudo chown -R root:root "$release"
if ! id bunfork >/dev/null 2>&1; then
sudo useradd --system --home-dir /var/lib/bunfork --shell /usr/sbin/nologin bunfork
fi
sudo install -d -o bunfork -g bunfork -m 0750 /var/lib/bunfork /etc/bunfork
if ! sudo test -s /etc/bunfork/db.key; then
sudo -u bunfork "$release/bunfork" keygen --out /etc/bunfork/db.key
fi
if ! sudo test -s /etc/bunfork/api.token; then
sudo -u bunfork "$release/bunfork" keygen --out /etc/bunfork/api.token
fi
sudo chmod 0600 /etc/bunfork/db.key /etc/bunfork/api.token
sudo chown bunfork:bunfork /etc/bunfork/db.key /etc/bunfork/api.token
sudo systemctl stop bunfork.service 2>/dev/null || true
if ! sudo -u bunfork "$release/bunfork" \
--database /var/lib/bunfork/bunfork.db \
--key-file /etc/bunfork/db.key migrate; then
if [ -n "$previous" ] && [ -d "$previous" ]; then
sudo systemctl start bunfork.service || true
fi
echo "Bunfork migration failed; existing release restarted" >&2
exit 1
fi
sudo ln -sfn "$release" "$base/current"
sudo tee /etc/systemd/system/bunfork.service >/dev/null <<UNIT
[Unit]
Description=Bunfork web runtime
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=bunfork
Group=bunfork
WorkingDirectory=/opt/bunfork/current
Environment=BUNFORK_ADDRESS=127.0.0.1:$port
Environment=BUNFORK_PUBLIC_ORIGIN=https://$domain
ExecStart=/opt/bunfork/current/bunfork --database /var/lib/bunfork/bunfork.db --key-file /etc/bunfork/db.key serve --token-file /etc/bunfork/api.token
Restart=on-failure
RestartSec=2
UMask=0077
NoNewPrivileges=true
PrivateDevices=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadOnlyPaths=/opt/bunfork /etc/bunfork
ReadWritePaths=/var/lib/bunfork
CapabilityBoundingSet=
LockPersonality=true
MemoryDenyWriteExecute=true
RestrictSUIDSGID=true
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
[Install]
WantedBy=multi-user.target
UNIT
sudo systemctl daemon-reload
sudo systemctl enable --now bunfork.service
healthy=0
for _ in 1 2 3 4 5 6 7 8 9 10; do
if curl -fsS --max-time 4 "http://127.0.0.1:$port/_bunfork/ready" >/dev/null 2>&1; then
healthy=1
break
fi
sleep 2
done
if [ "$healthy" != 1 ]; then
sudo systemctl stop bunfork.service || true
if [ -n "$previous" ] && [ -d "$previous" ]; then
sudo ln -sfn "$previous" "$base/current"
sudo systemctl start bunfork.service || true
fi
echo "Bunfork readiness failed; restored previous release when available" >&2
exit 1
fi
sudo systemctl is-active --quiet bunfork.service
printf 'release=%s previous=%s\n' "$release" "${previous:-none}"
REMOTE
had_cert=0
ssh_run "sudo test -f /etc/letsencrypt/live/$domain/fullchain.pem" && had_cert=1
"$0" nginx-proxy "$domain" "$port"
if [ "$had_cert" != 1 ]; then "$0" tls "$domain"; fi
curl -fsS --max-time 20 "https://$domain/_bunfork/ready" >/dev/null || die "public HTTPS readiness failed: $domain"
say "deployed Bunfork: https://$domain (systemd bunfork.service, release $release_id)"
;;
nginx-proxy)
[ $# -eq 2 ] || die "usage: akurai-ec2 nginx-proxy <domain> <port>"
domain="$1"; port="$2"
say "nginx reverse proxy: $domain → 127.0.0.1:$port"
ssh_run "sudo tee /etc/nginx/sites-available/$domain >/dev/null" <<EOF
# Map the client's Upgrade header to the Connection value nginx must forward for
# a WebSocket handshake: "upgrade" when upgrading, "close" otherwise.
map \$http_upgrade \$connection_upgrade {
default upgrade;
'' close;
}
server {
listen 80;
listen [::]:80;
server_name $domain;
location / {
proxy_pass http://127.0.0.1:$port;
proxy_http_version 1.1;
proxy_set_header Host \$host;
proxy_set_header X-Real-IP \$remote_addr;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto \$scheme;
# WebSocket upgrade + long-lived realtime streams (WS / SSE).
proxy_set_header Upgrade \$http_upgrade;
proxy_set_header Connection \$connection_upgrade;
proxy_read_timeout 3600s;
proxy_send_timeout 3600s;
proxy_buffering off; # stream SSE/WS frames immediately
}
}
EOF
ssh_run "sudo ln -sf /etc/nginx/sites-available/$domain /etc/nginx/sites-enabled/$domain && sudo nginx -t && sudo systemctl reload nginx"
say "nginx vhost live for $domain (HTTP). Public IP: $PUBLIC_IP"
# TLS idempotency: the HTTP vhost written above clobbers any certbot-managed
# 443 block, which is why every redeploy used to silently drop HTTPS. If a
# cert already exists for this domain, re-assert HTTPS so it never regresses.
if ssh_run "sudo test -f /etc/letsencrypt/live/$domain/fullchain.pem"; then
say "existing cert found — re-asserting HTTPS for $domain"
ssh_run "sudo certbot --nginx -d $domain --non-interactive --agree-tos -m olibuijr@olibuijr.com --redirect >/dev/null 2>&1 && sudo nginx -t && sudo systemctl reload nginx"
say "HTTPS re-asserted for $domain"
fi
;;
cert-status)
domain="mail.olibuijr.com"; warn_days=30; json=0
if [ $# -gt 0 ] && [[ "$1" != --* ]]; then domain="$1"; shift; fi
while [ $# -gt 0 ]; do
case "$1" in
--warn-days) [ $# -ge 2 ] || die "--warn-days requires a value"; warn_days="$2"; shift 2 ;;
--json) json=1; shift ;;
*) die "usage: akurai-ec2 cert-status [domain] [--warn-days N] [--json]" ;;
esac
done
[[ "$domain" =~ ^[A-Za-z0-9.-]+$ ]] || die "invalid domain: $domain"
[[ "$warn_days" =~ ^[0-9]+$ ]] && [ "$warn_days" -le 3650 ] || die "warn-days must be 0..3650"
python3 - "$domain" "$warn_days" "$json" <<'PY'
import datetime, json, socket, ssl, sys
host, warn, emit_json = sys.argv[1], int(sys.argv[2]), sys.argv[3] == "1"
try:
context = ssl.create_default_context()
with socket.create_connection((host, 443), timeout=10) as raw:
with context.wrap_socket(raw, server_hostname=host) as tls:
cert = tls.getpeercert()
expiry = datetime.datetime.strptime(cert["notAfter"], "%b %d %H:%M:%S %Y %Z").replace(tzinfo=datetime.timezone.utc)
now = datetime.datetime.now(datetime.timezone.utc)
days = max(0, int((expiry - now).total_seconds() // 86400))
sans = [value for kind, value in cert.get("subjectAltName", []) if kind == "DNS"]
state = "healthy" if days >= warn else "warning"
result = {"domain": host, "state": state, "hostname_verified": True, "days_remaining": days, "not_after": expiry.isoformat(), "dns_sans": sans}
if emit_json:
print(json.dumps(result, separators=(",", ":")))
else:
print(f"{host}: {state}; {days} days remaining; expires {expiry.isoformat()}")
raise SystemExit(0 if state == "healthy" else 1)
except (OSError, ssl.SSLError, KeyError, ValueError) as exc:
result = {"domain": host, "state": "error", "hostname_verified": False, "error": str(exc)}
print(json.dumps(result, separators=(",", ":")) if emit_json else f"{host}: error: {exc}")
raise SystemExit(2)
PY
;;
tls)
[ $# -eq 1 ] || die "usage: akurai-ec2 tls <domain>"
domain="$1"
say "requesting Let's Encrypt cert for $domain"
ssh_run "sudo certbot --nginx -d $domain --non-interactive --agree-tos -m olibuijr@olibuijr.com --redirect"
say "verifying public TLS and hostname"
"$0" cert-status "$domain" --warn-days 0
;;
release)
# release [patch|minor|major|publish|ec2] [flags] — unified release engine.
#
# Reads ./akurai-deploy.toml (or -C <dir>) and runs the whole release ONCE,
# here, instead of in each project's bespoke deploy.sh:
# gate (fmt/clippy/test) → bump VERSION + Cargo.toml (lockstep, keyed on
# the old value) → changelog ([Unreleased] + generated draft) →
# git commit/tag → sync the trusted checkout to AkurAI Build →
# deploy-binary + nginx-proxy + tls.
# `--mode publish|update` (legacy: `publish`, `ec2`) ships the current
# binary with NO bump. `--mode release --bump patch|minor|major` is the
# canonical release form. --dry-run previews changes and writes nothing.
# --changelog "text" uses one-line text; --yes accepts the draft unattended.
# Manifest schema:
# v1 flat TOML stays supported as-is (name=, domain=, port=, build=, artifact=, ...)
# v2 nested TOML can use dotted groups such as app.name, build.command,
# deploy.port, release.gates.fmt, and service.exec_start.
SELF="$(command -v akurai-ec2 2>/dev/null || true)"; [ -n "$SELF" ] || SELF="$0"
mode=""; mode_source=""; bump=""; bump_source=""; dry=0; cl_text=""; have_cl_flag=0
assume_yes=0; skip_gates=0; workdir="."; emit_json=0
release_usage="release [patch|minor|major|publish|ec2] [--mode update|publish|release] [--bump patch|minor|major] [--dry-run] [--skip-gates] [--changelog TEXT] [--json] [--yes] [-C DIR]"
release_set_mode() {
local value="$1" source="$2"
if [ -n "$mode" ] && [ "$mode" != "$value" ]; then
die "release: conflicting mode values ($mode_source=$mode, $source=$value)"
fi
mode="$value"
mode_source="$source"
}
release_set_bump() {
local value="$1" source="$2"
if [ -n "$bump" ] && [ "$bump" != "$value" ]; then
die "release: conflicting bump values ($bump_source=$bump, $source=$value)"
fi
bump="$value"
bump_source="$source"
}
release_apply_mode_alias() {
local raw="$1" source="$2"
case "$raw" in
update|publish|release) release_set_mode "$raw" "$source" ;;
ec2) release_set_mode "publish" "$source" ;;
patch|minor|major) release_set_mode "release" "$source"; release_set_bump "$raw" "$source" ;;
*) die "manifest: unsupported release default '$raw'" ;;
esac
}
while [ $# -gt 0 ]; do
case "$1" in
patch|minor|major) release_set_mode "release" "legacy '$1'"; release_set_bump "$1" "legacy '$1'" ;;
propagate) release_set_mode "propagate" "explicit" ;;
publish|ec2) release_apply_mode_alias "$1" "legacy '$1'" ;;
--mode)
shift || die "release: --mode requires a value (usage: $release_usage)"
[ $# -gt 0 ] || die "release: --mode requires a value (usage: $release_usage)"
case "$1" in
update|publish|release) release_set_mode "$1" "--mode" ;;
*) die "release: invalid --mode '$1' (expected: update, publish, release)" ;;
esac
;;
--bump)
shift || die "release: --bump requires a value (usage: $release_usage)"
[ $# -gt 0 ] || die "release: --bump requires a value (usage: $release_usage)"
case "$1" in
patch|minor|major) release_set_bump "$1" "--bump" ;;
*) die "release: invalid --bump '$1' (expected: patch, minor, major)" ;;
esac
;;
--dry-run) dry=1 ;;
--skip-gates) skip_gates=1 ;;
--yes|-y) assume_yes=1 ;;
--json) emit_json=1 ;;
--changelog) shift; cl_text="${1:-}"; have_cl_flag=1 ;;
-C) shift; workdir="${1:-.}" ;;
*) die "release: bad arg '$1' (usage: $release_usage)" ;;
esac
shift || true
done
cd "$workdir" || die "cannot cd to $workdir"
[ -f akurai-deploy.toml ] || die "no akurai-deploy.toml in $(pwd) — every app needs one (schema in this command's header / the _AWSEC2 'release' workflow)"
release_load_manifest "akurai-deploy.toml"
name="$(mf_require "name (systemd unit)" name app.name)"
retired="$(mf_get retired app.retired || true)"
[ -z "$retired" ] || die "release: $name is retired on EC2: $retired"
domain="$(mf_get domain app.domain || true)"; port="$(mf_get port deploy.port || true)"
buildcmd="$(mf_get build build.command || true)"; artifact="$(mf_get artifact build.artifact || true)"; appdir="$(mf_get appdir build.appdir || true)"
artifact="${artifact/#\~/$HOME}" # allow ~ in artifact path (e.g. shared Framework binary)
if [ -n "${AKURAI_RELEASE_ARTIFACT:-}" ]; then
artifact="${AKURAI_RELEASE_ARTIFACT/#\~/$HOME}"
buildcmd=""
fi
expected_artifact_sha256="${AKURAI_RELEASE_SHA256:-}"
cli="$(mf_get cli deploy.cli || true)"
cli_target="$(mf_get cli_target deploy.cli_target || true)"
if [ -n "$cli" ]; then
cli_target="${cli_target:-/usr/local/bin/$(basename "$cli")}"
[[ "$cli_target" == /usr/local/bin/* ]] ||
die "manifest: cli_target must be below /usr/local/bin"
fi
vfile="$(mf_get version_file release.version_file || true)"; vfile="${vfile:-VERSION}"
cargover="$(mf_get cargo_version release.cargo_version || true)"; cargover="${cargover:-workspace}"
clfile="$(mf_get changelog release.changelog || true)"; clfile="${clfile:-CHANGELOG.md}"
clmode="$(mf_get changelog_mode release.changelog_mode || true)"; clmode="${clmode:-hybrid}"
g_fmt="$(mf_get_bool true gate_fmt release.gates.fmt)"
g_clippy="$(mf_get_bool true gate_clippy release.gates.clippy)"
g_test="$(mf_get_bool true gate_test release.gates.test)"
dbsnap="$(mf_get db_snapshot deploy.db_snapshot || true)"
dbkeep="$(mf_get db_keep deploy.db_keep || true)"; dbkeep="${dbkeep:-7}"
repo="$(mf_get repo app.repo || true)"
build_repo="$(mf_get build_repo app.build_repo || true)"
# Parsed directly (not via mf_get): hook bodies are often multi-line, and
# the generic flat/dotted loader's TSV serialization can't carry embedded
# newlines (each record is one tab-separated line).
posthook=""; prehook=""
if [ -f akurai-deploy.toml ]; then
posthook="$(python3 -c "
import tomllib
with open('akurai-deploy.toml', 'rb') as f:
cfg = tomllib.load(f)
print(cfg.get('hooks', {}).get('post_deploy', cfg.get('post_deploy', '')), end='')
" 2>/dev/null || true)"
prehook="$(python3 -c "
import tomllib
with open('akurai-deploy.toml', 'rb') as f:
cfg = tomllib.load(f)
print(cfg.get('hooks', {}).get('pre_deploy', cfg.get('pre_deploy', '')), end='')
" 2>/dev/null || true)"
fi
# custom-unit passthrough (non-Framework-convention apps) + default release mode
execstart="$(mf_get exec_start service.exec_start || true)"
envfile="$(mf_get env_file service.env_file || true)"
appuser="$(mf_get user service.user || true)"
udatadir_raw="$(mf_get data_dir service.data_dir || true)"
udatadir="$(mf_get_string_array_lines "service.data_dir" data_dir service.data_dir || true)"
harden="$(mf_get_bool false harden service.harden)"
mng_nginx="$(mf_get_bool true manage_nginx deploy.manage_nginx)" # false → leave the app's existing nginx vhost untouched on redeploy
defmode="$(mf_get release.default_mode release.default_action default_action || true)"
if [ -z "$mode" ] && [ -z "$bump" ] && [ -n "$defmode" ]; then
release_apply_mode_alias "$defmode" "manifest default"
fi
if [ -n "$bump" ] && [ -z "$mode" ]; then
release_set_mode "release" "implicit from bump"
fi
if [ -z "$mode" ]; then
mode="release"
mode_source="default"
fi
if [ "$mode" != "release" ] && [ -n "$bump" ]; then
die "release: --bump applies only to release mode (current mode: $mode)"
fi
if [ "$mode" = "release" ] && [ -z "$bump" ]; then
bump="patch"
bump_source="default"
fi
if [ "$dry" = 1 ]; then say "$mode: $name @ $(pwd) [DRY-RUN]"; else say "$mode: $name @ $(pwd)"; fi
musl_build() {
if [ -z "$buildcmd" ]; then
say "build: (none — using prebuilt artifact ${artifact:-<unset>})"
else
say "build: $buildcmd"
if [ "$dry" = 0 ]; then rustup target add x86_64-unknown-linux-musl >/dev/null 2>&1 || true; eval "$buildcmd" || die "build failed"; fi
fi
if [ "$dry" = 1 ]; then say " [dry-run] skipping build + artifact check"; return 0; fi
if [ -z "$artifact" ] || [ ! -f "$artifact" ]; then die "build artifact not found: ${artifact:-<unset>}"; fi
if [ -n "$expected_artifact_sha256" ]; then
actual_artifact_sha256="$(sha256sum "$artifact" | cut -d' ' -f1)"
[ "$actual_artifact_sha256" = "$expected_artifact_sha256" ] ||
die "build artifact digest mismatch: expected $expected_artifact_sha256, got $actual_artifact_sha256"
say "artifact: verified SHA-256 $actual_artifact_sha256"
fi
}
do_publish() {
musl_build
[ -n "$port" ] || die "manifest: 'port' required to publish"
if [ "$dry" = 1 ]; then
say " [dry-run] would: deploy-binary $name $artifact $port ${appdir:+$appdir}"
if [ -n "$cli" ]; then say " [dry-run] would install CLI $cli → $cli_target"; fi
if [ -n "$execstart" ]; then say " [dry-run] custom unit: ExecStart='$execstart' User='${appuser:-ubuntu}'${envfile:+ EnvironmentFile='$envfile'}${udatadir_raw:+ data='$udatadir_raw'}"; fi
if [ -n "$domain" ] && [ "$domain" != "-" ] && [ "$mng_nginx" = true ]; then say " [dry-run] would: nginx-proxy $domain $port; tls $domain"; elif [ -n "$domain" ] && [ "$domain" != "-" ]; then say " [dry-run] nginx: leaving existing $domain vhost untouched (manage_nginx=false)"; fi
deploy_extra_units
if [ -n "$posthook" ]; then say " [dry-run] would run post_deploy hook: $posthook"; fi
if [ -n "$prehook" ]; then say " [dry-run] would run pre_deploy hook: $prehook"; fi
return 0
fi
if [ -n "$prehook" ]; then
say "pre-deploy hook"
phook="${prehook//\{bin\}/\/opt\/$name\/bin\/$name}"; phook="${phook//\{port\}/$port}"; phook="${phook//\{app\}/\/opt\/$name\/app}"
"$SELF" ssh "$phook" || die "pre_deploy hook failed"
fi
if [ -n "$dbsnap" ]; then
say "db-snapshot: $dbsnap (keep $dbkeep)"
"$SELF" ssh "ts=\$(date +%Y%m%d-%H%M%S); sudo systemctl stop $name 2>/dev/null||true; sudo mkdir -p /var/backups/$name; [ -d '$dbsnap' ] && sudo cp -a '$dbsnap' /var/backups/$name/data-\$ts || true; sudo systemctl start $name 2>/dev/null||true; ls -1dt /var/backups/$name/data-* 2>/dev/null | tail -n +$((dbkeep+1)) | xargs -r sudo rm -rf" || say " snapshot warning (continuing)"
fi
deploy_args=("$name" "$artifact" "$port")
[ -z "$appdir" ] || deploy_args+=("$appdir")
[ "$emit_json" = 0 ] || deploy_args+=(--json)
AKURAI_EXEC_START="$execstart" AKURAI_ENV_FILE="$envfile" AKURAI_USER="$appuser" AKURAI_DATA_DIR="$udatadir" AKURAI_HARDEN="$harden" "$SELF" deploy-binary "${deploy_args[@]}"
if [ -n "$cli" ]; then
[ -f "$cli" ] || die "CLI artifact not found: $cli"
cli_tmp="/tmp/${name}-$(basename "$cli").$$"
say "installing CLI $(basename "$cli") → $cli_target"
"$SELF" ship "$cli" "$cli_tmp"
"$SELF" ssh "sudo install -o root -g root -m 755 '$cli_tmp' '$cli_target' && rm -f '$cli_tmp'" ||
die "CLI installation failed"
fi
if [ -n "$domain" ] && [ "$domain" != "-" ] && [ "$mng_nginx" = true ]; then
"$SELF" nginx-proxy "$domain" "$port"
"$SELF" tls "$domain" || say " TLS deferred — run 'akurai-ec2 tls $domain' once DNS resolves"
fi
deploy_extra_units
if [ -n "$posthook" ]; then
say "post-deploy hook"
hook="${posthook//\{bin\}/\/opt\/$name\/bin\/$name}"; hook="${hook//\{port\}/$port}"; hook="${hook//\{app\}/\/opt\/$name\/app}"
"$SELF" ssh "$hook" || die "post_deploy hook failed"
fi
if [ -n "$domain" ] && [ "$domain" != "-" ]; then say "published → https://$domain"; else say "published (internal: 127.0.0.1:$port)"; fi
if [ "$emit_json" = 1 ]; then
public_status=0; tls_json="null"
if [ -n "$domain" ] && [ "$domain" != "-" ]; then
public_status="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "https://$domain/" 2>/dev/null || printf 0)"
tls_json="$("$SELF" cert-status "$domain" --warn-days 0 --json 2>/dev/null || printf null)"
fi
RECEIPT_APP="$name" RECEIPT_DOMAIN="$domain" RECEIPT_PUBLIC="$public_status" RECEIPT_TLS="$tls_json" \
RECEIPT_MODE="$mode" python3 - <<'PY'
import json
import os
try:
tls = json.loads(os.environ["RECEIPT_TLS"])
except json.JSONDecodeError:
tls = None
print(json.dumps({
"app": os.environ["RECEIPT_APP"],
"domain": None if os.environ["RECEIPT_DOMAIN"] in ("", "-") else os.environ["RECEIPT_DOMAIN"],
"mode": os.environ["RECEIPT_MODE"],
"public_http_status": int(os.environ["RECEIPT_PUBLIC"]),
"tls": tls,
"ok": int(os.environ["RECEIPT_PUBLIC"]) > 0 or os.environ["RECEIPT_DOMAIN"] in ("", "-"),
}, separators=(",", ":")))
PY
fi
}
# extra_unit: additional systemd services deployed from the SAME artifact
# under a different name/port/domain (e.g. a companion MCP endpoint next
# to the main HTTP app). [[extra_unit]] is an array-of-tables, which the
# generic flat/dotted manifest loader can't express, so it's parsed here
# directly from akurai-deploy.toml instead of going through mf_get.
deploy_extra_units() {
[ -f akurai-deploy.toml ] || return 0
local units
units="$(python3 -c "
import tomllib
with open('akurai-deploy.toml', 'rb') as f:
cfg = tomllib.load(f)
for u in cfg.get('extra_unit', []):
print('\t'.join([
u.get('name', ''), str(u.get('port', '')), u.get('domain', '-'),
u.get('exec_start', ''), u.get('env_file', ''), u.get('user', 'ubuntu'),
]))
" 2>/dev/null || true)"
[ -n "$units" ] || return 0
while IFS=$'\t' read -r eu_name eu_port eu_domain eu_exec eu_env eu_user; do
[ -n "$eu_name" ] || continue
if [ "$dry" = 1 ]; then
say " [dry-run] would: deploy-binary $eu_name $artifact $eu_port (extra unit, shares $name's artifact)"
continue
fi
say "extra unit: $eu_name on :$eu_port"
AKURAI_EXEC_START="$eu_exec" AKURAI_ENV_FILE="$eu_env" AKURAI_USER="$eu_user" AKURAI_HARDEN="$harden" "$SELF" deploy-binary "$eu_name" "$artifact" "$eu_port"
if [ -n "$eu_domain" ] && [ "$eu_domain" != "-" ]; then
"$SELF" nginx-proxy "$eu_domain" "$eu_port"
"$SELF" tls "$eu_domain" || say " TLS deferred — run 'akurai-ec2 tls $eu_domain' once DNS resolves"
fi
done <<< "$units"
}
# Gates follow the project kind: cargo gates only in a Cargo project (a SvelteKit/Node app
# declaring gate_* used to fail or silently pass on cargo); package.json projects run their
# own lint/typecheck/check and test scripts. clippy/test cover the WHOLE workspace, with the
# committed lockfile, so a warning or failure in a non-root member cannot ship.
release_gates() {
if [ "$skip_gates" = 1 ]; then say "WARNING: --skip-gates: fmt/clippy/tests NOT run for this $mode"; return 0; fi
if [ -f Cargo.toml ]; then
if [ "$g_fmt" = true ]; then say "gate: rustfmt"; if [ "$dry" = 0 ]; then cargo fmt --all -- --check || die "fmt failed (run: cargo fmt --all)"; fi; fi
if [ "$g_clippy" = true ]; then say "gate: clippy -D warnings"; if [ "$dry" = 0 ]; then cargo clippy --workspace --all-targets --locked -- -D warnings || die "clippy failed"; fi; fi
if [ "$g_test" = true ]; then say "gate: tests"; if [ "$dry" = 0 ]; then cargo test --workspace --locked || die "tests failed"; fi; fi
elif [ -f package.json ]; then
local pm=pnpm s
if [ -f bun.lock ] || [ -f bun.lockb ]; then pm=bun; fi
[ "$dry" = 1 ] || command -v "$pm" >/dev/null || die "gate: $pm not installed (needed by this project's lockfile)"
node_has_script() { python3 -c 'import json,sys; sys.exit(0 if json.load(open("package.json")).get("scripts",{}).get(sys.argv[1]) else 1)' "$1"; }
if [ "$g_clippy" = true ]; then
for s in lint typecheck check; do
if node_has_script "$s"; then say "gate: $pm run $s"; if [ "$dry" = 0 ]; then "$pm" run "$s" || die "$s failed"; fi; fi
done
fi
if [ "$g_test" = true ]; then
node_has_script test || die "gate_test is true but package.json has no \"test\" script — add tests or set gate_test = false"
say "gate: $pm run test"; if [ "$dry" = 0 ]; then "$pm" run test || die "tests failed"; fi
fi
else
say "gate: skipped (no Cargo.toml or package.json)"
fi
}
if [ "$mode" = "propagate" ]; then
# propagate: build the Framework binary once, deploy to all apps that share it
APPS_TOML="$HOME/Projects/AkurAI-Framework/AKURAI_APPS.toml"
[ -f "$APPS_TOML" ] || die "AKURAI_APPS.toml not found at $APPS_TOML"
say "propagate: building Framework binary"
cd "$HOME/Projects/AkurAI-Framework"
g_fmt=true; g_clippy=true; g_test=true
release_gates
rustup target add x86_64-unknown-linux-musl >/dev/null 2>&1 || true
cargo build --release --target x86_64-unknown-linux-musl -p akurai || die "Framework build failed"
ARTIFACT="target/x86_64-unknown-linux-musl/release/akurai"
while IFS= read -r line; do
app_name=$(echo "$line" | cut -d: -f1)
app_domain=$(echo "$line" | cut -d: -f2)
app_port=$(echo "$line" | cut -d: -f3)
app_dir=$(echo "$line" | cut -d: -f4)
[ -z "$app_name" ] && continue
say "propagate: deploying $app_name → $app_domain:$app_port"
[ -n "$app_dir" ] && appdir_arg="$app_dir" || appdir_arg=""
"$SELF" deploy-binary "$app_name" "$ARTIFACT" "$app_port" $appdir_arg
"$SELF" nginx-proxy "$app_domain" "$app_port" 2>/dev/null || true
say "propagate: $app_name ✓"
done <<< "$(python3 -c "
import tomllib
with open('$APPS_TOML', 'rb') as f:
apps = tomllib.load(f)
for cfg in apps.get('app', []):
name = cfg.get('name', '?')
if cfg.get('default_action') == 'publish' or cfg.get('framework_based'):
print(f'{name}:{cfg.get(\"domain\",\"\")}:{cfg.get(\"port\",\"\")}:{cfg.get(\"appdir\",\"\")}')
" 2>/dev/null || true)"
say "propagate: done"
exit 0
fi
if [ "$mode" = "publish" ] || [ "$mode" = "update" ]; then
release_gates
do_publish
if [ "$dry" = 1 ]; then
say "$mode (dry-run): $name - nothing written, synchronized, or deployed"
else
say "$mode: $name ✓"
fi
exit 0
fi
release_gates
[ -f "$vfile" ] || die "no $vfile (manifest version_file)"
cur="$(tr -d '[:space:]' < "$vfile")"
IFS='.' read -r MA MI PA <<< "$cur"
case "$bump" in major) MA=$((MA+1)); MI=0; PA=0 ;; minor) MI=$((MI+1)); PA=0 ;; patch) PA=$((PA+1)) ;; esac
new="$MA.$MI.$PA"
say "version: $cur → $new ($bump)"
if [ "$dry" = 0 ]; then
printf '%s\n' "$new" > "$vfile"
if [ "$cargover" != none ] && [ -f Cargo.toml ]; then
sed -i -E "s/^version = \"$cur\"/version = \"$new\"/" Cargo.toml
# Sync Cargo.lock's workspace-member version to the bumped manifest BEFORE
# the release commit. Otherwise the build (which runs after the commit)
# regenerates Cargo.lock with the new version and leaves it dirty every
# release. --workspace touches only local members, --offline avoids network.
[ -f Cargo.lock ] && cargo update --workspace --offline >/dev/null 2>&1 || true
fi
fi
draft=".akurai-changelog.draft"
rm -f "$draft" 2>/dev/null || true
trap 'rm -f "$draft" 2>/dev/null || true' EXIT
unreleased_body() { [ -f "$clfile" ] || return 0; awk '/^## \[Unreleased\]$/{f=1;next} /^## \[/{f=0} f' "$clfile" | sed '/^[[:space:]]*$/d'; }
git_draft() {
local range="HEAD" lt feats fixes chgd any=0
lt="$(git describe --tags --abbrev=0 2>/dev/null || true)"; if [ -n "$lt" ]; then range="$lt..HEAD"; fi
feats="$(git log --no-merges --pretty=%s "$range" 2>/dev/null | sed -nE 's/^feat(\([^)]*\))?!?: /- /p' || true)"
fixes="$(git log --no-merges --pretty=%s "$range" 2>/dev/null | sed -nE 's/^fix(\([^)]*\))?!?: /- /p' || true)"
chgd="$(git log --no-merges --pretty=%s "$range" 2>/dev/null | sed -nE 's/^(refactor|perf|docs|chore|build|ci|style|test)(\([^)]*\))?!?: /- /p' || true)"
if [ -n "$feats" ]; then printf '### Added\n%s\n\n' "$feats"; any=1; fi
if [ -n "$fixes" ]; then printf '### Fixed\n%s\n\n' "$fixes"; any=1; fi
if [ -n "$chgd" ]; then printf '### Changed\n%s\n\n' "$chgd"; any=1; fi
if [ "$any" = 0 ]; then printf -- '- maintenance release\n'; fi
}
if [ "$have_cl_flag" = 1 ]; then
printf -- '- %s\n' "$cl_text" > "$draft"
elif [ "$clmode" = manual ]; then
printf -- '- \n' > "$draft"; if [ -t 0 ] && [ "$dry" = 0 ]; then "${EDITOR:-vi}" "$draft" || true; fi
else
{ mu="$(unreleased_body)"; if [ -n "$mu" ]; then printf '%s\n\n' "$mu"; fi; git_draft; } > "$draft"
fi
if [ "$clmode" = hybrid ] && [ "$have_cl_flag" = 0 ] && [ "$assume_yes" = 0 ] && [ "$dry" = 0 ] && [ -t 0 ]; then
say "draft changelog for [$new] (commits since $(git describe --tags --abbrev=0 2>/dev/null || echo start)):"
sed 's/^/ /' "$draft"
printf ' [e]dit / [a]ccept / [s]kip ? '; read -r ans || ans=a
case "${ans:-a}" in e|E) "${EDITOR:-vi}" "$draft" || true ;; s|S) : > "$draft" ;; esac
fi
if [ "$dry" = 1 ]; then
say "[dry-run] changelog [$new] would be:"; sed 's/^/ /' "$draft"
else
hdr="## [$new] - $(date +%Y-%m-%d)"
if [ ! -f "$clfile" ]; then
{ printf '# Changelog\n\nAll notable changes to this project are documented here. Format: Keep a Changelog; SemVer.\n\n## [Unreleased]\n\n%s\n\n' "$hdr"; cat "$draft"; printf '\n'; } > "$clfile"
elif grep -q '^## \[Unreleased\]$' "$clfile"; then
awk -v hdr="$hdr" -v bf="$draft" '/^## \[Unreleased\]$/{print "## [Unreleased]"; print ""; print hdr; print ""; while ((getline l < bf) > 0) print l; print ""; next} {print}' "$clfile" > "$clfile.x" && mv "$clfile.x" "$clfile"
else
{ printf '%s\n\n' "$hdr"; cat "$draft"; printf '\n'; cat "$clfile"; } > "$clfile.x" && mv "$clfile.x" "$clfile"
fi
say "changelog: cut [$new] into $clfile"
fi
rm -f "$draft" 2>/dev/null || true
if [ "$dry" = 0 ]; then
say "git: commit + tag v$new"
git add -A
git commit -q -m "release: v$new" || die "git commit failed (nothing to commit?)"
git tag -a "v$new" -m "v$new" || die "git tag v$new failed"
[ -n "$build_repo" ] || build_repo="${repo##*/}"
[ -n "$build_repo" ] || die "manifest: 'build_repo' required to publish the release commit"
build_deploy="$HOME/Projects/AkurAI-Build/deploy.sh"
[ -x "$build_deploy" ] || die "AkurAI Build deploy entrypoint missing: $build_deploy"
say "git: sync $build_repo to AkurAI Build"
"$build_deploy" cli repo sync "$build_repo" "$(pwd -P)" ||
die "AkurAI Build repository sync failed: $build_repo"
fi
do_publish
if [ "$dry" = 1 ]; then say "release (dry-run): $name → v$new — nothing written, synchronized, or deployed ✓"; else say "release: $name v$new ✓"; fi
;;
rollback)
# rollback <name> — restore the previous binary saved by deploy-binary.
[ $# -eq 1 ] || die "usage: akurai-ec2 rollback <name>"
name="$1"; base="/opt/$name"
ssh_run "[ -f $base/bin/$name.prev ]" || die "no $base/bin/$name.prev backup found for $name"
say "rolling back $name to its previous binary"
ssh_run "sudo systemctl stop $name.service; cp -p $base/bin/$name.prev $base/bin/$name; sudo systemctl start $name.service; sleep 2; systemctl is-active $name.service"
say "rolled back: $name"
;;
logs)
[ $# -eq 1 ] || die "usage: akurai-ec2 logs <service>"
ssh_run "journalctl -u $1 -n 80 --no-pager"
;;
mail-autoconfig)
# Email client auto-setup lives in nginx vhost 'mail-autoconfig' + snippets
# mail-autoconfig-<domain>.conf / mail-autodiscover.conf, payloads under
# /var/www/autoconfig/<domain>/, certs autoconfig.<domain> (+autodiscover SAN).
# Advertises mail.olibuijr.com IMAP 993 SSL / SMTP 587 STARTTLS for every domain.
action="${1:-verify}"; shift || true
case "$action" in
verify)
domains="olibuijr.com golfsetridak.is"
[ $# -gt 0 ] && domains="$*"
rc=0
for d in $domains; do
say "autoconfig: $d"
code=$(curl -s -o /dev/null -w '%{http_code}' "https://autoconfig.$d/mail/config-v1.1.xml" || echo 000)
printf ' mozilla autoconfig.%s/mail/config-v1.1.xml -> HTTP %s\n' "$d" "$code"; [ "$code" = 200 ] || rc=1
code=$(curl -s -o /dev/null -w '%{http_code}' -X POST "https://autodiscover.$d/autodiscover/autodiscover.xml" || echo 000)
printf ' ms autodiscover.%s (POST) -> HTTP %s\n' "$d" "$code"; [ "$code" = 200 ] || rc=1
for h in "autoconfig.$d" "autodiscover.$d"; do
if echo | timeout 10 openssl s_client -servername "$h" -connect "$h:443" 2>/dev/null | openssl x509 -noout -checkhost "$h" >/dev/null 2>&1; then
printf ' tls %s -> cert matches\n' "$h"
else printf ' tls %s -> MISMATCH/UNREACHABLE\n' "$h"; rc=1; fi
done
for s in _imaps._tcp _submission._tcp _autodiscover._tcp; do
ans=$(dig +short SRV "$s.$d" 2>/dev/null | tr '\n' ' ')
printf ' srv %s.%s -> %s\n' "$s" "$d" "${ans:-<none yet>}"
done
done
[ $rc -eq 0 ] && say "autoconfig OK (SRV may lag on 1984.is rebuild)" || die "autoconfig has failures (see above)"
;;
*) die "usage: akurai-ec2 mail-autoconfig verify [domain...]" ;;
esac
;;
allow-ssh)
# Oneshot: authorize an IP (default: this host's current public IP) for SSH
# (port 22) on the box's security group, then verify connectivity — in one
# command. The SG locks port 22 to specific /32s, so a dynamic-IP change (or a
# new host such as a Titan CI worker) times out SSH until its IP is re-added.
# Idempotent: an already-authorized IP is a no-op. `ssh_run` also calls
# `ensure_my_ip` automatically on a connection timeout.
# akurai-ec2 allow-ssh authorize this host's public IP + verify
# akurai-ec2 allow-ssh <ip> authorize an explicit IP + verify
ensure_my_ip "${1:-}" || die "could not authorize SSH"
say "Verifying SSH to $SSH_HOST…"
if ssh_cmd -o ConnectTimeout=15 -o BatchMode=yes "$SSH_HOST" 'echo ok' >/dev/null 2>&1; then
say "SSH OK ✓ ($SSH_HOST)"
else
die "authorized, but SSH to $SSH_HOST still fails (route/key/DNS?) — investigate"
fi
;;
ports)
[ -f "$PORTS_REGISTRY" ] || die "registry not found: $PORTS_REGISTRY (see the _VMPORTS skill)"
sub="${1:-list}"; shift || true
# ---- fetch live state once (port->unit via cgroup, nginx vhost->port) ----
live="$(ssh_run 'bash -s' <<'RFETCH'
echo "### PORTS"
sudo ss -ltnpH 2>/dev/null | while read -r line; do
laddr=$(echo "$line" | awk '{print $4}'); port=${laddr##*:}
case "$port" in (*[!0-9]*|"") continue;; esac
pid=$(echo "$line" | grep -oE 'pid=[0-9]+' | head -1 | grep -oE '[0-9]+'); unit=""
[ -n "$pid" ] && unit=$(grep -aoE '[A-Za-z0-9_.@-]+\.service' /proc/$pid/cgroup 2>/dev/null | head -1)
printf '%s|%s\n' "$port" "${unit%.service}"
done | sort -t'|' -k1,1n -u
echo "### VHOSTS"
for f in /etc/nginx/sites-enabled/*; do
n=$(basename "$f")
# Resolve the ROOT "location / " upstream (trailing space avoids matching /mcp etc).
# Handle both direct (proxy_pass http://127.0.0.1:PORT) and named upstreams
# (proxy_pass http://name; upstream name { server 127.0.0.1:PORT; }).
tgt=$(grep -A4 'location / ' "$f" 2>/dev/null | grep -oE 'proxy_pass http://[^;]+' | head -1 | sed 's#proxy_pass http://##')
case "$tgt" in
127.0.0.1:*) p=${tgt##*:} ;;
"") p=static ;;
*) p=$(grep -E "upstream[[:space:]]+${tgt}[[:space:]]*\{" -A3 "$f" 2>/dev/null | grep -oE 'server[[:space:]]+127\.0\.0\.1:[0-9]+' | grep -oE '[0-9]+$' | head -1)
[ -z "$p" ] && p=static ;;
esac
printf '%s|%s\n' "$n" "${p:-static}"
done
RFETCH
)"
# ---- parse: live_unit[port]=unit ; vhost_port[domain]=port ----------------
declare -A live_unit vhost_port
section=""
while IFS= read -r ln; do
case "$ln" in
"### PORTS") section=p; continue;;
"### VHOSTS") section=v; continue;;
"") continue;;
esac
k="${ln%%|*}"; v="${ln#*|}"
[ "$section" = p ] && live_unit["$k"]="$v"
[ "$section" = v ] && vhost_port["$k"]="$v"
done <<< "$live"
# ---- parse registry: reg_unit[port], reg_domain[port], reg_pool[port] -----
declare -A reg_unit reg_domain reg_pool reg_note
reg_ports=()
trim() { local s="$1"; s="${s#"${s%%[![:space:]]*}"}"; s="${s%"${s##*[![:space:]]}"}"; printf '%s' "$s"; }
while IFS='|' read -r c_port c_unit c_dom c_repo c_pool c_note; do
[ -z "${c_port// }" ] && continue
case "$(trim "$c_port")" in (\#*) continue;; esac
p=$(trim "$c_port"); case "$p" in (*[!0-9]*|"") continue;; esac
reg_ports+=("$p")
reg_unit["$p"]="$(trim "$c_unit")"
reg_domain["$p"]="$(trim "$c_dom")"
reg_pool["$p"]="$(trim "$c_pool")"
reg_note["$p"]="$(trim "$c_note")"
done < "$PORTS_REGISTRY"
is_taken() { # taken if claimed in registry OR bound live
local p="$1"; [ -n "${reg_unit[$p]:-}" ] && return 0; [ -n "${live_unit[$p]:-}" ] && return 0; return 1
}
case "$sub" in
list)
printf '\033[1m%-6s %-20s %-7s %-30s %s\033[0m\n' PORT UNIT POOL DOMAIN LIVE
for p in $(printf '%s\n' "${reg_ports[@]}" | sort -n); do
want="${reg_unit[$p]}"; got="${live_unit[$p]:-}"
# system services carry a descriptive label, not an exact unit name —
# any live binding counts as bound for them.
if [ "$want" = "$got" ] || { [ "${reg_pool[$p]}" = system ] && [ -n "$got" ]; }; then
mark=$'\033[0;32m●\033[0m bound'
elif [ -n "$got" ]; then mark=$'\033[0;31m✗\033[0m '"$got"
else mark=$'\033[0;33m○\033[0m not bound'; fi
printf '%-6s %-20s %-7s %-30s %b\n' "$p" "$want" "${reg_pool[$p]}" "${reg_domain[$p]}" "$mark"
done
# live app-pool ports with no registry row = unregistered
unreg=""
for p in "${!live_unit[@]}"; do
[ "$p" -ge "$APP_POOL_LO" ] && [ "$p" -le "$APP_POOL_HI" ] || continue
[ -n "${reg_unit[$p]:-}" ] || unreg+=$'\n'" $p -> ${live_unit[$p]}"
done
[ -n "$unreg" ] && { printf '\n\033[1;31munregistered live ports (app pool):\033[0m'; printf '%b\n' "$unreg"; }
;;
free)
n="${1:-1}"; found=0
for ((p=APP_POOL_LO; p<=APP_POOL_HI; p++)); do
is_taken "$p" || { echo "$p"; found=$((found+1)); [ "$found" -ge "$n" ] && break; }
done
[ "$found" -gt 0 ] || die "no free ports in $APP_POOL_LO-$APP_POOL_HI"
;;
check)
[ $# -ge 1 ] || die "usage: akurai-ec2 ports check <port>"
p="$1"
say "port $p"
if [ -n "${reg_unit[$p]:-}" ]; then echo " registry: ${reg_unit[$p]} (${reg_pool[$p]}) ${reg_domain[$p]} — ${reg_note[$p]}"
else echo " registry: (unclaimed)"; fi
if [ -n "${live_unit[$p]:-}" ]; then echo " live: bound by ${live_unit[$p]}"
else echo " live: not bound"; fi
is_taken "$p" && printf '\033[1;31m => TAKEN\033[0m\n' || printf '\033[1;32m => FREE\033[0m\n'
;;
drift)
issues=0
# (a) registry port expected unit U, but live differs / missing
for p in $(printf '%s\n' "${reg_ports[@]}" | sort -n); do
[ "${reg_pool[$p]}" = system ] && continue
want="${reg_unit[$p]}"; got="${live_unit[$p]:-}"
[ -z "$want" ] && continue
if [ -z "$got" ]; then
printf '\033[0;33m○ %-18s NOT BOUND on its port %s\033[0m (%s)\n' "$want" "$p" "${reg_note[$p]}"; issues=$((issues+1))
elif [ "$got" != "$want" ]; then
printf '\033[0;31m✗ port %s: expected %s, live is %s\033[0m\n' "$p" "$want" "$got"; issues=$((issues+1))
fi
done
# (b) nginx vhost points at a port other than what the registry intends.
# A port may carry comma-separated domain aliases; a "host/path" entry is
# path-routed (e.g. notes /mcp) and is not its own vhost — skip it.
declare -A flagged_dom
for p in "${!reg_domain[@]}"; do
raw="${reg_domain[$p]}"; [ "$raw" = "-" ] && continue
IFS=',' read -ra _doms <<< "$raw"
for d in "${_doms[@]}"; do
case "$d" in (*/*) continue;; esac
lv="${vhost_port[$d]:-}"; [ -z "$lv" ] && continue
[ "$lv" = static ] && { printf '\033[0;33m○ vhost %s is static — registry routes it to %s (%s)\033[0m\n' "$d" "$p" "${reg_unit[$p]}"; flagged_dom[$d]=1; issues=$((issues+1)); continue; }
[ "$lv" != "$p" ] && { printf '\033[0;31m✗ vhost %s -> %s, registry intends %s (%s)\033[0m\n' "$d" "$lv" "$p" "${reg_unit[$p]}"; flagged_dom[$d]=1; issues=$((issues+1)); }
done
done
# (c) live app-pool listeners with no registry row
for p in "${!live_unit[@]}"; do
[ "$p" -ge "$APP_POOL_LO" ] && [ "$p" -le "$APP_POOL_HI" ] || continue
[ -n "${reg_unit[$p]:-}" ] || { printf '\033[0;31m✗ port %s bound by %s — not in registry\033[0m\n' "$p" "${live_unit[$p]}"; issues=$((issues+1)); }
done
# (d) live vhost serving an app-pool port that belongs to a different app
# (the port's registered domain — or any of its comma aliases — isn't this one)
for d in "${!vhost_port[@]}"; do
[ -n "${flagged_dom[$d]:-}" ] && continue # already reported in (b)
lv="${vhost_port[$d]}"; case "$lv" in (*[!0-9]*|"") continue;; esac
[ "$lv" -ge "$APP_POOL_LO" ] && [ "$lv" -le "$APP_POOL_HI" ] || continue
raw="${reg_domain[$lv]:-}"; { [ -z "$raw" ] || [ "$raw" = "-" ]; } && continue
owner=0; IFS=',' read -ra _owners <<< "$raw"
for o in "${_owners[@]}"; do [ "$o" = "$d" ] && owner=1; done
if [ "$owner" = 0 ]; then
printf '\033[0;31m✗ vhost %s -> %s, but %s is %s (%s)\033[0m\n' "$d" "$lv" "$lv" "${reg_unit[$lv]}" "$raw"
issues=$((issues+1))
fi
done
if [ "$issues" -eq 0 ]; then say "no drift — registry matches the box"; else die "$issues drift issue(s) — see above"; fi
;;
*) die "usage: akurai-ec2 ports [list|free [n]|check <port>|drift]" ;;
esac
;;
processes|ps)
# List the running AkurAI fleet on the box: unit, state, PID, memory, uptime,
# and listening port(s). Optional [pattern] overrides the default fleet match
# (an egrep alternation). Goes beyond `status`: per-unit PID/mem/uptime + ports.
pat="${1:-akurai|golfset|idp|mail|router|drive|fulltrui|platform|notes|passvault|crm|vpn|monitor|bunfork}"
say "AkurAI processes on $SSH_HOST (match: $pat)"
ssh_run "bash -s -- '$pat'" <<'EOF'
pat="$1"
# pid -> listening port(s), so each unit can show what it's actually serving.
declare -A pid_ports
while read -r line; do
laddr=$(echo "$line" | awk '{print $4}'); port=${laddr##*:}
case "$port" in (*[!0-9]*|"") continue;; esac
pid=$(echo "$line" | grep -oE 'pid=[0-9]+' | head -1 | grep -oE '[0-9]+')
[ -n "$pid" ] && pid_ports[$pid]+="$port "
done < <(sudo ss -ltnpH 2>/dev/null)
printf '%-24s %-9s %-7s %-8s %-19s %s\n' UNIT STATE PID MEM SINCE PORTS
n=0
for unit in $(systemctl list-units --type=service --state=running --no-legend --plain 2>/dev/null | awk '{print $1}' | grep -iE "$pat" | sort -u); do
pid=$(systemctl show "$unit" -p MainPID --value 2>/dev/null)
sub=$(systemctl show "$unit" -p SubState --value 2>/dev/null)
memraw=$(systemctl show "$unit" -p MemoryCurrent --value 2>/dev/null)
since=$(systemctl show "$unit" -p ActiveEnterTimestamp --value 2>/dev/null | awk '{print $2" "$3}')
mem="-"
case "$memraw" in (''|'[not set]'|18446744073709551615) ;; (*) mem=$(numfmt --to=iec "$memraw" 2>/dev/null || echo "$memraw");; esac
ports="${pid_ports[$pid]:-—}"
printf '%-24s %-9s %-7s %-8s %-19s %s\n' "${unit%.service}" "$sub" "$pid" "$mem" "$since" "$ports"
n=$((n+1))
done
echo
echo "total running: $n"
EOF
;;
health)
# health [--all] — fan-out health check across all registered apps.
# --all is the explicit compatibility spelling; health already checks the
# complete registry, so reject every other argument instead of ignoring it.
while [ $# -gt 0 ]; do
case "$1" in
--all) shift ;;
*) die "usage: akurai-ec2 health [--all]" ;;
esac
done
# Reads AKURAI_APPS.toml from AkurAI-Framework repo.
APPS_TOML="$HOME/Projects/AkurAI-Framework/AKURAI_APPS.toml"
if [ ! -f "$APPS_TOML" ]; then
die "AKURAI_APPS.toml not found at $APPS_TOML"
fi
echo "=== Fleet Health Check ==="
python3 -c "
import tomllib, subprocess, sys, json
with open('$APPS_TOML', 'rb') as f:
apps = tomllib.load(f)
for cfg in apps.get('app', []):
name = cfg.get('name', '?')
if cfg.get('retired'):
continue
# DNS-only services expose no HTTP endpoint; probe a TXT health record.
if cfg.get('health_type') == 'dns':
server = cfg.get('dns_server', '')
qname = cfg.get('dns_name', '')
expect = cfg.get('dns_expect', 'ok')
try:
r = subprocess.run(['dig', f'@{server}', qname, 'TXT', '+short', '+time=3', '+tries=1'], capture_output=True, text=True, timeout=10)
if r.returncode == 0 and expect in r.stdout:
print(f' [\033[32mOK\033[0m] {name:30s} {\"dns:\"+qname:25s} status={expect}')
else:
print(f' [\033[31mFAIL\033[0m] {name:30s} dig {qname} @{server} (want TXT {expect})')
except Exception as e:
print(f' [\033[31mERR\033[0m] {name:30s} {e}')
continue
domain = cfg.get('domain', '')
if not domain:
continue
health_url = f'https://{domain}/health' if cfg.get('health_path') == '/health' else f'https://{domain}/api/health'
try:
r = subprocess.run(['curl', '-fsS', '--max-time', '5', health_url], capture_output=True, text=True, timeout=10)
if r.returncode == 0:
try:
data = json.loads(r.stdout)
app = data.get('app', data.get('service', data.get('framework', name)))
status = data.get('status', 'ok')
except Exception:
app, status = name, 'ok'
print(f' [\033[32mOK\033[0m] {name:30s} {app:25s} status={status}')
else:
print(f' [\033[31mFAIL\033[0m] {name:30s} {health_url}')
except Exception as e:
print(f' [\033[31mERR\033[0m] {name:30s} {e}')
" 2>&1 | grep -v '^$' || true
;;
search)
# search <pattern> — cross-repo search across all AkurAI projects
[ $# -ge 1 ] || die "usage: akurai-ec2 search <pattern> [--include *.rs]"
pattern="$1"; shift
include="*.rs"
while [ $# -gt 0 ]; do
case "$1" in
--include) include="$2"; shift 2 ;;
*) break ;;
esac
done
echo "=== Searching AkurAI repos for: $pattern ==="
for d in "$HOME/Projects"/AkurAI-* "$HOME/Projects"/akurai-*; do
[ -d "$d" ] || continue
name=$(basename "$d")
matches=$(rg -l --include "$include" "$pattern" "$d/src" "$d/crates" 2>/dev/null | head -5 || true)
if [ -n "$matches" ]; then
echo " [$name]"
echo "$matches" | while read -r f; do
echo " ${f#$d/}"
done
fi
done
;;
repos)
# repos status — multi-repo git status across all AkurAI projects
case "${1:-status}" in
status)
echo "=== AkurAI Repos Status ==="
for d in "$HOME/Projects"/AkurAI-* "$HOME/Projects"/akurai-* "$HOME/Projects"/golfsetridak*; do
[ -d "$d/.git" ] || continue
name=$(basename "$d")
branch=$(git -C "$d" rev-parse --abbrev-ref HEAD 2>/dev/null || echo "?")
dirty=$(git -C "$d" status --porcelain 2>/dev/null | wc -l)
ahead=$(git -C "$d" rev-list --count @{upstream}..HEAD 2>/dev/null || echo 0)
behind=$(git -C "$d" rev-list --count HEAD..@{upstream} 2>/dev/null || echo 0)
printf ' %-35s %-22s dirty=%-3s ahead=%-3s behind=%-3s\n' "$name" "$branch" "$dirty" "$ahead" "$behind"
done
;;
*) die "usage: akurai-ec2 repos status" ;;
esac
;;
validate-config)
# validate-config [path] — validate an akurai-deploy.toml file
toml_path="${1:-./akurai-deploy.toml}"
[ -f "$toml_path" ] || die "not found: $toml_path"
python3 -c "
import tomllib, sys
with open('$toml_path', 'rb') as f:
cfg = tomllib.load(f)
errors = []
required = ['name', 'port']
for k in required:
if k not in cfg:
errors.append(f'missing required key: {k}')
if 'port' in cfg:
p = cfg['port']
if not isinstance(p, int) or p < 1024 or p > 65535:
errors.append(f'invalid port: {p} (must be 1024-65535)')
if 'domain' in cfg:
d = cfg['domain']
if d != '-' and '.' not in d:
errors.append(f'domain \"{d}\" looks invalid')
if errors:
print('VALIDATION FAILED:')
for e in errors:
print(f' - {e}')
sys.exit(1)
else:
print(f'✅ {cfg.get(\"name\",\"?\")} — config valid')
print(f' domain: {cfg.get(\"domain\",\"-\")}')
print(f' port: {cfg.get(\"port\",\"?\")}')
print(f' build: {cfg.get(\"build\",\"(shared binary)\")[:60]}')
" 2>&1 || true
;;
mariadb)
# mariadb <command...> — delegate to the on-box akurai-mariadbctl, which
# operates the /opt/akurai-mariadb docker compose stack (mariadb-prod
# :3306, mariadb-dev :3307; loopback-only, hard memory/cpu limits). The
# 2026-07-16 migration replaced the native low-mem mariadb-server install;
# root credentials live in container secret files and PassVault folder
# "EC2-MariaDB", never printed here. create-user reads the password from
# STDIN of this command (pipe from a secret file — never argv).
[ $# -ge 1 ] || set -- status
case "$1" in
status|health|up|down|restart|mode|sql|create-db|create-user|copy-db|dump|logs)
quoted=""
for argument in "$@"; do quoted="$quoted $(printf '%q' "$argument")"; done
ssh_run "akurai-mariadbctl$quoted"
;;
install)
say "installing akurai-mariadbctl from the repo copy…"
ssh_run 'sudo tee /usr/local/bin/akurai-mariadbctl > /dev/null && sudo chmod +x /usr/local/bin/akurai-mariadbctl && akurai-mariadbctl health' \
< "$(dirname "$(readlink -f "$0")")/akurai-mariadbctl"
;;
*) die "usage: akurai-ec2 mariadb <status|health|up|down|restart|mode|sql|create-db|create-user|copy-db|dump|logs|install> (see akurai-mariadbctl --help)" ;;
esac
;;
provision-oidc-client)
[ $# -ge 5 ] && [ $# -le 6 ] || die "usage: akurai-ec2 provision-oidc-client NAME REDIRECT ENV_PREFIX EMAIL FOLDER [REMOTE_ENV_FILE]"
exec "$HOME/Projects/AkurAI-Build/scripts/provision-app-oidc.py" "$@"
;;
secret-sync)
# Secret value is stdin-only: never place it in argv, logs, or shell history.
[ $# -eq 3 ] || die "usage: akurai-ec2 secret-sync SERVICE KEY ENV_FILE"
service="$1"
key="$2"
env_path="$3"
[[ "$service" =~ ^[A-Za-z0-9_.@-]+$ ]] || die "invalid service name"
[[ "$key" =~ ^[A-Z][A-Z0-9_]*$ ]] || die "invalid environment key"
[[ "$env_path" == /etc/* ]] || die "environment file must be below /etc"
secret="$(cat)"
[ -n "$secret" ] || die "secret-sync requires a secret on stdin"
[[ "$secret" != *$'\n'* ]] || die "secret must be one line"
# Values are transported verbatim (stdin -> remote `IFS= read -r` ->
# printf) into a KEY=value systemd EnvironmentFile line, so any printable
# character without shell-metacharacter risk in THAT path is safe. The
# previous [A-Za-z0-9._~-] set rejected legitimate secrets (e.g. tokens
# containing '!' or base64 '+/='). Still excluded: whitespace, quotes,
# backslash, $, backtick, #, ;, &, |, <, > — the write path would carry
# them, but keeping them out avoids any downstream consumer ambiguity.
[[ "$secret" =~ ^[A-Za-z0-9._~!@%^*+=:,/-]+$ ]] ||
die "secret contains unsupported characters"
if ! printf '%s\n' "$secret" | ssh_run \
"sudo sh -c 'set -eu
tmp=\$(mktemp)
trap \"rm -f \$tmp\" EXIT
if [ -f \"$env_path\" ]; then
grep -v \"^[[:space:]]*\\(export[[:space:]]\\+\\)\\?$key=\" \"$env_path\" >\"\$tmp\" || true
fi
IFS= read -r value
printf \"%s=%s\\n\" \"$key\" \"\$value\" >>\"\$tmp\"
install -D -o root -g root -m 600 \"\$tmp\" \"$env_path\"'"; then
unset secret
die "failed to install secret"
fi
unset secret
ssh_run "sudo systemctl restart '$service' && sudo systemctl is-active --quiet '$service'" ||
die "secret installed but $service failed to restart"
say "secret synchronized and $service active"
;;
help|*)
grep -E '^#( |$)' "$0" | sed 's/^# \{0,1\}//'
;;
esac