AkurAI Build
Sign in with AkurAI ID
Menu

popagent

public

Latest change 2d93b38c8d425f0bed669e8156340feac8a1fd35 - Release deployment lock before reexec by Ólafur Búi Ólafsson

#!/usr/bin/env bash
set -Eeuo pipefail

REMOTE_URL="${POPAGENT_GIT_REMOTE:-https://akurai-build.olibuijr.com/git/popagent.git}"
DEPLOY_HOST="${POPAGENT_DEPLOY_HOST:-titan}"
DEPLOY_PATH="${POPAGENT_DEPLOY_PATH:-/home/olibuijr/Projects/popagent}"
BRANCH="${POPAGENT_DEPLOY_BRANCH:-main}"
COMMAND="${1:-help}"
ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
HOST="${HOSTNAME:-$(uname -n)}"; HOST="${HOST%%.*}"; HOST="${HOST,,}"

say() { printf '[popagent] %s\n' "$*"; }
die() { say "ERROR: $*" >&2; exit 1; }
need() { command -v "$1" >/dev/null || die "$1 is required"; }
remote() { git remote get-url origin >/dev/null 2>&1 || git remote add origin "$REMOTE_URL"; }
clean_tree() { [[ -z "$(git status --porcelain)" ]] || die "working tree is dirty; commit or restore it first"; }
ui_only_paths() {
  local path
  (($#)) || return 1
  for path; do case "$path" in src/ui/*|AGENTS_UI.md) ;; *) return 1 ;; esac; done
}

ui_only_change() {
  local before="$1" after="$2"
  [[ -n "$before" && "$before" != "$after" ]] || return 1
  mapfile -t paths < <(git diff --name-only "$before" "$after")
  ui_only_paths "${paths[@]}"
}
install_dependencies() {
  local before="${1:-}" after="${2:-HEAD}"
  if [[ ! -d node_modules ]] || { [[ -n "$before" ]] && ! git diff --quiet "$before" "$after" -- package.json bun.lock; }; then
    say "installing locked dependencies"
    bun install --frozen-lockfile
  fi
}
self_check() {
  ui_only_paths src/ui/AgentSettingsPage.tsx AGENTS_UI.md || die "UI-only change detection failed"
  ! ui_only_paths src/ui/AgentSettingsPage.tsx deploy.sh || die "mixed change detection failed"
  say "deployment checks passed"
}

privacy_check() {
  local forbidden pattern
  forbidden='(^|/)(\.env$|\.env\.(local|production|development|test)$|\.secrets/|data/|workspace/[^.]|\.playwright-cli/|\.tmp/)|\.(pem|key|p12|pfx|sqlite|db|dump)$'
  if git ls-files | grep -E "$forbidden" >/dev/null; then
    git ls-files | grep -E "$forbidden" >&2
    die "tracked private or runtime files found"
  fi
  pattern='BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY|((api|access|secret)[_-]?key|token|password)[[:space:]]*[:=][[:space:]]*[^$<{[:space:]#][^[:space:]#]{7,}'
  local matches
  matches="$(git grep -n -I -E "$pattern" -- ':!.env.example' ':!deploy.sh' 2>/dev/null | grep -v 'secret_key: "popagent-local-search"' || true)"
  if [[ -n "$matches" ]]; then
    printf '%s\n' "$matches" >&2
    die "possible secret found in tracked content"
  fi
  if command -v gitleaks >/dev/null; then gitleaks git --no-banner --redact; else say "gitleaks not installed; built-in scan passed"; fi
}

publish() {
  need git
  cd "$ROOT"
  clean_tree
  privacy_check
  [[ "$(git branch --show-current)" == "$BRANCH" ]] || die "publish from $BRANCH"
  remote
  local askpass="" url="$REMOTE_URL"
  if [[ -n "${POPAGENT_GIT_TOKEN:-}" ]]; then
    [[ -n "${POPAGENT_GIT_USERNAME:-}" ]] || die "POPAGENT_GIT_USERNAME is required with POPAGENT_GIT_TOKEN"
    askpass="$(mktemp)"
    chmod 700 "$askpass"
    cat >"$askpass" <<'EOF'
#!/usr/bin/env bash
case "$1" in *Username*) printf '%s\n' "$POPAGENT_GIT_USERNAME";; *) printf '%s\n' "$POPAGENT_GIT_TOKEN";; esac
EOF
    trap 'rm -f "${askpass:-}"' RETURN
    GIT_ASKPASS="$askpass" GIT_TERMINAL_PROMPT=0 git push -u origin "$BRANCH"
  else
    git push -u origin "$BRANCH"
  fi
  say "published $(git rev-parse --short HEAD) to $url"
}

on_midget() {
  need git; need ssh
  cd "$ROOT"
  clean_tree
  remote
  git fetch origin "$BRANCH"
  [[ "$(git rev-parse HEAD)" == "$(git rev-parse "origin/$BRANCH")" ]] || die "local HEAD is not published origin/$BRANCH; run '$0 publish' first"
  exec ssh "$DEPLOY_HOST" "cd '$DEPLOY_PATH' && ./deploy.sh deploy"
}

load_environment() {
  [[ -f .env ]] || die ".env is missing; create it from .env.example with real local credentials"
  chmod 600 .env
  set -a; source .env; set +a
  export PORT="${PORT:-5180}"
  local required=(DATABASE_URL OBSERVABILITY_DATABASE_URL POSTGRES_PASSWORD OBSERVABILITY_POSTGRES_PASSWORD NINEROUTER_URL NINEROUTER_API_KEY POPAGENT_SECRET_KEY)
  local name
  for name in "${required[@]}"; do [[ -n "${!name:-}" ]] || die "$name is required in .env"; done
  DATA_DIR="${POPAGENT_DATA_DIR:-$ROOT/data}"
  [[ "$DATA_DIR" = /* ]] || DATA_DIR="$ROOT/$DATA_DIR"
  mkdir -p "$DATA_DIR/deploy" "$DATA_DIR/backups"
  chmod 700 "$DATA_DIR" "$DATA_DIR/deploy" "$DATA_DIR/backups"
}

port_open() { timeout 2 bash -c "</dev/tcp/127.0.0.1/$1" 2>/dev/null; }

ensure_dependency() {
  local port="$1" service="$2" legacy_name="$3"
  port_open "$port" && return 0
  if docker container inspect "$legacy_name" >/dev/null 2>&1; then
    docker start "$legacy_name" >/dev/null
  else
    docker compose up -d --wait "$service"
  fi
  port_open "$port" || die "$service is unavailable on port $port"
}

ensure_agent_workspace() {
  local root="${POPAGENT_WORKSPACE_ROOT:-$ROOT/workspace}" repository="${POPAGENT_DEFAULT_REPOSITORY:-popagent}"
  [[ "$root" = /* ]] || root="$ROOT/$root"
  [[ "$repository" != /* && "$repository" != *".."* ]] || die "POPAGENT_DEFAULT_REPOSITORY must be a contained relative path"
  local checkout="$root/$repository"
  mkdir -p "$root"
  if [[ ! -e "$checkout" ]]; then
    say "cloning the agent workspace into $checkout"
    git clone "$REMOTE_URL" "$checkout"
  fi
  [[ -d "$checkout/.git" ]] || die "default agent workspace is not a Git checkout: $checkout"
}

start_dependencies() {
  need docker
  docker compose version >/dev/null
  ensure_dependency 5433 postgres popagent-postgres
  ensure_dependency 5435 observability-postgres popagent-observability-postgres
  ensure_dependency 8889 searxng popagent-searxng
  curl --fail --silent --show-error --max-time 5 http://127.0.0.1:8889/healthz >/dev/null || die "SearXNG is unhealthy"
  if ! curl --fail --silent --max-time 5 "${NINEROUTER_URL%/v1}/v1/models" -H "Authorization: Bearer $NINEROUTER_API_KEY" >/dev/null; then
    docker start omniroute >/dev/null 2>&1 || true
    sleep 3
    curl --fail --silent --show-error --max-time 5 "${NINEROUTER_URL%/v1}/v1/models" -H "Authorization: Bearer $NINEROUTER_API_KEY" >/dev/null || die "9Router is unavailable"
  fi
}

start_test_databases() {
  TEST_PROJECT="popagent-test-$$"
  TEST_COMPOSE="$(mktemp)"
  cat >"$TEST_COMPOSE" <<EOF
services:
  app:
    image: pgvector/pgvector:0.8.1-pg17
    environment: { POSTGRES_DB: popagent_test, POSTGRES_USER: popagent, POSTGRES_PASSWORD: test-only }
    ports: ["127.0.0.1::5432"]
    healthcheck: { test: ["CMD-SHELL", "pg_isready -U popagent -d popagent_test"], interval: 2s, timeout: 2s, retries: 30 }
  observability:
    image: postgres:17-alpine
    environment: { POSTGRES_DB: popagent_observability_test, POSTGRES_USER: popagent, POSTGRES_PASSWORD: test-only }
    ports: ["127.0.0.1::5432"]
    healthcheck: { test: ["CMD-SHELL", "pg_isready -U popagent -d popagent_observability_test"], interval: 2s, timeout: 2s, retries: 30 }
EOF
  docker compose -p "$TEST_PROJECT" -f "$TEST_COMPOSE" up -d --wait
  local app_port obs_port
  app_port="$(docker compose -p "$TEST_PROJECT" -f "$TEST_COMPOSE" port app 5432 | awk -F: '{print $NF}')"
  obs_port="$(docker compose -p "$TEST_PROJECT" -f "$TEST_COMPOSE" port observability 5432 | awk -F: '{print $NF}')"
  TEST_DATABASE_URL="postgresql://popagent:test-only@127.0.0.1:$app_port/popagent_test"
  TEST_OBSERVABILITY_URL="postgresql://popagent:test-only@127.0.0.1:$obs_port/popagent_observability_test"
}

cleanup_test_databases() {
  if [[ -n "${TEST_COMPOSE:-}" ]]; then
    docker compose -p "$TEST_PROJECT" -f "$TEST_COMPOSE" down -v --remove-orphans >/dev/null 2>&1 || true
    rm -f "$TEST_COMPOSE"
  fi
  [[ -z "${TEST_ROOT:-}" ]] || rm -rf "$TEST_ROOT"
}
run_isolated_tests() {
  local test_root="$1"; shift
  DATABASE_URL="$TEST_DATABASE_URL" \
  OBSERVABILITY_DATABASE_URL="$TEST_OBSERVABILITY_URL" \
  POPAGENT_DATA_DIR="$test_root/data" \
  POPAGENT_WORKSPACE_ROOT="$test_root/workspace" \
  POPAGENT_DEFAULT_REPOSITORY=. \
  POPAGENT_ISOLATED_TEST=1 \
  bun test "$@"
}

test_suite() {
  cd "$ROOT"
  load_environment
  start_dependencies
  start_test_databases
  trap cleanup_test_databases EXIT
  TEST_ROOT="$(mktemp -d)"
  mkdir -p "$TEST_ROOT/data" "$TEST_ROOT/workspace"
  run_isolated_tests "$TEST_ROOT" "$@"
}


full_suite() {
  install_dependencies "${1:-}"
  start_test_databases
  trap cleanup_test_databases EXIT
  say "running TypeScript and full suite"
  TEST_ROOT="$(mktemp -d)"
  mkdir -p "$TEST_ROOT/data" "$TEST_ROOT/workspace"
  bun x tsc --noEmit
  run_isolated_tests "$TEST_ROOT"
}

health() {
  local port="$1" args=(--fail --silent --max-time 5)
  [[ -z "${POPAGENT_API_KEY:-}" ]] || args+=(-H "x-popagent-key: $POPAGENT_API_KEY")
  [[ "$(curl "${args[@]}" -o /dev/null -w '%{http_code}' "http://127.0.0.1:$port/api/health" 2>/dev/null)" == 200 ]]
}

candidate() {
  local port=15180
  while ss -ltn "sport = :$port" | grep -q LISTEN; do ((port++)); done
  say "validating candidate on port $port"
  DATABASE_URL="$TEST_DATABASE_URL" OBSERVABILITY_DATABASE_URL="$TEST_OBSERVABILITY_URL" PORT="$port" bun run src/server.ts >"$LOG.candidate" 2>&1 &
  CANDIDATE_PID=$!
  local ok=0
  for _ in {1..30}; do if health "$port"; then ok=1; break; fi; sleep 1; done
  kill "$CANDIDATE_PID" >/dev/null 2>&1 || true
  wait "$CANDIDATE_PID" >/dev/null 2>&1 || true
  [[ "$ok" == 1 ]] || die "candidate failed readiness; see $LOG.candidate"
}

container_publishing() {
  local id
  id="$(docker ps --filter "publish=$1" --format '{{.ID}}')"
  [[ -n "$id" && "$id" != *$'\n'* ]] || die "expected one running database container publishing port $1"
  printf '%s\n' "$id"
}

backup_if_needed() {
  local previous="$1"
  [[ -n "$previous" ]] || return 0
  git diff --quiet "$previous" HEAD -- '*.sql' && return 0
  local stamp; stamp="$(date -u +%Y%m%dT%H%M%SZ)"
  say "backing up databases before schema changes"
  docker exec "$(container_publishing 5433)" pg_dump -U popagent -d popagent >"$DATA_DIR/backups/application-$stamp.sql"
  docker exec "$(container_publishing 5435)" pg_dump -U popagent_observability -d popagent_observability >"$DATA_DIR/backups/observability-$stamp.sql"
  chmod 600 "$DATA_DIR/backups/"*"-$stamp.sql"
  find "$DATA_DIR/backups" -type f -printf '%T@ %p\n' | sort -nr | awk 'NR>10 {$1=""; sub(/^ /,""); print}' | xargs -r rm -f
}

service_command() {
  if [[ $EUID -eq 0 ]]; then
    SYSTEMCTL=(systemctl); UNIT_DIR=/etc/systemd/system; UNIT_TARGET=multi-user.target
  else
    SYSTEMCTL=(systemctl --user); UNIT_DIR="$HOME/.config/systemd/user"; UNIT_TARGET=default.target
  fi
}

install_service() {
  service_command
  mkdir -p "$UNIT_DIR"
  local bun_path user_line=""; bun_path="$(command -v bun)"
  [[ $EUID -ne 0 ]] || user_line="User=${SUDO_USER:-root}"
  cat >"$UNIT_DIR/popagent.service" <<EOF
[Unit]
Description=Popagent
After=network-online.target docker.service

[Service]
Type=simple
$user_line
WorkingDirectory=$ROOT
EnvironmentFile=$ROOT/.env
Environment=PORT=$PORT
ExecStart=$bun_path run src/server.ts
Restart=on-failure
RestartSec=2
TimeoutStopSec=20

[Install]
WantedBy=$UNIT_TARGET
EOF
  "${SYSTEMCTL[@]}" daemon-reload
  "${SYSTEMCTL[@]}" enable popagent.service >/dev/null
}

restart_and_watch() {
  service_command
  "${SYSTEMCTL[@]}" restart popagent.service
  local port="${PORT:-3000}"
  for second in {1..60}; do
    if ! "${SYSTEMCTL[@]}" is-active --quiet popagent.service; then return 1; fi
    health "$port" || { sleep 1; continue; }
    sleep 1
  done
  health "$port"
}

rollback() {
  local previous="$1"
  [[ -n "$previous" ]] || die "deployment failed and no previous release is recorded"
  say "rolling back to $previous"
  git reset --hard "$previous"
  install_dependencies
  install_service
  restart_and_watch || die "rollback failed; inspect the service journal"
}

deploy_titan() {
  need git; need bun; need curl; need flock; need ss
  cd "$ROOT"
  load_environment
  LOG="$DATA_DIR/deploy/$(date -u +%Y%m%dT%H%M%SZ).log"
  touch "$LOG"; chmod 600 "$LOG"; exec > >(tee -a "$LOG") 2>&1
  exec 9>"$DATA_DIR/deploy/deploy.lock"; flock -n 9 || die "another deployment is active"
  clean_tree
  remote
  local checkout_before; checkout_before="$(git rev-parse HEAD)"
  git fetch origin "$BRANCH"
  [[ "$(git branch --show-current)" == "$BRANCH" ]] || die "Titan checkout must be on $BRANCH"
  git merge --ff-only "origin/$BRANCH"
  local release; release="$(git rev-parse HEAD)"
  if [[ "${POPAGENT_DEPLOY_REEXECUTED:-0}" != 1 ]] && ! git diff --quiet "$checkout_before" "$release" -- deploy.sh; then
    say "continuing with updated deploy script"
    exec 9>&-
    POPAGENT_DEPLOY_REEXECUTED=1 exec "$ROOT/deploy.sh" deploy
  fi
  if ui_only_change "$checkout_before" "$release"; then
    say "UI-only update loaded without service deployment: $(git rev-parse --short HEAD)"
    return
  fi
  local previous=""; [[ -f "$DATA_DIR/deploy/current-release" ]] && previous="$(<"$DATA_DIR/deploy/current-release")"
  ensure_agent_workspace
  start_dependencies
  full_suite "$checkout_before"
  candidate
  backup_if_needed "$previous"
  cleanup_test_databases; trap - EXIT
  install_service
  if ! restart_and_watch; then rollback "$previous"; die "candidate rolled back after production health failure"; fi
  printf '%s\n' "$release" >"$DATA_DIR/deploy/current-release"
  chmod 600 "$DATA_DIR/deploy/current-release"
  find "$DATA_DIR/deploy" -name '*.log' -type f -printf '%T@ %p\n' | sort -nr | awk 'NR>20 {$1=""; sub(/^ /,""); print}' | xargs -r rm -f
  say "deployed $(git rev-parse --short HEAD)"
}

case "$COMMAND" in
  publish) publish ;;
  deploy) if [[ "$HOST" == midget* ]]; then on_midget; else deploy_titan; fi ;;
  test) shift; test_suite "$@" ;;
  self-check) self_check ;;
  privacy-check) cd "$ROOT"; privacy_check ;;
  *) printf 'Usage: %s {publish|deploy|test|self-check|privacy-check}\n' "$0"; exit 2 ;;
esac