Menu
popagent
publicLatest change 682410a285bb0a73662cc4650c92c31c918e1cc9 - Add idle autonomous improvement workflow by Ólafur Búi Ólafsson
import { createHash } from "node:crypto";
import { execFile, spawn } from "node:child_process";
import { access, mkdir, rm, symlink } from "node:fs/promises";
import { isAbsolute, join, posix, resolve } from "node:path";
import { agentSettings, type AgentSettingsStore } from "./agent-settings";
import { agentWorkspaces, type AgentWorkspaceStore } from "./agent-workspaces";
import type { AgentSettings, AgentTask } from "./api-types";
import { autonomySettings, type AutonomySettingsStore } from "./autonomy-settings";
import { appLogger } from "./observability";
const SELF_UPDATE_ROOT = resolve(
process.env.POPAGENT_DATA_DIR ?? join(process.cwd(), "data"),
"self-update",
);
const SELF_UPDATE_BRANCH_PREFIX = "autonomous/self-update/";
const ZERO_OBJECT_ID = "0".repeat(40);
const PROTECTED_PATHS: Record<string, true> = {
".agents": true,
".akurai.yml": true,
".circleci": true,
".env": true,
".github": true,
".gitlab-ci.yml": true,
".gitattributes": true,
".gitmodules": true,
".gitignore": true,
".env.example": true,
"bun.lock": true,
"bunfig.toml": true,
"deploy.sh": true,
"docker-compose.yml": true,
"package.json": true,
"src/agent-context.ts": true,
"src/agent-execution.ts": true,
"src/agent-role-seed.sql": true,
"src/agent-runtime-settings.sql": true,
"src/agent-runtime-settings.ts": true,
"src/agent-settings.ts": true,
"src/agent.ts": true,
"src/api-types.ts": true,
"src/agent-skills.ts": true,
"src/autonomy-settings.ts": true,
"src/browser-policy.ts": true,
"src/browser-settings.sql": true,
"src/browser-settings.ts": true,
"src/evolution-runtime.ts": true,
"src/evolution-store.ts": true,
"src/evolution.sql": true,
"src/hook-lifecycle.ts": true,
"src/hooks.ts": true,
"src/models.ts": true,
"src/secret-bindings.sql": true,
"src/secrets.ts": true,
"src/self-update-policy.ts": true,
"src/self-update-scheduler.ts": true,
"src/server.ts": true,
"src/tasks.ts": true,
"src/tools/secrets.ts": true,
"src/tools/memory.ts": true,
"src/workspace.ts": true,
"tsconfig.json": true,
};
const PROTECTED_PREFIXES = [
".agents/",
".circleci/",
".git/",
".github/",
".gitlab/",
"systemd/",
];
const CREDENTIAL_BASENAMES: Record<string, true> = {
".netrc": true,
".npmrc": true,
".pypirc": true,
"auth.json": true,
credentials: true,
"credentials.json": true,
id_dsa: true,
id_ed25519: true,
id_rsa: true,
"service-account.json": true,
};
type GitSnapshot = {
head: string;
branch: string;
remotes: string;
};
export type SelfUpdateRun = {
taskId: string;
workspaceId: string;
repositoryPath: string;
path: string;
branch: string;
snapshot: GitSnapshot;
cloneRemotes: string;
delegations: Set<string>;
deploymentEnabled: boolean;
};
type RoleRepository = Pick<AgentSettingsStore, "get">;
type WorkspaceRepository = Pick<AgentWorkspaceStore, "resolveRepository">;
type AutonomyRepository = Pick<AutonomySettingsStore, "get">;
export class SelfUpdatePolicyError extends Error {
override readonly name = "SelfUpdatePolicyError";
}
function protectedPath(path: string): boolean {
const slashPath = path.replaceAll("\\", "/");
if (isAbsolute(path) || slashPath.startsWith("/")) return true;
const normalized = posix.normalize(slashPath).replace(/^\.\//, "");
if (normalized === ".." || normalized.startsWith("../")) return true;
const lower = normalized.toLowerCase();
const base = posix.basename(lower);
return lower === ".git"
|| lower.startsWith(".env.")
|| (base.startsWith("agents") && base.endsWith(".md"))
|| Boolean(PROTECTED_PATHS[lower])
|| PROTECTED_PREFIXES.some((prefix) => lower.startsWith(prefix))
|| /(^|\/)(?:\.aws|\.ssh|credentials?|secrets?|tokens?)(?:[./_-]|$)/.test(lower)
|| /\.(?:kdbx|key|p12|pfx|pem)$/.test(lower)
|| Boolean(CREDENTIAL_BASENAMES[base]);
}
export function assertSelfUpdatePathAllowed(path: string): void {
if (protectedPath(path)) {
throw new SelfUpdatePolicyError("Self-update containment denied a protected path");
}
}
function assertGitCommand(tokens: string[]): void {
const operation = tokens[1];
const args = tokens.slice(2);
if (operation === "status") {
if (args.every((arg) => ["--short", "--porcelain", "--porcelain=v1", "--branch"].includes(arg))) return;
} else if (operation === "diff") {
if (args.every((arg) => ["--stat", "--name-only", "--cached", "--check", "--no-ext-diff"].includes(arg))) return;
} else if (operation === "log") {
if (args.every((arg) => arg === "--oneline" || arg === "--decorate" || /^-n\d+$/.test(arg) || /^--max-count=\d+$/.test(arg))) return;
} else if (operation === "branch") {
if (args.every((arg) => arg === "--show-current" || arg === "--list")) return;
} else if (operation === "rev-parse") {
if (args.length === 1 && ["HEAD", "--show-toplevel"].includes(args[0]!)) return;
if (args.length === 2 && args[0] === "--abbrev-ref" && args[1] === "HEAD") return;
} else if (operation === "add") {
const paths = args.filter((arg) => arg !== "-A" && arg !== "--all" && arg !== "--");
if (paths.some((path) => path.startsWith("-"))) {
throw new SelfUpdatePolicyError("Self-update containment denied a Git add option");
}
if (paths.length || args.includes("-A") || args.includes("--all")) {
for (const path of paths) assertSelfUpdatePathAllowed(path);
return;
}
} else if (operation === "commit") {
const forbidden = args.some((arg) => ["--amend", "-a", "--all", "--reuse-message", "-C", "-c"].includes(arg));
const messageIndex = args.indexOf("-m");
if (!forbidden
&& args.includes("--no-verify")
&& args.includes("--no-gpg-sign")
&& messageIndex >= 0
&& Boolean(args[messageIndex + 1])) return;
}
throw new SelfUpdatePolicyError("Self-update containment denied a Git operation");
}
function assertVerificationCommand(tokens: string[]): void {
const [executable, operation, target] = tokens;
if (executable === "bun" && operation === "test") return;
if (executable === "bunx" && ["biome", "eslint", "prettier", "svelte-check", "tsc", "vite"].includes(operation ?? "")) return;
if (["npm", "pnpm", "yarn"].includes(executable ?? "")) {
const script = operation === "run" ? target : operation;
if (["build", "check", "lint", "test", "typecheck"].includes(script ?? "")) return;
}
if (executable === "cargo" && ["check", "clippy", "fmt", "test"].includes(operation ?? "")) return;
if (executable === "go" && ["build", "test", "vet"].includes(operation ?? "")) return;
if (executable === "python" && operation === "-m" && ["mypy", "pytest", "ruff"].includes(target ?? "")) return;
if (["mypy", "pytest", "ruff"].includes(executable ?? "")) return;
if (executable === "deno" && ["check", "fmt", "lint", "test"].includes(operation ?? "")) return;
if (executable === "dotnet" && ["build", "test"].includes(operation ?? "")) return;
if (["gradle", "mvn"].includes(executable ?? "") && ["build", "check", "test", "verify"].includes(operation ?? "")) return;
if (executable === "make" && ["build", "check", "lint", "test"].includes(operation ?? "")) return;
if (executable === "cmake" && operation === "--build") return;
if (executable === "ninja") return;
throw new SelfUpdatePolicyError("Self-update containment denied a command");
}
function assertSelfUpdateCommand(command: string): void {
if (!command.trim() || /[\n\r;&|<>`$\\"']/.test(command)) {
throw new SelfUpdatePolicyError("Self-update containment requires one literal command");
}
const lower = command.toLowerCase();
if (/(^|\s)(?:\.env(?:\.|\s|$)|\.git(?:\/|\s|$)|credential|secret|deploy|publish|push|remote|systemctl|docker|ssh|curl|wget|gh)(?:\s|\/|$)/.test(lower)) {
throw new SelfUpdatePolicyError("Self-update containment denied a protected operation");
}
const tokens = command.trim().split(/\s+/);
if (tokens[0] === "git") assertGitCommand(tokens);
else assertVerificationCommand(tokens);
}
export function assertSelfUpdateToolCall(toolName: string, input: unknown): void {
const lowerName = toolName.toLowerCase();
if (toolName === "retainMemory"
|| toolName === "editMemory"
|| toolName === "useBrowserSecret"
|| toolName === "bifrostNavigator"
|| /browser|playwright|navigate|screenshot/.test(lowerName)) {
throw new SelfUpdatePolicyError("Self-update containment denied a browser or credential tool");
}
if (!input || typeof input !== "object" || Array.isArray(input)) return;
const values = input as Record<string, unknown>;
if (typeof values.path === "string") assertSelfUpdatePathAllowed(values.path);
if (typeof values.cwd === "string" && values.cwd !== ".") assertSelfUpdatePathAllowed(values.cwd);
if (toolName === "mastra_workspace_list_files" && values.showHidden === true) {
throw new SelfUpdatePolicyError("Self-update containment denied hidden-file listing");
}
if (toolName === "mastra_workspace_grep" && values.includeHidden === true) {
throw new SelfUpdatePolicyError("Self-update containment denied hidden-file search");
}
if (toolName === "mastra_workspace_execute_command") {
if (typeof values.command !== "string") {
throw new SelfUpdatePolicyError("Self-update command is missing");
}
assertSelfUpdateCommand(values.command);
}
}
function processOutput(command: string, args: string[], cwd?: string): Promise<Buffer> {
const { promise, resolve: resolveOutput, reject } = Promise.withResolvers<Buffer>();
const child = spawn(command, args, {
cwd,
env: {
PATH: process.env.PATH,
HOME: "/nonexistent",
GIT_CONFIG_NOSYSTEM: "1",
GIT_TERMINAL_PROMPT: "0",
GIT_OPTIONAL_LOCKS: "0",
},
stdio: ["ignore", "pipe", "pipe"],
});
const stdout: Buffer[] = [];
child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk));
child.stderr.resume();
child.once("error", reject);
child.once("close", (code) => {
if (code === 0) resolveOutput(Buffer.concat(stdout));
else reject(new SelfUpdatePolicyError("Self-update Git containment check failed"));
});
return promise;
}
const SAFE_GIT_CONFIG = [
"-c", "core.hooksPath=/dev/null",
"-c", "credential.helper=",
"-c", "core.pager=cat",
"-c", "diff.external=",
];
function git(path: string, args: string[]): Promise<Buffer> {
return processOutput("git", [...SAFE_GIT_CONFIG, "-C", path, ...args]);
}
async function gitSucceeds(path: string, args: string[]): Promise<boolean> {
try {
await git(path, args);
return true;
} catch {
return false;
}
}
function digest(value: Buffer): string {
return createHash("sha256").update(value).digest("hex");
}
async function snapshot(path: string): Promise<GitSnapshot> {
const [head, branch, status, remotes] = await Promise.all([
git(path, ["rev-parse", "HEAD"]),
git(path, ["symbolic-ref", "--quiet", "--short", "HEAD"]),
git(path, ["status", "--porcelain=v1", "--untracked-files=all"]),
git(path, ["remote", "-v"]),
]);
if (status.length) throw new SelfUpdatePolicyError("Self-update requires a clean registered Git workspace");
return {
head: head.toString("utf8").trim(),
branch: branch.toString("utf8").trim(),
remotes: digest(remotes),
};
}
function assertEligibleRoles(
supervisor: AgentSettings | undefined,
implementer: AgentSettings | undefined,
reviewer: AgentSettings | undefined,
): void {
if (supervisor?.workspaceAccess !== "read-write"
|| implementer?.workspaceAccess !== "read-write"
|| !implementer.delegationEnabled
|| reviewer?.workspaceAccess === "none"
|| !reviewer?.delegationEnabled) {
throw new SelfUpdatePolicyError("Self-update agent roles are not eligible");
}
}
export class SelfUpdateRuntimePolicy {
private readonly activeTasks = new Set<string>();
constructor(
private readonly workspaces: WorkspaceRepository = agentWorkspaces,
private readonly roles: RoleRepository = agentSettings,
private readonly settings: AutonomyRepository = autonomySettings,
) {}
async begin(task: AgentTask): Promise<SelfUpdateRun> {
if (task.source !== "self-update") throw new SelfUpdatePolicyError("Self-update task source is invalid");
if (this.activeTasks.has(task.id)) throw new SelfUpdatePolicyError("Self-update task is already executing");
this.activeTasks.add(task.id);
const path = join(
SELF_UPDATE_ROOT,
createHash("sha256").update(task.id).digest("hex"),
);
try {
const [currentSettings, supervisor, implementer, reviewer, repository] = await Promise.all([
this.settings.get(),
this.roles.get("orchistrator"),
this.roles.get("implementer"),
this.roles.get("reviewer"),
this.workspaces.resolveRepository(task.workspaceId),
]);
if (!currentSettings.enabled || !currentSettings.selfUpdateEnabled) {
throw new SelfUpdatePolicyError("Self-update is disabled");
}
assertEligibleRoles(supervisor, implementer, reviewer);
const captured = await snapshot(repository.path);
const branch = `${SELF_UPDATE_BRANCH_PREFIX}${task.id.replace(/[^a-zA-Z0-9._-]/g, "-")}-${crypto.randomUUID().slice(0, 8)}`;
await mkdir(SELF_UPDATE_ROOT, { recursive: true, mode: 0o700 });
await rm(path, { recursive: true, force: true });
await processOutput("git", [
...SAFE_GIT_CONFIG,
"clone",
"--no-hardlinks",
"--no-checkout",
"--",
repository.path,
path,
]);
await git(path, ["checkout", "--no-track", "-b", branch, captured.head]);
await git(path, ["config", "user.name", "Popagent Self Update"]);
await git(path, ["config", "user.email", "self-update@localhost"]);
await git(path, ["config", "commit.gpgsign", "false"]);
const dependencyPath = join(repository.path, "node_modules");
if (await gitSucceeds(path, ["check-ignore", "--quiet", "node_modules"])) {
try {
await access(dependencyPath);
await symlink(dependencyPath, join(path, "node_modules"), "dir");
} catch {
// Verification remains available for dependency-free repositories.
}
}
await git(path, ["config", "core.hooksPath", "/dev/null"]);
return {
taskId: task.id,
workspaceId: task.workspaceId,
repositoryPath: repository.path,
path,
branch,
snapshot: captured,
cloneRemotes: digest(await git(path, ["remote", "-v"])),
delegations: new Set<string>(),
deploymentEnabled: currentSettings.idleDeploymentEnabled,
};
} catch (error) {
await rm(path, { recursive: true, force: true }).catch(() => undefined);
this.activeTasks.delete(task.id);
throw error;
}
}
observeDelegation(run: SelfUpdateRun, agentId: string, success: boolean): void {
if (success && (agentId === "implementer" || agentId === "reviewer")) {
run.delegations.add(agentId);
}
}
async accept(run: SelfUpdateRun, signal?: AbortSignal): Promise<void> {
let importedHead: string | undefined;
try {
signal?.throwIfAborted();
const [currentSettings, supervisor, implementer, reviewer, repository] = await Promise.all([
this.settings.get(),
this.roles.get("orchistrator"),
this.roles.get("implementer"),
this.roles.get("reviewer"),
this.workspaces.resolveRepository(run.workspaceId),
]);
if (!currentSettings.enabled || !currentSettings.selfUpdateEnabled) {
throw new SelfUpdatePolicyError("Self-update was disabled before completion");
}
assertEligibleRoles(supervisor, implementer, reviewer);
if (resolve(repository.path) !== resolve(run.repositoryPath)) {
throw new SelfUpdatePolicyError("Self-update workspace registration changed during execution");
}
if (!run.delegations.has("implementer") || !run.delegations.has("reviewer")) {
throw new SelfUpdatePolicyError("Self-update requires successful implementer and reviewer delegation");
}
const [current, cloneStatus, cloneBranch, cloneHead, cloneRemotes] = await Promise.all([
snapshot(run.repositoryPath),
git(run.path, ["status", "--porcelain=v1", "--untracked-files=all"]),
git(run.path, ["symbolic-ref", "--quiet", "--short", "HEAD"]),
git(run.path, ["rev-parse", "HEAD"]),
git(run.path, ["remote", "-v"]),
]);
if (current.head !== run.snapshot.head
|| current.branch !== run.snapshot.branch
|| current.remotes !== run.snapshot.remotes) {
throw new SelfUpdatePolicyError("Registered workspace changed during self-update");
}
if (cloneStatus.length
|| cloneBranch.toString("utf8").trim() !== run.branch
|| digest(cloneRemotes) !== run.cloneRemotes) {
throw new SelfUpdatePolicyError("Self-update checkout failed completion containment");
}
const finalHead = cloneHead.toString("utf8").trim();
if (finalHead === run.snapshot.head) return;
if (await gitSucceeds(run.repositoryPath, ["cat-file", "-e", `${finalHead}^{commit}`])) {
throw new SelfUpdatePolicyError("Self-update result is not a new local commit");
}
if (!(await gitSucceeds(run.path, ["merge-base", "--is-ancestor", run.snapshot.head, finalHead]))) {
throw new SelfUpdatePolicyError("Self-update commit does not descend from the captured HEAD");
}
const changed = (await git(run.path, [
"diff",
"--no-renames",
"--name-only",
"-z",
`${run.snapshot.head}..${finalHead}`,
]))
.toString("utf8")
.split("\0")
.filter(Boolean);
if (!changed.length) throw new SelfUpdatePolicyError("Self-update commit has no source change");
for (const path of changed) assertSelfUpdatePathAllowed(path);
if (await gitSucceeds(run.repositoryPath, ["show-ref", "--verify", "--quiet", `refs/heads/${run.branch}`])) {
throw new SelfUpdatePolicyError("Self-update branch already exists");
}
await git(run.repositoryPath, ["fetch", "--no-tags", "--no-write-fetch-head", run.path, finalHead]);
signal?.throwIfAborted();
await git(run.repositoryPath, ["update-ref", `refs/heads/${run.branch}`, finalHead, ZERO_OBJECT_ID]);
importedHead = finalHead;
signal?.throwIfAborted();
const afterImport = await snapshot(run.repositoryPath);
if (afterImport.head !== run.snapshot.head
|| afterImport.branch !== run.snapshot.branch
|| afterImport.remotes !== run.snapshot.remotes) {
throw new SelfUpdatePolicyError("Registered workspace changed while importing self-update commit");
}
if (run.deploymentEnabled) {
await git(run.repositoryPath, ["merge", "--ff-only", finalHead]);
try {
await processOutput(join(run.repositoryPath, "deploy.sh"), ["publish"], run.repositoryPath);
} catch (error) {
await git(run.repositoryPath, ["reset", "--hard", run.snapshot.head]);
throw error;
}
execFile(join(run.repositoryPath, "deploy.sh"), ["deploy"], {
cwd: run.repositoryPath,
windowsHide: true,
}, (error) => {
if (error) appLogger().error("self_update.deploy_failed", { taskId: run.taskId, error: error.message });
}).unref();
}
} catch (error) {
if (importedHead) {
await git(run.repositoryPath, ["update-ref", "-d", `refs/heads/${run.branch}`, importedHead]).catch(() => undefined);
}
throw error;
} finally {
await this.release(run);
}
}
async reject(run: SelfUpdateRun): Promise<void> {
await this.release(run);
}
private async release(run: SelfUpdateRun): Promise<void> {
await rm(run.path, { recursive: true, force: true });
this.activeTasks.delete(run.taskId);
}
}
export const selfUpdateRuntimePolicy = new SelfUpdateRuntimePolicy();