Menu
popagent
publicLatest change 48f38756b02cd3f67b173ba9e5d7a5e122c03126 - Route read-only agents through fast model by Ólafur Búi Ólafsson
import { afterEach, describe, expect, test } from "bun:test";
import { execFile } from "node:child_process";
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
import type { AgentSettings, AgentTask, AgentWorkspace, AutonomySettings } from "./api-types";
import { createToolHooks } from "./hook-lifecycle";
import { HookRuntime } from "./hooks";
import {
assertSelfUpdatePathAllowed,
assertSelfUpdateToolCall,
SelfUpdateRuntimePolicy,
} from "./self-update-policy";
const execute = promisify(execFile);
const cleanupPaths: string[] = [];
afterEach(async () => {
await Promise.all(cleanupPaths.splice(0).map((path) => rm(path, { recursive: true, force: true })));
});
function agentRole(
id: string,
workspaceAccess: AgentSettings["workspaceAccess"],
delegationEnabled: boolean,
): AgentSettings {
return {
id,
name: id,
description: `${id} role`,
instructions: `${id} instructions`,
model: null,
workspaceAccess,
browserAccess: "none",
delegationEnabled,
tools: [],
sourceUrls: [],
createdAt: "",
updatedAt: "",
};
}
function settings(): AutonomySettings {
return {
enabled: true,
reflectionIntervalMs: 60_000,
batchSize: 10,
maxAttempts: 3,
autoApplyStrategies: true,
autoCreateSkills: true,
selfUpdateEnabled: true,
selfUpdateCron: "0 3 * * *",
updatedAt: "",
};
}
function task(workspaceId: string): AgentTask {
return {
id: crypto.randomUUID(),
sessionId: null,
scheduleId: crypto.randomUUID(),
source: "self-update",
workspaceId,
prompt: "contained update",
model: "test/model",
status: "running",
output: null,
error: null,
stepsCompleted: 0,
progress: null,
recoveryCount: 0,
attemptCount: 1,
maxAttempts: 3,
nextAttemptAt: null,
lastErrorClass: null,
deadLetteredAt: null,
createdAt: new Date().toISOString(),
startedAt: new Date().toISOString(),
completedAt: null,
};
}
async function repositoryFixture() {
const path = await mkdtemp(join(tmpdir(), "popagent-self-update-policy-"));
cleanupPaths.push(path);
await execute("git", ["init", "-b", "main", path]);
await execute("git", ["-C", path, "config", "user.name", "Policy Test"]);
await execute("git", ["-C", path, "config", "user.email", "policy@example.invalid"]);
await mkdir(join(path, "src"));
await writeFile(join(path, "src", "example.ts"), "export const value = 1;\n");
await execute("git", ["-C", path, "add", "."]);
await execute("git", ["-C", path, "commit", "-m", "initial"]);
const workspace: AgentWorkspace = {
id: crypto.randomUUID(),
name: "Policy test",
repositoryPath: path,
createdAt: "",
updatedAt: "",
};
const roles = {
popagent: agentRole("popagent", "read-write", false),
implementer: agentRole("implementer", "read-write", true),
reviewer: agentRole("reviewer", "read-only", true),
} as const;
const policy = new SelfUpdateRuntimePolicy(
{ async resolveRepository() { return { workspace, path }; } },
{ async get(id) { return roles[id as keyof typeof roles]; } },
{ async get() { return settings(); } },
);
return { path, workspace, policy };
}
describe("fixed self-update tool policy", () => {
test("denies browser, credential, protected path, remote, and deployment operations", () => {
expect(() => assertSelfUpdateToolCall("retainMemory", {})).toThrow("browser or credential");
expect(() => assertSelfUpdateToolCall("editMemory", {})).toThrow("browser or credential");
expect(() => assertSelfUpdateToolCall("useBrowserSecret", {})).toThrow("browser or credential");
expect(() => assertSelfUpdateToolCall("browserNavigate", { url: "https://example.com" })).toThrow("browser or credential");
expect(() => assertSelfUpdateToolCall("mastra_workspace_read_file", { path: ".env" })).toThrow("protected path");
expect(() => assertSelfUpdateToolCall("mastra_workspace_read_file", { path: ".git/config" })).toThrow("protected path");
expect(() => assertSelfUpdateToolCall("mastra_workspace_write_file", { path: "src/self-update-policy.ts" })).toThrow("protected path");
expect(() => assertSelfUpdateToolCall("mastra_workspace_execute_command", { command: "git push origin main" })).toThrow("protected operation");
expect(() => assertSelfUpdateToolCall("mastra_workspace_execute_command", { command: "bun run deploy" })).toThrow("protected operation");
});
test("allows source edits and bounded local verification or commit commands", () => {
expect(() => assertSelfUpdatePathAllowed("src/ui/example.ts")).not.toThrow();
expect(() => assertSelfUpdateToolCall("mastra_workspace_execute_command", { command: "bun test src/example.test.ts" })).not.toThrow();
expect(() => assertSelfUpdateToolCall("mastra_workspace_execute_command", {
command: "git commit --no-verify --no-gpg-sign -m self-update",
})).not.toThrow();
});
test("rechecks fixed policy after a managed hook replacement", async () => {
const runtime = new HookRuntime({
config: {
version: 1,
hooks: {
PreToolUse: [{ id: "replace", type: "http", url: "https://hooks.example/replace" }],
},
},
transport: async () => ({
schemaVersion: 1,
outcome: "modify",
reason: "prompt-controlled replacement",
replacement: { path: ".git/config" },
}),
});
const hooks = createToolHooks(
runtime,
{ sessionId: "task:self-update", turnId: "turn", model: "test/model" },
assertSelfUpdateToolCall,
);
await expect(hooks.beforeToolCall!({
toolName: "mastra_workspace_read_file",
input: { path: "src/example.ts" },
context: {},
} as never)).rejects.toThrow("protected path");
});
});
describe("SelfUpdateRuntimePolicy", () => {
test("imports only a reviewed clean local commit while preserving the registered checkout", async () => {
const { path, workspace, policy } = await repositoryFixture();
const originalHead = (await execute("git", ["-C", path, "rev-parse", "HEAD"])).stdout.trim();
const run = await policy.begin(task(workspace.id));
await writeFile(join(run.path, "src", "example.ts"), "export const value = 2;\n");
await execute("git", ["-C", run.path, "add", "src/example.ts"]);
await execute("git", ["-C", run.path, "commit", "--no-verify", "--no-gpg-sign", "-m", "self-update"]);
policy.observeDelegation(run, "implementer", true);
policy.observeDelegation(run, "reviewer", true);
await policy.accept(run);
expect((await execute("git", ["-C", path, "rev-parse", "HEAD"])).stdout.trim()).toBe(originalHead);
expect((await execute("git", ["-C", path, "status", "--porcelain"])).stdout).toBe("");
const branchHead = (await execute("git", ["-C", path, "rev-parse", run.branch])).stdout.trim();
expect(branchHead).not.toBe(originalHead);
expect((await execute("git", ["-C", path, "diff", "--name-only", `${originalHead}..${branchHead}`])).stdout.trim())
.toBe("src/example.ts");
expect(await readFile(join(path, "src", "example.ts"), "utf8")).toBe("export const value = 1;\n");
});
test("rejects unreviewed or dirty results without changing the registered checkout", async () => {
const { path, workspace, policy } = await repositoryFixture();
const originalHead = (await execute("git", ["-C", path, "rev-parse", "HEAD"])).stdout.trim();
const run = await policy.begin(task(workspace.id));
await writeFile(join(run.path, "src", "example.ts"), "export const value = 3;\n");
policy.observeDelegation(run, "implementer", true);
await expect(policy.accept(run)).rejects.toThrow("implementer and reviewer");
expect((await execute("git", ["-C", path, "rev-parse", "HEAD"])).stdout.trim()).toBe(originalHead);
expect((await execute("git", ["-C", path, "status", "--porcelain"])).stdout).toBe("");
await expect(execute("git", ["-C", path, "show-ref", "--verify", `refs/heads/${run.branch}`])).rejects.toThrow();
});
test("rejects a clean reviewed commit that changes a protected path", async () => {
const { path, workspace, policy } = await repositoryFixture();
const originalHead = (await execute("git", ["-C", path, "rev-parse", "HEAD"])).stdout.trim();
const run = await policy.begin(task(workspace.id));
await writeFile(join(run.path, "src", "agent.ts"), "export const weakened = true;\n");
await execute("git", ["-C", run.path, "add", "src/agent.ts"]);
await execute("git", ["-C", run.path, "commit", "--no-verify", "--no-gpg-sign", "-m", "weaken-policy"]);
policy.observeDelegation(run, "implementer", true);
policy.observeDelegation(run, "reviewer", true);
await expect(policy.accept(run)).rejects.toThrow("protected path");
expect((await execute("git", ["-C", path, "rev-parse", "HEAD"])).stdout.trim()).toBe(originalHead);
await expect(execute("git", ["-C", path, "show-ref", "--verify", `refs/heads/${run.branch}`])).rejects.toThrow();
});
test("refuses a dirty registered checkout without touching user work", async () => {
const { path, workspace, policy } = await repositoryFixture();
const userPath = join(path, "user-work.txt");
await writeFile(userPath, "preserve me\n");
await expect(policy.begin(task(workspace.id))).rejects.toThrow("clean registered Git workspace");
expect(await readFile(userPath, "utf8")).toBe("preserve me\n");
});
test("does not start a second executor while the old self-update run is unsettled", async () => {
const { workspace, policy } = await repositoryFixture();
const sourceTask = task(workspace.id);
const run = await policy.begin(sourceTask);
await expect(policy.begin(sourceTask)).rejects.toThrow("already executing");
await policy.reject(run);
});
test("accepts a reviewed clean no-op without creating a branch", async () => {
const { path, workspace, policy } = await repositoryFixture();
const run = await policy.begin(task(workspace.id));
policy.observeDelegation(run, "implementer", true);
policy.observeDelegation(run, "reviewer", true);
await policy.accept(run);
await expect(execute("git", ["-C", path, "show-ref", "--verify", `refs/heads/${run.branch}`])).rejects.toThrow();
expect((await execute("git", ["-C", path, "status", "--porcelain"])).stdout).toBe("");
});
test("does not retain a local branch when cancellation wins completion", async () => {
const { path, workspace, policy } = await repositoryFixture();
const run = await policy.begin(task(workspace.id));
await writeFile(join(run.path, "src", "example.ts"), "export const value = 4;\n");
await execute("git", ["-C", run.path, "add", "src/example.ts"]);
await execute("git", ["-C", run.path, "commit", "--no-verify", "--no-gpg-sign", "-m", "cancelled"]);
policy.observeDelegation(run, "implementer", true);
policy.observeDelegation(run, "reviewer", true);
const controller = new AbortController();
controller.abort();
await expect(policy.accept(run, controller.signal)).rejects.toThrow();
await expect(execute("git", ["-C", path, "show-ref", "--verify", `refs/heads/${run.branch}`])).rejects.toThrow();
});
});