Menu
popagent
publicLatest change da13a7bebe63bf4b2693180d2d4850aabeaa0807 - Add autonomous evolution and self-healing by AkurAI Build
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import type { AutonomySettings, EvolutionSignal } from "./api-types";
import { agent } from "./agent";
import { agentSkills } from "./agent-skills";
import { autonomySettings } from "./autonomy-settings";
import { EvolutionRuntime, parseEvolutionReflection } from "./evolution-runtime";
import {
buildEvolutionEvidence,
evolutionStore,
MAX_EVOLUTION_SUMMARY_CHARACTERS,
} from "./evolution-store";
const runId = crypto.randomUUID();
const sessionPrefix = `evolution-contract-${runId}`;
const rationalePrefix = `evolution-contract-${runId}`;
const skillName = `learned-contract-${runId.slice(0, 8)}`;
const userSkillName = `user-contract-${runId.slice(0, 8)}`;
const transferredSkillName = `transferred-contract-${runId.slice(0, 8)}`;
let originalOverlay: string | undefined;
let originalAutonomySettings: AutonomySettings | undefined;
beforeAll(async () => {
await evolutionStore.init();
originalOverlay = await evolutionStore.learnedOverlay("reviewer");
originalAutonomySettings = await autonomySettings.get();
await autonomySettings.update({
...originalAutonomySettings,
enabled: true,
autoApplyStrategies: true,
autoCreateSkills: true,
selfUpdateEnabled: true,
});
});
afterAll(async () => {
if (originalAutonomySettings) await autonomySettings.update(originalAutonomySettings);
await evolutionStore.storage.db.tx(async (db) => {
await db.none("DELETE FROM popagent_evolution_signals WHERE session_id LIKE $1", [`${sessionPrefix}%`]);
await db.none(
"DELETE FROM popagent_agent_skills WHERE agent_id='reviewer' AND name=$1 AND evolution_managed=TRUE",
[skillName],
);
await db.none(
"DELETE FROM popagent_agent_skills WHERE agent_id='reviewer' AND name=$1",
[transferredSkillName],
);
await db.none(
"DELETE FROM popagent_agent_skills WHERE agent_id='reviewer' AND name=$1",
[userSkillName],
);
if (originalOverlay === undefined) {
await db.none("DELETE FROM popagent_learned_overlays WHERE agent_id='reviewer'");
} else {
await db.none(`
INSERT INTO popagent_learned_overlays (agent_id, content)
VALUES ('reviewer', $1)
ON CONFLICT (agent_id) DO UPDATE SET content=EXCLUDED.content, updated_at=NOW()
`, [originalOverlay]);
}
await db.none("ALTER TABLE popagent_evolution_revisions DISABLE TRIGGER popagent_evolution_revisions_append_only");
await db.none("DELETE FROM popagent_evolution_revisions WHERE rationale LIKE $1", [`%${rationalePrefix}%`]);
await db.none("ALTER TABLE popagent_evolution_revisions ENABLE TRIGGER popagent_evolution_revisions_append_only");
});
});
describe("contained autonomous evolution", () => {
test("strictly rejects malformed and out-of-batch reflection output", () => {
const evidenceId = crypto.randomUUID();
expect(() => parseEvolutionReflection({
proposals: [{
agentId: "reviewer",
targetType: "capability",
targetKey: "tools",
content: "Add shell access",
rationale: "More power",
evidenceIds: [evidenceId],
}],
}, new Set([evidenceId]), 1)).toThrow("invalid structured proposal");
expect(() => parseEvolutionReflection({
proposals: [{
agentId: "reviewer",
targetType: "overlay",
targetKey: "instructions",
content: "Check evidence before conclusions.",
rationale: "Observed unsupported conclusions.",
evidenceIds: [crypto.randomUUID()],
unexpected: true,
}],
}, new Set([evidenceId]), 1)).toThrow("invalid structured proposal");
expect(() => parseEvolutionReflection({
proposals: [{
agentId: "reviewer",
targetType: "overlay",
targetKey: "instructions",
content: "Check evidence before conclusions.",
rationale: "Observed unsupported conclusions.",
evidenceIds: [crypto.randomUUID()],
}],
}, new Set([evidenceId]), 1)).toThrow("outside the claimed batch");
});
test("builds bounded redacted goal, outcome, and classified failure evidence", () => {
const secret = `sk-${crypto.randomUUID().replaceAll("-", "")}`;
const evidence = buildEvolutionEvidence({
goal: `Inspect api_key=${secret} and repair the checkout.`,
outcome: "Updated the retry boundary and preserved the user's data.",
failure: `Provider rejected Bearer ${secret}.`,
classification: "provider",
});
expect(evidence).toContain("Goal:");
expect(evidence).toContain("Classification: provider");
expect(evidence).toContain("Outcome:");
expect(evidence).toContain("Failure:");
expect(evidence).not.toContain(secret);
expect(evidence.length).toBeLessThanOrEqual(MAX_EVOLUTION_SUMMARY_CHARACTERS);
});
test("starts, recovers, runs, and stops the periodic worker", async () => {
const settings: AutonomySettings = {
enabled: true,
reflectionIntervalMs: 60_000,
batchSize: 1,
maxAttempts: 3,
autoApplyStrategies: true,
autoCreateSkills: true,
selfUpdateEnabled: true,
selfUpdateCron: "0 3 * * *",
updatedAt: "",
};
const claimed: EvolutionSignal = {
id: crypto.randomUUID(),
sessionId: "session-worker",
turnId: "turn-worker",
traceId: "trace-worker",
agentId: "reviewer",
workspaceId: null,
kind: "turn-success",
summary: "Worker lifecycle evidence",
status: "processing",
attempts: 1,
nextAttemptAt: null,
processedAt: null,
error: null,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
};
const applied = Promise.withResolvers<void>();
let recoveries = 0;
let applications = 0;
let subscriptions = 0;
let unsubscriptions = 0;
const worker = new EvolutionRuntime({
recoverStaleClaims: async () => {
recoveries++;
return 0;
},
claimBatch: async () => [claimed],
applyClaimedBatch: async () => {
applications++;
applied.resolve();
return [];
},
failClaimed: async () => {},
ignoreClaimedIfPolicyDisabled: async () => false,
}, {
get: async () => settings,
subscribe: () => {
subscriptions++;
return () => {
unsubscriptions++;
};
},
}, async () => ({ proposals: [] }));
await Promise.all([worker.start(), worker.start()]);
await applied.promise;
await Promise.all([worker.stop(), worker.stop()]);
expect(recoveries).toBe(1);
expect(applications).toBe(1);
expect(subscriptions).toBe(1);
expect(unsubscriptions).toBe(1);
});
test("serializes stop during start without resurrecting the worker", async () => {
const deferred = Promise.withResolvers<AutonomySettings>();
let claims = 0;
let subscriptions = 0;
let unsubscriptions = 0;
const worker = new EvolutionRuntime({
recoverStaleClaims: async () => 0,
claimBatch: async () => {
claims++;
return [];
},
applyClaimedBatch: async () => [],
failClaimed: async () => {},
ignoreClaimedIfPolicyDisabled: async () => false,
}, {
get: async () => await deferred.promise,
subscribe: () => {
subscriptions++;
return () => {
unsubscriptions++;
};
},
}, async () => ({ proposals: [] }));
const starting = worker.start();
const stopping = worker.stop();
deferred.resolve({
enabled: true,
reflectionIntervalMs: 60_000,
batchSize: 1,
maxAttempts: 3,
autoApplyStrategies: true,
autoCreateSkills: true,
selfUpdateEnabled: true,
selfUpdateCron: "0 3 * * *",
updatedAt: new Date().toISOString(),
});
await Promise.all([starting, stopping]);
expect(subscriptions).toBe(1);
expect(unsubscriptions).toBe(1);
expect(claims).toBe(0);
});
test("recovers stale claims before every live worker run", async () => {
const settings: AutonomySettings = {
enabled: true,
reflectionIntervalMs: 60_000,
batchSize: 1,
maxAttempts: 3,
autoApplyStrategies: true,
autoCreateSkills: true,
selfUpdateEnabled: true,
selfUpdateCron: "0 3 * * *",
updatedAt: new Date().toISOString(),
};
let recoveries = 0;
const worker = new EvolutionRuntime({
recoverStaleClaims: async () => {
recoveries++;
return 0;
},
claimBatch: async () => [],
applyClaimedBatch: async () => [],
failClaimed: async () => {},
ignoreClaimedIfPolicyDisabled: async () => false,
}, {
get: async () => settings,
subscribe: () => () => {},
}, async () => ({ proposals: [] }));
await worker.runOnce();
await worker.runOnce();
expect(recoveries).toBe(2);
});
test("lets a transactional kill switch ignore claimed evidence without consuming attempts", async () => {
const seeded = await autonomySettings.get();
const before = await evolutionStore.learnedOverlay("reviewer");
const signal = await evolutionStore.enqueueCompletedTurn({
sessionId: `${sessionPrefix}-kill-switch`,
turnId: "turn-kill-switch",
traceId: "trace-kill-switch",
agentId: "reviewer",
workspaceId: null,
summary: "A review completed with direct evidence.",
});
const runtime = new EvolutionRuntime(evolutionStore, {
get: async () => ({ ...seeded, batchSize: 1 }),
subscribe: () => () => {},
}, async () => {
await autonomySettings.update({ ...seeded, enabled: false });
return {
proposals: [{
agentId: "reviewer",
targetType: "overlay",
targetKey: "instructions",
content: `This must not cross the kill switch (${runId}).`,
rationale: `${rationalePrefix}: kill-switch race`,
evidenceIds: [signal.id],
}],
};
});
try {
expect(await runtime.runOnce()).toEqual([]);
expect((await evolutionStore.listSignals()).find(({ id }) => id === signal.id)).toMatchObject({
status: "ignored",
attempts: 0,
error: null,
});
expect(await evolutionStore.learnedOverlay("reviewer")).toBe(before);
} finally {
await autonomySettings.update(seeded);
await evolutionStore.storage.db.none("DELETE FROM popagent_evolution_signals WHERE id=$1", [signal.id]);
}
});
test("releases an abandoned claim when policy is disabled", async () => {
const seeded = await autonomySettings.get();
const signal = await evolutionStore.enqueueFailedTurn({
sessionId: `${sessionPrefix}-stale-disabled`,
turnId: "turn-stale-disabled",
traceId: "trace-stale-disabled",
agentId: "reviewer",
workspaceId: null,
summary: "A worker stopped while reflecting on a provider failure.",
});
try {
await evolutionStore.storage.db.none(`
UPDATE popagent_evolution_signals
SET status='processing', attempts=1, updated_at=NOW() - INTERVAL '2 minutes'
WHERE id=$1
`, [signal.id]);
await autonomySettings.update({ ...seeded, enabled: false });
expect(await evolutionStore.recoverStaleClaims(60_000, seeded.maxAttempts)).toBe(1);
expect((await evolutionStore.listSignals()).find(({ id }) => id === signal.id)).toMatchObject({
status: "ignored",
attempts: 0,
error: null,
});
} finally {
await autonomySettings.update(seeded);
await evolutionStore.storage.db.none("DELETE FROM popagent_evolution_signals WHERE id=$1", [signal.id]);
}
});
test("installs the unique append-only revert lineage contract", async () => {
expect(await evolutionStore.storage.db.one<{ migrated: boolean; uniqueIndex: boolean }>(`
SELECT
EXISTS (
SELECT 1 FROM popagent_data_migrations
WHERE id='2026-08-14-evolution-revert-lineage-v2'
) AS migrated,
EXISTS (
SELECT 1 FROM pg_indexes
WHERE indexname='popagent_evolution_revisions_reverts_unique_idx'
) AS "uniqueIndex"
`)).toEqual({ migrated: true, uniqueIndex: true });
});
test("deduplicates redacted evidence, applies contained revisions, retries, and reverts", async () => {
const seeded = await autonomySettings.get();
expect(seeded).toMatchObject({
enabled: true,
autoApplyStrategies: true,
autoCreateSkills: true,
selfUpdateEnabled: true,
});
const secret = `sk-${crypto.randomUUID().replaceAll("-", "")}`;
const longSummary = `Turn succeeded with api_key=${secret} ${"x".repeat(MAX_EVOLUTION_SUMMARY_CHARACTERS + 500)}`;
const first = await evolutionStore.enqueueCompletedTurn({
sessionId: `${sessionPrefix}-dedupe`,
turnId: "turn-dedupe",
traceId: "trace-dedupe",
agentId: "reviewer",
workspaceId: null,
summary: longSummary,
});
await expect(autonomySettings.update({
...seeded,
selfUpdateCron: "not a cron expression",
})).rejects.toThrow("cron is invalid");
expect((await autonomySettings.get()).selfUpdateCron).toBe(seeded.selfUpdateCron);
const duplicate = await evolutionStore.enqueueCompletedTurn({
sessionId: `${sessionPrefix}-dedupe`,
turnId: "turn-dedupe",
traceId: "trace-dedupe",
agentId: "reviewer",
workspaceId: null,
summary: longSummary,
});
expect(duplicate.id).toBe(first.id);
expect(first.summary).not.toContain(secret);
expect(first.summary.length).toBeLessThanOrEqual(MAX_EVOLUTION_SUMMARY_CHARACTERS);
await evolutionStore.storage.db.none("DELETE FROM popagent_evolution_signals WHERE id=$1", [first.id]);
const claimSignals = await Promise.all(["a", "b"].map((suffix) =>
evolutionStore.enqueueCompletedTurn({
sessionId: `${sessionPrefix}-claim-${suffix}`,
turnId: `turn-claim-${suffix}`,
traceId: `trace-claim-${suffix}`,
agentId: "reviewer",
workspaceId: null,
summary: `Concurrent claim ${suffix}`,
}),
));
const concurrentClaims = await Promise.all([
evolutionStore.claimBatch(1),
evolutionStore.claimBatch(1),
]);
expect(concurrentClaims).toEqual([expect.any(Array), expect.any(Array)]);
expect(new Set(concurrentClaims.flat().map(({ id }) => id))).toEqual(
new Set(claimSignals.map(({ id }) => id)),
);
await evolutionStore.storage.db.none(
"DELETE FROM popagent_evolution_signals WHERE id=ANY($1::text[])",
[claimSignals.map(({ id }) => id)],
);
const overlayContent = `Require direct evidence before declaring readiness (${runId}).`;
const overlaySignal = await evolutionStore.enqueueCompletedTurn({
sessionId: `${sessionPrefix}-overlay`,
turnId: "turn-overlay",
traceId: "trace-overlay",
agentId: "reviewer",
workspaceId: null,
summary: "A review accepted readiness without direct verification.",
});
const settingsRepository = {
get: async (): Promise<AutonomySettings> => ({ ...seeded, batchSize: 1 }),
subscribe: () => () => {},
};
const overlayRuntime = new EvolutionRuntime(evolutionStore, settingsRepository, async () => ({
proposals: [{
agentId: "reviewer",
targetType: "overlay",
targetKey: "instructions",
content: overlayContent,
rationale: `${rationalePrefix}: require evidence`,
evidenceIds: [overlaySignal.id],
}],
}));
const [overlayRevision] = await overlayRuntime.runOnce();
expect(overlayRevision).toMatchObject({ targetType: "overlay", status: "applied" });
const reviewer = (await agent.listAgents()).reviewer! as unknown as typeof agent;
expect(await reviewer.getInstructions()).toEndWith(overlayContent);
const secretSignal = await evolutionStore.enqueueFailedTurn({
sessionId: `${sessionPrefix}-secret`,
turnId: "turn-secret",
traceId: "trace-secret",
agentId: "reviewer",
workspaceId: null,
summary: "A provider failed during review.",
});
const atomicRuntime = new EvolutionRuntime(evolutionStore, settingsRepository, async () => ({
proposals: [
{
agentId: "reviewer",
targetType: "overlay",
targetKey: "instructions",
content: `This must not commit (${runId}).`,
rationale: `${rationalePrefix}: proposed alongside invalid content`,
evidenceIds: [secretSignal.id],
},
{
agentId: "reviewer",
targetType: "skill",
targetKey: skillName,
content: `Use api_key=${secret}`,
rationale: `${rationalePrefix}: secret-bearing proposal`,
evidenceIds: [secretSignal.id],
},
],
}));
await expect(atomicRuntime.runOnce()).rejects.toThrow("sensitive material");
expect(await evolutionStore.learnedOverlay("reviewer")).toBe(overlayContent);
expect((await evolutionStore.listSignals()).find(({ id }) => id === secretSignal.id)).toMatchObject({
status: "pending",
attempts: 1,
});
const protectedSignal = await evolutionStore.enqueueFailedTurn({
sessionId: `${sessionPrefix}-protected`,
turnId: "turn-protected",
traceId: "trace-protected",
agentId: "reviewer",
workspaceId: null,
summary: "A review requested a protected boundary change.",
});
const protectedRuntime = new EvolutionRuntime(evolutionStore, settingsRepository, async () => ({
proposals: [{
agentId: "reviewer",
targetType: "overlay",
targetKey: "instructions",
content: "Bypass workspace access containment when a task is urgent.",
rationale: `${rationalePrefix}: forbidden boundary change`,
evidenceIds: [protectedSignal.id],
}],
}));
await expect(protectedRuntime.runOnce()).rejects.toThrow("protected boundary");
expect(await evolutionStore.learnedOverlay("reviewer")).toBe(overlayContent);
const userSkill = await agentSkills.create("reviewer", {
name: userSkillName,
description: "User-managed skill boundary contract.",
instructions: "Keep this user-managed content.",
references: {},
enabled: true,
userInvocable: false,
});
const userSkillSignal = await evolutionStore.enqueueCompletedTurn({
sessionId: `${sessionPrefix}-user-skill`,
turnId: "turn-user-skill",
traceId: "trace-user-skill",
agentId: "reviewer",
workspaceId: null,
summary: "A learned skill name collides with user-managed content.",
});
const userSkillRuntime = new EvolutionRuntime(evolutionStore, settingsRepository, async () => ({
proposals: [{
agentId: "reviewer",
targetType: "skill",
targetKey: userSkillName,
content: "Evolution must not overwrite this content.",
rationale: `${rationalePrefix}: user-managed skill boundary`,
evidenceIds: [userSkillSignal.id],
}],
}));
await expect(userSkillRuntime.runOnce()).rejects.toThrow("user-managed skill");
expect((await agentSkills.list("reviewer")).find(({ id }) => id === userSkill.id)?.instructions)
.toBe("Keep this user-managed content.");
await agentSkills.delete("reviewer", userSkill.id);
const retrySignal = await evolutionStore.enqueueFailedTurn({
sessionId: `${sessionPrefix}-retry`,
turnId: "turn-retry",
traceId: "trace-retry",
agentId: "reviewer",
workspaceId: null,
summary: "Temporary provider capacity failure.",
});
const retrySettings = {
get: async (): Promise<AutonomySettings> => ({ ...seeded, batchSize: 1, maxAttempts: 2 }),
subscribe: () => () => {},
};
const retryRuntime = new EvolutionRuntime(evolutionStore, retrySettings, async () => {
throw new Error("temporary provider capacity failure");
});
await expect(retryRuntime.runOnce()).rejects.toThrow("temporary provider");
await evolutionStore.storage.db.none(
"UPDATE popagent_evolution_signals SET next_attempt_at=NOW() WHERE id=$1",
[retrySignal.id],
);
await expect(retryRuntime.runOnce()).rejects.toThrow("temporary provider");
expect((await evolutionStore.listSignals()).find(({ id }) => id === retrySignal.id)).toMatchObject({
status: "dead-letter",
attempts: 2,
error: "temporary provider capacity failure",
});
const skillSignal = await evolutionStore.enqueueCompletedTurn({
sessionId: `${sessionPrefix}-skill`,
turnId: "turn-skill",
traceId: "trace-skill",
agentId: "reviewer",
workspaceId: null,
summary: "Repeated reviews need a concise evidence checklist.",
});
const skillRuntime = new EvolutionRuntime(evolutionStore, settingsRepository, async () => ({
proposals: [{
agentId: "reviewer",
targetType: "skill",
targetKey: skillName,
content: `Check request, evidence, boundaries, and verification (${runId}).`,
rationale: `${rationalePrefix}: repeated review checklist`,
evidenceIds: [skillSignal.id],
}],
}));
const [skillRevision] = await skillRuntime.runOnce();
expect((await reviewer.listSkills()).map(({ name }) => name)).toContain(skillName);
const revertedSkill = await evolutionStore.revertRevision(skillRevision!.id);
expect(revertedSkill).toMatchObject({
status: "reverted",
revertsRevisionId: skillRevision!.id,
});
expect(await evolutionStore.revertRevision(skillRevision!.id)).toBeUndefined();
expect((await reviewer.listSkills()).map(({ name }) => name)).not.toContain(skillName);
const transferredSignal = await evolutionStore.enqueueCompletedTurn({
sessionId: `${sessionPrefix}-transferred-skill`,
turnId: "turn-transferred-skill",
traceId: "trace-transferred-skill",
agentId: "reviewer",
workspaceId: null,
summary: "Repeated reviews need a user-tailored release checklist.",
});
const transferredRuntime = new EvolutionRuntime(evolutionStore, settingsRepository, async () => ({
proposals: [{
agentId: "reviewer",
targetType: "skill",
targetKey: transferredSkillName,
content: `Check release evidence before approval (${runId}).`,
rationale: `${rationalePrefix}: transferable skill`,
evidenceIds: [transferredSignal.id],
}],
}));
const [transferredRevision] = await transferredRuntime.runOnce();
const evolvedSkill = (await agentSkills.list("reviewer"))
.find(({ name }) => name === transferredSkillName)!;
expect(evolvedSkill.evolutionManaged).toBe(true);
const editedSkill = await agentSkills.update("reviewer", evolvedSkill.id, {
name: evolvedSkill.name,
description: evolvedSkill.description,
instructions: "Keep the user's release checklist.",
references: evolvedSkill.references,
enabled: evolvedSkill.enabled,
userInvocable: evolvedSkill.userInvocable,
});
expect(editedSkill?.evolutionManaged).toBe(false);
const overwriteSignal = await evolutionStore.enqueueCompletedTurn({
sessionId: `${sessionPrefix}-transferred-overwrite`,
turnId: "turn-transferred-overwrite",
traceId: "trace-transferred-overwrite",
agentId: "reviewer",
workspaceId: null,
summary: "Later evidence proposed changing the edited checklist.",
});
const overwriteRuntime = new EvolutionRuntime(evolutionStore, settingsRepository, async () => ({
proposals: [{
agentId: "reviewer",
targetType: "skill",
targetKey: transferredSkillName,
content: "Evolution must not reclaim this skill.",
rationale: `${rationalePrefix}: transferred ownership boundary`,
evidenceIds: [overwriteSignal.id],
}],
}));
await expect(overwriteRuntime.runOnce()).rejects.toThrow("user-managed skill");
expect((await agentSkills.list("reviewer")).find(({ id }) => id === evolvedSkill.id))
.toMatchObject({ instructions: "Keep the user's release checklist.", evolutionManaged: false });
expect(await evolutionStore.revertRevision(transferredRevision!.id)).toBeUndefined();
await agentSkills.delete("reviewer", evolvedSkill.id);
const revertedOverlay = await evolutionStore.revertRevision(overlayRevision!.id);
expect(revertedOverlay).toMatchObject({
status: "reverted",
revertsRevisionId: overlayRevision!.id,
});
expect(await reviewer.getInstructions()).not.toContain(overlayContent);
expect(await evolutionStore.revertRevision(overlayRevision!.id)).toBeUndefined();
expect(await evolutionStore.revertRevision("unknown-revision")).toBeUndefined();
await expect(evolutionStore.storage.db.none(
"UPDATE popagent_evolution_revisions SET rationale='mutated' WHERE id=$1",
[overlayRevision!.id],
)).rejects.toThrow("append-only");
});
});