popagent

Latest change 49f74360a0212c33a7c6b76450380f567839ebaa - Fall back on upstream availability failures and show first-response wait by AkurAI Build

import { describe, expect, test } from "bun:test";
import { ModelRouter } from "./model-router";
import { applyReasoningEffort } from "./models";
import { modelSpec, parseModelSpec, ROUTING_MODEL_ID, type ModelRouting } from "./api-types";

const source = (routing: Partial<ModelRouting>, defaultModel = "cc/claude-sonnet-5") => ({
  get: async () => ({
    defaultModel,
    modelRouting: { enabled: true, strategy: "fallback" as const, cooldownMs: 600_000, routes: [], ...routing },
  }),
});
const routes = [
  { model: "cc/claude-sonnet-5", reasoningEffort: "low" as const },
  { model: "cx/gpt-5.6-sol", reasoningEffort: "low" as const },
];

describe("model specs", () => {
  test("encode reasoning effort in the model id and strip it on the wire", () => {
    expect(modelSpec(routes[0]!)).toBe("cc/claude-sonnet-5@low");
    expect(parseModelSpec("cx/gpt-5.6-sol@high")).toEqual({ model: "cx/gpt-5.6-sol", reasoningEffort: "high" });
    expect(parseModelSpec("cx/gpt-5.6-sol")).toEqual({ model: "cx/gpt-5.6-sol", reasoningEffort: "default" });
    expect(applyReasoningEffort({ model: "cc/claude-sonnet-5@low", messages: [] }))
      .toEqual({ model: "cc/claude-sonnet-5", reasoning_effort: "low", messages: [] });
    expect(applyReasoningEffort({ model: "cc/claude-sonnet-5", messages: [] })).toEqual({ model: "cc/claude-sonnet-5", messages: [] });
  });
});

describe("ModelRouter", () => {
  test("passes concrete model ids through untouched", async () => {
    const router = new ModelRouter(source({ routes }));
    expect(await router.pick("ag/gemini-3-flash")).toBe("ag/gemini-3-flash");
  });

  test("falls back to the runtime default when routing is disabled or empty", async () => {
    expect(await new ModelRouter(source({ enabled: false, routes })).pick(ROUTING_MODEL_ID)).toBe("cc/claude-sonnet-5");
    expect(await new ModelRouter(source({ routes: [] })).pick(ROUTING_MODEL_ID)).toBe("cc/claude-sonnet-5");
  });

  test("fallback prefers the primary and moves on only after a quota or unknown-model failure", async () => {
    let now = 1_000_000;
    const router = new ModelRouter(source({ routes }), () => now);
    expect(await router.pick(ROUTING_MODEL_ID)).toBe("cc/claude-sonnet-5@low");
    expect(await router.reportFailure("cc/claude-sonnet-5@low", new Error("tool input validation failed"))).toBe(false);
    expect(await router.pick(ROUTING_MODEL_ID)).toBe("cc/claude-sonnet-5@low");
    expect(await router.reportFailure("cc/claude-sonnet-5@low", Object.assign(new Error("[429]: The usage limit has been reached (reset after 30m)"), { statusCode: 429 }))).toBe(true);
    expect(await router.pick(ROUTING_MODEL_ID)).toBe("cx/gpt-5.6-sol@low");
    // The 30 m reset hint extends the configured 10 m cooldown.
    now += 15 * 60_000;
    expect(await router.pick(ROUTING_MODEL_ID)).toBe("cx/gpt-5.6-sol@low");
    now += 16 * 60_000;
    expect(await router.pick(ROUTING_MODEL_ID)).toBe("cc/claude-sonnet-5@low");
    expect(await router.reportFailure("cx/gpt-5.6-sol@low", new Error("Unknown model: cx/gpt-5.6-sol"))).toBe(true);
    const health = await router.health();
    expect(health.routes.map((route) => [route.spec, route.cooldownUntil !== null])).toEqual([
      ["cc/claude-sonnet-5@low", false],
      ["cx/gpt-5.6-sol@low", true],
    ]);
  });

  test("round-robin rotates through healthy routes and never stalls when all are cooling", async () => {
    let now = 5_000;
    const router = new ModelRouter(source({ strategy: "round-robin", routes }), () => now);
    expect(await router.pick(ROUTING_MODEL_ID)).toBe("cc/claude-sonnet-5@low");
    expect(await router.pick(ROUTING_MODEL_ID)).toBe("cx/gpt-5.6-sol@low");
    expect(await router.pick(ROUTING_MODEL_ID)).toBe("cc/claude-sonnet-5@low");
    await router.reportFailure("cc/claude-sonnet-5@low", new Error("rate limit"));
    expect(await router.pick(ROUTING_MODEL_ID)).toBe("cx/gpt-5.6-sol@low");
    expect(await router.pick(ROUTING_MODEL_ID)).toBe("cx/gpt-5.6-sol@low");
    now += 1_000;
    await router.reportFailure("cx/gpt-5.6-sol@low", new Error("rate limit"));
    // Both cooling: the one that recovers first is used rather than nothing.
    expect(await router.pick(ROUTING_MODEL_ID)).toBe("cc/claude-sonnet-5@low");
  });

  test("moves on after upstream availability failures with a shorter cooldown, but not for the task's own faults", async () => {
    let now = 0;
    const router = new ModelRouter(source({ routes, cooldownMs: 900_000 }), () => now);
    expect(await router.reportFailure("cc/claude-sonnet-5@low", new Error("[claude/claude-sonnet-5] [502]: fetch connect timeout (reset after 30s)"))).toBe(true);
    expect(await router.pick(ROUTING_MODEL_ID)).toBe("cx/gpt-5.6-sol@low");
    now += 3 * 60_000 + 1;
    expect(await router.pick(ROUTING_MODEL_ID)).toBe("cc/claude-sonnet-5@low");
    expect(await router.reportFailure("cc/claude-sonnet-5@low", new Error("The operation was aborted"))).toBe(false);
    expect(await router.reportFailure("cc/claude-sonnet-5@low", new Error("Agent role has read-only access: permission denied"))).toBe(false);
    expect(await router.pick(ROUTING_MODEL_ID)).toBe("cc/claude-sonnet-5@low");
  });

  test("ignores failures for specs outside the policy", async () => {
    const router = new ModelRouter(source({ routes }));
    expect(await router.reportFailure("ag/gemini-3-flash", new Error("rate limit"))).toBe(false);
  });
});