popagent

Latest change 49f74360a0212c33a7c6b76450380f567839ebaa - Fall back on upstream availability failures and show first-response wait by AkurAI Build

import {
  DEFAULT_MODEL_ROUTING,
  modelSpec,
  parseModelSpec,
  ROUTING_MODEL_ID,
  type AgentRuntimeSettings,
  type ModelRouting,
} from "./api-types";
import { agentRuntimeSettings } from "./agent-runtime-settings";
import { appLogger } from "./observability";

type RoutingSource = { get(): Promise<Pick<AgentRuntimeSettings, "defaultModel" | "modelRouting">> };

export type ModelRouteHealth = {
  spec: string;
  model: string;
  reasoningEffort: string;
  cooldownUntil: string | null;
  lastError: string | null;
  picks: number;
};

/**
 * Resolves the `policy/routing` pseudo model to one concrete `model@effort`
 * spec. `fallback` always prefers the first healthy route in order;
 * `round-robin` rotates through healthy routes. A route that fails with a
 * rate limit or an unknown-model error leaves rotation for `cooldownMs`.
 * State is per process: a restart clears cooldowns, which is the desired
 * behaviour for a bounded quota window.
 */
export class ModelRouter {
  private cooldowns = new Map<string, { until: number; error: string }>();
  private picks = new Map<string, number>();
  private cursor = 0;

  constructor(private readonly settings: RoutingSource = agentRuntimeSettings, private readonly now: () => number = Date.now) {}

  /** True when the requested model must be resolved through the policy. */
  static isRouted(requested: string | null | undefined): boolean {
    return requested === ROUTING_MODEL_ID;
  }

  /** Effective `model@effort` spec for one attempt. Non-routed ids pass through untouched. */
  async pick(requested: string): Promise<string> {
    if (!ModelRouter.isRouted(requested)) return requested;
    const settings = await this.settings.get();
    const routing = settings.modelRouting ?? DEFAULT_MODEL_ROUTING;
    const routes = routing.enabled ? routing.routes : [];
    if (routes.length === 0) {
      appLogger().warn("model.routing.unconfigured", { fallback: settings.defaultModel });
      return settings.defaultModel;
    }
    const specs = routes.map(modelSpec);
    const healthy = specs.filter((spec) => !this.cooling(spec));
    // Every route is cooling down: take the one whose cooldown ends first
    // rather than refusing to work at all.
    const candidates = healthy.length ? healthy : [...specs].sort((a, b) => (this.cooldowns.get(a)?.until ?? 0) - (this.cooldowns.get(b)?.until ?? 0)).slice(0, 1);
    const chosen = routing.strategy === "round-robin"
      ? candidates[this.cursor++ % candidates.length]!
      : candidates[0]!;
    this.picks.set(chosen, (this.picks.get(chosen) ?? 0) + 1);
    return chosen;
  }

  /**
   * Record a failed attempt for one effective spec. Quota, unknown-model, and
   * upstream availability failures (5xx, gateway/connect timeouts, overload)
   * take a route out of rotation so the retry lands on the next route; errors
   * that belong to the task itself (aborts, validation, permissions, hook
   * denials) leave the route alone.
   */
  async reportFailure(spec: string, error: unknown): Promise<boolean> {
    const message = error instanceof Error ? error.message : String(error);
    const status = Number((error as { statusCode?: unknown; status?: unknown } | null)?.statusCode ?? (error as { status?: unknown } | null)?.status);
    const rateLimited = status === 429 || /\brate[- ]?limit|\btoo many requests|\busage limit|\bquota\b/i.test(message);
    const unknownModel = status === 404 || /\bunknown model|\bmodel .* not found|\bno such model|\bunsupported model/i.test(message);
    const unavailable = [408, 500, 502, 503, 504, 529].includes(status)
      || /\[(?:408|50[0-4]|529)\]|\bconnect timeout|\bfetch failed|\bECONN|\bETIMEDOUT|\btimed? ?out\b|\boverloaded?\b|\bunavailable\b|\bbad gateway\b|\bupstream\b/i.test(message);
    const ownFault = /\baborted?\b|\bcancell?ed\b|\bvalidation\b|\bpermission\b|\bunauthori[sz]ed\b|\bforbidden\b|\bhook\b/i.test(message) && !rateLimited && !unavailable;
    if (ownFault || (!rateLimited && !unknownModel && !unavailable)) return false;
    const routing = (await this.settings.get()).modelRouting ?? DEFAULT_MODEL_ROUTING;
    const specs = new Set(routing.routes.map(modelSpec));
    if (!specs.has(spec)) return false;
    const resetMinutes = /reset after (\d+)\s*m/i.exec(message);
    // Availability blips are usually short: park the route for a fraction of
    // the quota cooldown (at least two minutes) unless the gateway says otherwise.
    const base = rateLimited || unknownModel ? routing.cooldownMs : Math.max(120_000, Math.round(routing.cooldownMs / 5));
    const cooldownMs = resetMinutes ? Math.max(base, Number(resetMinutes[1]) * 60_000) : base;
    this.cooldowns.set(spec, { until: this.now() + cooldownMs, error: message.slice(0, 300) });
    appLogger().warn("model.routing.cooldown", { spec, cooldownMs, reason: rateLimited ? "rate-limit" : unknownModel ? "unknown-model" : "unavailable" });
    return true;
  }

  /** Per-route state for the settings page. */
  async health(): Promise<{ strategy: ModelRouting["strategy"]; enabled: boolean; routes: ModelRouteHealth[] }> {
    const routing = (await this.settings.get()).modelRouting ?? DEFAULT_MODEL_ROUTING;
    return {
      strategy: routing.strategy,
      enabled: routing.enabled,
      routes: routing.routes.map((route) => {
        const spec = modelSpec(route);
        const cooling = this.cooling(spec) ? this.cooldowns.get(spec)! : undefined;
        const parsed = parseModelSpec(spec);
        return {
          spec,
          model: parsed.model,
          reasoningEffort: parsed.reasoningEffort,
          cooldownUntil: cooling ? new Date(cooling.until).toISOString() : null,
          lastError: this.cooldowns.get(spec)?.error ?? null,
          picks: this.picks.get(spec) ?? 0,
        };
      }),
    };
  }

  /** Test seam: forget every cooldown and counter. */
  reset() {
    this.cooldowns.clear();
    this.picks.clear();
    this.cursor = 0;
  }

  private cooling(spec: string): boolean {
    const entry = this.cooldowns.get(spec);
    return entry !== undefined && entry.until > this.now();
  }
}

export const modelRouter = new ModelRouter();