popagent

Latest change d5a81ac70da6d50a1569e53ed1ca90ed4e9f707d - Support local Qwen chat streams by AkurAI Build

import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
import { appLogger } from "./observability";
import type { ModelCatalog } from "./api-types";

const BASE_URL = process.env.NINEROUTER_URL ?? "http://127.0.0.1:20128/v1";
const API_KEY = process.env.NINEROUTER_API_KEY ?? "";

/** llama.cpp's Qwen template accepts exactly one leading system message. */
export function normalizeChatRequest(request: Record<string, unknown>): Record<string, unknown> {
  if (typeof request.model !== "string" || !/\/qwen3\.5-4b$/i.test(request.model) || !Array.isArray(request.messages)) return request;
  const messages = request.messages as Array<{ role?: unknown; content?: unknown }>;
  let systemCount = 0;
  while (messages[systemCount]?.role === "system") systemCount++;
  if (systemCount < 2 || messages.slice(0, systemCount).some((message) => typeof message.content !== "string")) return request;
  return {
    ...request,
    messages: [
      { ...messages[0], content: messages.slice(0, systemCount).map((message) => message.content).join("\n\n") },
      ...messages.slice(systemCount),
    ],
  };
}

const ninerouter = createOpenAICompatible({
  name: "9router",
  baseURL: BASE_URL,
  apiKey: API_KEY,
  transformRequestBody: normalizeChatRequest,
});

/** Language model for one explicit 9Router model id. */
export function resolveModel(modelId: string) {
  return ninerouter.chatModel(modelId);
}

/** Embedding model routed through the same local OpenAI-compatible gateway. */
export function resolveEmbeddingModel(
  modelId = process.env.POPAGENT_EMBEDDING_MODEL ?? "text-embedding-3-small",
) {
  return ninerouter.embeddingModel(modelId);
}


/** Selectable model ids and their context limits from the 9router gateway. */
export async function listModelCatalog(): Promise<ModelCatalog> {
  const res = await fetch(`${BASE_URL}/models`, {
    headers: { authorization: `Bearer ${API_KEY}` },
    signal: AbortSignal.timeout(5000),
  });
  if (!res.ok) throw new Error(`9router /models failed: ${res.status}`);
  const { data } = (await res.json()) as {
    data: {
      id: string;
      context_length?: number;
      max_input_tokens?: number;
      capabilities?: { contextWindow?: number };
    }[];
  };
  const contextWindows: Record<string, number> = {};
  for (const model of data) {
    const limit =
      model.context_length ??
      model.max_input_tokens ??
      model.capabilities?.contextWindow ??
      0;
    contextWindows[model.id] = Math.max(contextWindows[model.id] ?? 0, limit);
  }
  return {
    models: data.map((model) => model.id),
    contextWindows,
  };
}

let catalogCache: { catalog: ModelCatalog; expires: number } | undefined;
export async function isKnownModel(model: string): Promise<boolean> {
  if (catalogCache && catalogCache.expires > Date.now()) return catalogCache.catalog.models.includes(model);
  try {
    const catalog = await listModelCatalog();
    catalogCache = { catalog, expires: Date.now() + 60_000 };
    return catalog.models.includes(model);
  } catch (error) {
    appLogger().warn("model.catalog.unavailable", { error });
    return true;
  }
}