popagent

Latest change 248b7c0673ea9d544f9e4738ee4d3e8ec2925924 - Add native BifrOSt Navigator browser provider with per-browser status LEDs by AkurAI Build

import { AgentBrowser, BROWSER_TOOLS, type BrowserToolName } from "@mastra/agent-browser";
import type { Agent } from "@mastra/core/agent";
import { existsSync } from "node:fs";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { chromium } from "playwright-core";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { agentSettings, type AgentSettingsStore } from "./agent-settings";
import { storage } from "./storage";
import type { BrowserHealth, BrowserProviderHealth, BrowserSettings, BrowserSettingsInput } from "./api-types";
import { retryableInit } from "./retryable-init";
import { validateBrowserDestination } from "./browser-policy";
import { BrowserSessionRegistry, type ManagedBrowser } from "./browser-sessions";
import { browserProfiles, type BrowserProfileStore } from "./browser-profiles";
import { callBifrOSt, listBifrOStTools } from "./tools/bifrost";

const BROWSER_COLUMNS = `enabled, provider, scope, viewport_width AS "viewportWidth",
  viewport_height AS "viewportHeight", timeout_ms AS "timeoutMs",
  max_sessions AS "maxSessions", idle_timeout_ms AS "idleTimeoutMs",
  screencast_enabled AS "screencastEnabled", screenshots_enabled AS "screenshotsEnabled",
  multi_tab_enabled AS "multiTabEnabled", forms_enabled AS "formsEnabled",
  dialogs_enabled AS "dialogsEnabled", drag_enabled AS "dragEnabled",
  evaluate_enabled AS "evaluateEnabled", recording_enabled AS "recordingEnabled",
  recording_retention_days AS "recordingRetentionDays",
  recording_max_files AS "recordingMaxFiles", allow_hosts AS "allowHosts",
  deny_hosts AS "denyHosts", updated_at::text AS "updatedAt"`;

const READ_ONLY_BROWSER_EXCLUSIONS = [
  BROWSER_TOOLS.CLICK,
  BROWSER_TOOLS.TYPE,
  BROWSER_TOOLS.PRESS,
  BROWSER_TOOLS.SELECT,
  BROWSER_TOOLS.CLOSE,
  BROWSER_TOOLS.TABS,
  BROWSER_TOOLS.DIALOG,
  BROWSER_TOOLS.DRAG,
  BROWSER_TOOLS.EVALUATE,
] satisfies BrowserToolName[];


type BoundSecretFill = {
  threadId: string;
  ref: string;
  name: string;
  value: string;
  allowedOrigin: string;
};

type PolicyBrowser = AgentBrowser & {
  fillBoundSecret(input: BoundSecretFill): Promise<{ used: true; name: string; origin: string }>;
};

export function assertBoundSecretOrigin(
  pageUrl: string,
  elementOrigin: string,
  allowedOrigin: string,
): string {
  let expected: URL;
  let page: URL;
  try {
    expected = new URL(allowedOrigin);
    page = new URL(pageUrl);
  } catch {
    throw new Error("Secret origin validation failed");
  }
  if (
    expected.protocol !== "https:"
    || expected.origin !== allowedOrigin
    || expected.pathname !== "/"
    || expected.search
    || expected.hash
  ) {
    throw new Error("Secret binding is not one exact HTTPS origin");
  }
  if (page.origin !== allowedOrigin) {
    throw new Error(`Secret is not authorized for the current page origin ${page.origin}`);
  }
  if (elementOrigin !== allowedOrigin) {
    throw new Error(`Secret is not authorized for the target element origin ${elementOrigin}`);
  }
  return allowedOrigin;
}

function capabilityExclusions(settings: BrowserSettings, readOnly: boolean): BrowserToolName[] {
  const excluded = new Set<BrowserToolName>(readOnly ? READ_ONLY_BROWSER_EXCLUSIONS : []);
  if (!settings.screenshotsEnabled) excluded.add(BROWSER_TOOLS.SCREENSHOT);
  if (!settings.multiTabEnabled) excluded.add(BROWSER_TOOLS.TABS);
  if (!settings.formsEnabled) {
    excluded.add(BROWSER_TOOLS.TYPE);
    excluded.add(BROWSER_TOOLS.PRESS);
    excluded.add(BROWSER_TOOLS.SELECT);
  }
  if (!settings.dialogsEnabled) excluded.add(BROWSER_TOOLS.DIALOG);
  if (!settings.dragEnabled) excluded.add(BROWSER_TOOLS.DRAG);
  if (!settings.evaluateEnabled) excluded.add(BROWSER_TOOLS.EVALUATE);
  return [...excluded];
}

async function createPolicyBrowser(
  settings: BrowserSettings,
  readOnly: boolean,
  sessions: BrowserSessionRegistry,
  storageState?: string,
): Promise<PolicyBrowser> {
  const recordingDir = join(process.env.POPAGENT_DATA_DIR ?? join(process.cwd(), "data"), "browser-recordings");
  if (settings.recordingEnabled && !readOnly) await mkdir(recordingDir, { recursive: true, mode: 0o700 });
  const browser = new AgentBrowser({
    headless: true,
    scope: settings.scope,
    viewport: { width: settings.viewportWidth, height: settings.viewportHeight },
    timeout: settings.timeoutMs,
    excludeTools: capabilityExclusions(settings, readOnly),
    ...(storageState ? { storageState } : {}),
    ...(settings.screencastEnabled ? { screencast: {
      format: "jpeg" as const,
      quality: 75,
      maxWidth: 1280,
      maxHeight: 720,
      everyNthFrame: 2,
    } } : {}),
    ...(settings.recordingEnabled && !readOnly ? { recording: { outputDir: recordingDir } } : {}),
  });
  const rules = { allowHosts: settings.allowHosts, denyHosts: settings.denyHosts };
  const goto = browser.goto.bind(browser);
  browser.goto = async (input, threadId) => {
    const result = await validateBrowserDestination(input.url, rules);
    if (!result.allowed) return `destination blocked by browser policy: ${result.reason ?? "destination denied"}` as never;
    const response = await goto(input, threadId);
    if ("url" in response) {
      const final = await validateBrowserDestination(response.url, rules);
      if (!final.allowed) {
        await browser.closeThreadSession(threadId ?? browser.getCurrentThread());
        return `redirect blocked by browser policy: ${final.reason ?? "destination denied"}` as never;
      }
    }
    return response;
  };
  const tabs = browser.tabs.bind(browser);
  browser.tabs = async (input, threadId) => {
    if (input.action === "new" && input.url) {
      const result = await validateBrowserDestination(input.url, rules);
      if (!result.allowed) return `destination blocked by browser policy: ${result.reason ?? "destination denied"}` as never;
    }
    return tabs(input, threadId);
  };

  const protectedContexts = new WeakSet<object>();
  const protectThread = async (threadId: string) => {
    const manager = await browser.getManagerForThread(threadId);
    const context = manager.getContext();
    if (!context || protectedContexts.has(context)) return;
    protectedContexts.add(context);
    await context.route("**/*", async (route) => {
      const url = route.request().url();
      if (!url.startsWith("http://") && !url.startsWith("https://")) return route.continue();
      const result = await validateBrowserDestination(url, rules);
      if (result.allowed) await route.continue();
      else await route.abort("blockedbyclient");
    });
  };

  const trackedMethods = [
    "goto", "snapshot", "screenshot", "click", "type", "press", "select", "scroll",
    "hover", "back", "dialog", "wait", "tabs", "drag", "evaluate",
  ] as const;
  const target = browser as unknown as Record<string, (...args: unknown[]) => Promise<unknown>>;
  const managed: ManagedBrowser = {
    getState: async (threadId) => {
      const state = await browser.getBrowserState(threadId);
      if (!state) return null;
      return {
        currentUrl: state.tabs[state.activeTabIndex]?.url ?? null,
        tabs: state.tabs.map((tab, index) => ({ id: String(index), url: tab.url, title: tab.title ?? tab.url })),
        activeTabIndex: state.activeTabIndex,
      };
    },
    closeThreadSession: async (threadId) => browser.closeThreadSession(threadId),
    startScreencast: async (options) => browser.startScreencast(options),
    injectMouseEvent: async (event, threadId) => browser.injectMouseEvent(event, threadId),
    injectKeyboardEvent: async (event, threadId) => browser.injectKeyboardEvent(event, threadId),
  };
  for (const method of trackedMethods) {
    const original = target[method]?.bind(browser);
    if (!original) continue;
    target[method] = async (...args: unknown[]) => {
      const last = args.at(-1);
      const threadId = typeof last === "string" ? last : browser.getCurrentThread();
      await protectThread(threadId);
      const result = await original(...args);
      await sessions.touch(threadId, readOnly ? "read-only" : "interactive", managed);
      return result;
    };
  }
  const policyBrowser = browser as PolicyBrowser;
  policyBrowser.fillBoundSecret = async ({ threadId, ref, name, value, allowedOrigin }) => {
    if (readOnly || !settings.formsEnabled) {
      throw new Error("Secret use requires interactive browser form access");
    }
    await protectThread(threadId);
    const manager = await browser.getManagerForThread(threadId);
    const page = manager.getPage();
    const locator = manager.getLocatorFromRef(ref);
    if (!locator) throw new Error(`Browser element ${ref} is stale or unavailable`);
    const elementOrigin = await locator.evaluate(
      (element) => element.ownerDocument.defaultView?.location.origin ?? "null",
    );
    const origin = assertBoundSecretOrigin(page.url(), elementOrigin, allowedOrigin);
    await locator.fill("", { timeout: settings.timeoutMs });
    await locator.fill(value, { timeout: settings.timeoutMs });
    await sessions.touch(threadId, "interactive", managed);
    return { used: true, name, origin };
  };
  return policyBrowser;
}

export class BrowserSettingsStore {
  readonly storage = storage;
  private readonly initializeOnce = retryableInit(() => this.initialize());

  init(): Promise<void> {
    return this.initializeOnce();
  }

  private async initialize() {
    await this.storage.init();
    const schema = await Bun.file(new URL("./browser-settings.sql", import.meta.url)).text();
    await this.storage.db.none(schema);
  }

  async get(): Promise<BrowserSettings> {
    await this.init();
    return this.storage.db.one<BrowserSettings>(`
      SELECT ${BROWSER_COLUMNS} FROM popagent_browser_settings WHERE singleton = TRUE
    `);
  }

  async update(input: BrowserSettingsInput): Promise<BrowserSettings> {
    await this.init();
    return this.storage.db.one<BrowserSettings>(`
      UPDATE popagent_browser_settings SET
        enabled=$1,scope=$2,viewport_width=$3,viewport_height=$4,timeout_ms=$5,
        max_sessions=$6,idle_timeout_ms=$7,screencast_enabled=$8,screenshots_enabled=$9,
        multi_tab_enabled=$10,forms_enabled=$11,dialogs_enabled=$12,drag_enabled=$13,
        evaluate_enabled=$14,recording_enabled=$15,recording_retention_days=$16,
        recording_max_files=$17,allow_hosts=$18,deny_hosts=$19,provider=$20,updated_at=NOW()
      WHERE singleton=TRUE RETURNING ${BROWSER_COLUMNS}
    `, [
      input.enabled, input.scope, input.viewportWidth, input.viewportHeight, input.timeoutMs,
      input.maxSessions, input.idleTimeoutMs, input.screencastEnabled, input.screenshotsEnabled,
      input.multiTabEnabled, input.formsEnabled, input.dialogsEnabled, input.dragEnabled,
      input.evaluateEnabled, input.recordingEnabled, input.recordingRetentionDays,
      input.recordingMaxFiles, input.allowHosts, input.denyHosts, input.provider,
    ]);
  }
}

export class BrowserRuntime {
  private browsers: AgentBrowser[] = [];
  private interactiveBrowser?: PolicyBrowser;
  private activeSettings?: BrowserSettings;
  private applied?: string;
  private profileStatePath?: string;
  private profileStateDir?: string;
  readonly sessions = new BrowserSessionRegistry();

  constructor(
    readonly settings: BrowserSettingsStore,
    private readonly roles: Pick<AgentSettingsStore, "list"> = agentSettings,
    private readonly profiles: Pick<BrowserProfileStore, "active"> = browserProfiles,
  ) {}

  async useBoundSecret(input: {
    agentId: string;
    threadId: string;
    ref: string;
    name: string;
    value: string;
    allowedOrigin: string;
    signal?: AbortSignal;
  }): Promise<{ used: true; name: string; origin: string }> {
    input.signal?.throwIfAborted();
    const settings = this.activeSettings;
    const role = (await this.roles.list()).find(({ id }) => id === input.agentId);
    if (!role || role.browserAccess !== "interactive") {
      throw new Error("Agent role is not authorized for interactive secret use");
    }
    if (!settings?.enabled || !settings.formsEnabled || !this.interactiveBrowser) {
      throw new Error("Interactive browser form access is not enabled");
    }
    const result = await this.interactiveBrowser.fillBoundSecret(input);
    input.signal?.throwIfAborted();
    return result;
  }

  async apply(agent: Agent, settings?: BrowserSettings): Promise<void> {

    settings ??= await this.settings.get();
    this.activeSettings = settings;
    const profiles = await this.roles.list();
    const activeProfile = await this.profiles.active();
    const fingerprint = JSON.stringify([
      settings,
      activeProfile?.profile,
      profiles.map(({ id, browserAccess }) => ({ id, browserAccess })).sort((a, b) => a.id.localeCompare(b.id)),
    ]);
    this.sessions.configure(settings);
    await this.sessions.cleanupIdle();
    if (this.applied === fingerprint) return;
    await this.sessions.closeAll();
    await Promise.all(this.browsers.map((browser) => browser.close()));
    this.browsers = [];
    if (this.profileStateDir) await rm(this.profileStateDir, { recursive: true, force: true });
    this.profileStatePath = undefined;
    this.profileStateDir = undefined;
    if (activeProfile) {
      this.profileStateDir = await mkdtemp(join(tmpdir(), "popagent-browser-profile-"));
      this.profileStatePath = join(this.profileStateDir, "storage-state.json");
      await writeFile(this.profileStatePath, JSON.stringify(activeProfile.state), { mode: 0o600 });
    }

    const headlessActive = settings.enabled && settings.provider === "agent-browser";
    const accesses = headlessActive ? profiles.map((profile) => profile.browserAccess) : [];
    const interactiveBrowser = accesses.includes("interactive")
      ? await createPolicyBrowser(settings, false, this.sessions, this.profileStatePath)
      : undefined;
    const readOnlyBrowser = accesses.includes("read-only")
      ? await createPolicyBrowser(settings, true, this.sessions, this.profileStatePath)
      : undefined;
    const browserFor = (id: string) => {
      if (!headlessActive) return undefined;
      const access = profiles.find((profile) => profile.id === id)?.browserAccess ?? "none";
      return access === "interactive" ? interactiveBrowser : access === "read-only" ? readOnlyBrowser : undefined;
    };

    agent.setBrowser(browserFor(agent.id));
    const specialists = Object.entries(await agent.listAgents()) as unknown as [string, Agent][];
    for (const [id, specialist] of specialists) specialist.setBrowser(browserFor(id));
    this.browsers = [interactiveBrowser, readOnlyBrowser].filter((browser): browser is PolicyBrowser => browser !== undefined);
    this.interactiveBrowser = interactiveBrowser;
    this.applied = fingerprint;
  }

  /**
   * Per-provider readiness for the settings page LEDs; never launches a
   * browser. The headless provider is ready when its Chromium build exists;
   * BifrOSt is ready when its MCP socket answers `ping`.
   */
  async health(settings?: BrowserSettings): Promise<Pick<BrowserHealth, "provider" | "healthy" | "headless" | "selected" | "providers">> {
    settings ??= await this.settings.get();
    const [chromiumReady, bifrostTools] = await Promise.all([
      Promise.resolve().then(() => existsSync(chromium.executablePath())).catch(() => false),
      listBifrOStTools(AbortSignal.timeout(3_000)).catch(() => undefined),
    ]);
    const bifrostReady = bifrostTools !== undefined && bifrostTools.length > 0;
    const bifrostDetail = bifrostTools === undefined
      ? "MCP socket unreachable"
      : bifrostTools.length === 0
        ? "Socket answers, but no MCP capabilities are granted (start BifrOSt with --allow-mcp-capability …)"
        : `MCP socket answering · ${bifrostTools.length} tools exposed`;
    const providers: BrowserProviderHealth[] = [
      {
        id: "agent-browser",
        label: "Headless AgentBrowser",
        healthy: chromiumReady,
        headless: true,
        detail: chromiumReady ? "Chromium installed" : "Chromium build missing",
      },
      {
        id: "bifrost-navigator",
        label: "BifrOSt Navigator",
        healthy: bifrostReady,
        headless: false,
        detail: bifrostDetail,
      },
    ];
    const selected = providers.find((provider) => provider.id === settings!.provider) ?? providers[0]!;
    return { provider: selected.label, healthy: selected.healthy, headless: selected.headless, selected: selected.id, providers };
  }

  async test(): Promise<{ url: string; title: string; elementCount: number }> {
    if (this.activeSettings?.provider === "bifrost-navigator") {
      const result = await callBifrOSt("browser_current_page", {}, AbortSignal.timeout(15_000));
      const page = result.structuredContent ?? {};
      return {
        url: String(page.url ?? ""),
        title: String(page.title ?? ""),
        elementCount: 0,
      };
    }
    const browser = this.interactiveBrowser;
    if (!browser) throw new Error("Interactive browser is not enabled");
    const threadId = `browser-test:${crypto.randomUUID()}`;
    try {
      const navigation = await browser.goto({ url: "https://example.com", waitUntil: "domcontentloaded" }, threadId);
      if (!("success" in navigation) || !navigation.success) throw new Error("Browser navigation test failed");
      const snapshot = await browser.snapshot({ interactiveOnly: false, maxDepth: 4 }, threadId);
      if (!("success" in snapshot) || !snapshot.success) throw new Error("Browser snapshot test failed");
      return { url: snapshot.url, title: snapshot.title, elementCount: snapshot.elementCount };
    } finally {
      await this.sessions.closeThread(threadId);
    }
  }

  invalidate() {
    this.applied = undefined;
    this.activeSettings = undefined;
  }

  /** Close every open browser and its sessions during process shutdown. */
  async shutdown(): Promise<void> {
    await this.sessions.closeAll();
    await Promise.all(this.browsers.map((browser) => browser.close()));
    this.browsers = [];
    this.interactiveBrowser = undefined;
    if (this.profileStateDir) await rm(this.profileStateDir, { recursive: true, force: true });
    this.profileStatePath = undefined;
    this.profileStateDir = undefined;
    this.applied = undefined;
  }
}

export const browserSettings = new BrowserSettingsStore();
export const browserRuntime = new BrowserRuntime(browserSettings);