popagent

Latest change 7f0ff66d6d9fb6468416c58bee46bd3d08169501 - Checkpoint browser channels and memory work by AkurAI Build

import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { ExternalLink, Globe2, Loader2, MonitorUp, MousePointer2, Power, X } from "lucide-react";
import type { BrowserSession } from "../api-types";
import { apiFetch } from "./api";
import { parseBrowserSse } from "./browser-stream";
import { toast } from "./toast";
import { useFocusTrap } from "./use-focus-trap";

type Frame = { data: string; viewport?: { width: number; height: number } };

export function BrowserPanel({ threadId }: { threadId: string }) {
  const [sessions, setSessions] = useState<BrowserSession[]>([]);
  const [selectedId, setSelectedId] = useState<string>();
  const [dismissedId, setDismissedId] = useState<string>();
  const [frame, setFrame] = useState<Frame>();
  const [streaming, setStreaming] = useState(false);
  const [takeover, setTakeover] = useState(false);
  const [mobile, setMobile] = useState(() => window.matchMedia("(max-width: 767px)").matches);
  const panelRef = useRef<HTMLElement>(null);
  const imageRef = useRef<HTMLImageElement>(null);
  const lastPointerMove = useRef(0);
  const selected = useMemo(() => sessions.find((session) => session.id === selectedId) ?? sessions[0], [selectedId, sessions]);
  const visible = Boolean(selected && dismissedId !== selected.id);
  useFocusTrap(panelRef, visible && mobile);
  useEffect(() => {
    const media = window.matchMedia("(max-width: 767px)");
    const change = () => setMobile(media.matches);
    media.addEventListener("change", change);
    return () => media.removeEventListener("change", change);
  }, []);
  useEffect(() => {
    if (visible && mobile) panelRef.current?.querySelector<HTMLButtonElement>('[aria-label="Hide live browser"]')?.focus();
  }, [mobile, visible]);

  const loadSessions = useCallback(async () => {
    const response = await apiFetch(`/api/browser/sessions?threadId=${encodeURIComponent(threadId)}`);
    if (!response.ok) return;
    const next = ((await response.json()) as { sessions: BrowserSession[] }).sessions;
    setSessions(next);
    setSelectedId((current) => next.some((session) => session.id === current) ? current : next[0]?.id);
    setDismissedId((current) => current && next.some((session) => session.id === current) ? current : undefined);
  }, [threadId]);

  useEffect(() => {
    setSessions([]);
    setSelectedId(undefined);
    setDismissedId(undefined);
    setFrame(undefined);
    setTakeover(false);
    void loadSessions();
    const timer = window.setInterval(() => void loadSessions(), 1_000);
    return () => window.clearInterval(timer);
  }, [loadSessions]);

  const streamingSessionId = visible ? selected?.id : undefined;
  useEffect(() => {
    if (!streamingSessionId) return;
    const controller = new AbortController();
    setFrame(undefined);
    setStreaming(true);
    void apiFetch(`/api/browser/sessions/${encodeURIComponent(streamingSessionId)}/stream`, { signal: controller.signal })
      .then(async (response) => {
        if (!response.ok || !response.body) throw new Error(`HTTP ${response.status}`);
        const reader = response.body.getReader();
        const decoder = new TextDecoder();
        let pending = "";
        while (true) {
          const chunk = await reader.read();
          if (chunk.done) break;
          pending += decoder.decode(chunk.value, { stream: true });
          const parsed = parseBrowserSse(pending);
          pending = parsed.rest;
          for (const event of parsed.events) {
            if (event.event === "frame" && event.data && typeof event.data === "object" && "data" in event.data) {
              setFrame(event.data as Frame);
            }
          }
        }
      })
      .catch((error) => {
        if (!controller.signal.aborted) toast(`Browser stream unavailable: ${error instanceof Error ? error.message : String(error)}`);
      })
      .finally(() => { if (!controller.signal.aborted) setStreaming(false); });
    return () => controller.abort();
  }, [streamingSessionId]);

  const setAdminTakeover = async (enabled: boolean) => {
    if (!selected) return;
    const response = await apiFetch(`/api/browser/sessions/${encodeURIComponent(selected.id)}/takeover`, {
      method: "PATCH",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({ enabled }),
    });
    if (response.ok) setTakeover(enabled);
    else toast("Browser takeover is unavailable for this session");
  };

  const sendInput = (payload: unknown) => {
    if (!selected || !takeover) return;
    void apiFetch(`/api/browser/sessions/${encodeURIComponent(selected.id)}/input`, {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify(payload),
    });
  };

  const takeoverSessionId = selected?.id;
  useEffect(() => () => {
    if (takeoverSessionId && takeover) void apiFetch(`/api/browser/sessions/${encodeURIComponent(takeoverSessionId)}/takeover`, {
      method: "PATCH", headers: { "content-type": "application/json" }, body: JSON.stringify({ enabled: false }),
    });
  }, [takeoverSessionId, takeover]);

  const pointerEvent = (type: "mousePressed" | "mouseReleased" | "mouseMoved", event: React.PointerEvent<HTMLImageElement>) => {
    if (!takeover) return;
    if (type === "mouseMoved" && performance.now() - lastPointerMove.current < 50) return;
    if (type === "mouseMoved") lastPointerMove.current = performance.now();
    const rect = event.currentTarget.getBoundingClientRect();
    const width = frame?.viewport?.width ?? event.currentTarget.naturalWidth;
    const height = frame?.viewport?.height ?? event.currentTarget.naturalHeight;
    const x = Math.max(0, Math.min(width, ((event.clientX - rect.left) / rect.width) * width));
    const y = Math.max(0, Math.min(height, ((event.clientY - rect.top) / rect.height) * height));
    sendInput({ kind: "mouse", event: { type, x, y, button: "left", clickCount: type === "mouseMoved" ? 0 : 1 } });
  };

  if (!selected) return null;
  if (!visible) {
    return <button type="button" onClick={() => setDismissedId(undefined)} className="fixed bottom-5 right-5 z-20 flex min-h-11 items-center gap-2 border border-accent/40 bg-panel px-3 text-sm shadow-lg" aria-label="Open live browser"><MonitorUp className="size-4 text-accent" />Live browser</button>;
  }

  return (
    <aside ref={panelRef} role={mobile ? "dialog" : "complementary"} aria-modal={mobile || undefined} className="browser-panel fixed inset-0 z-40 flex min-w-0 flex-col border-l border-line bg-bg md:relative md:inset-auto md:z-0 md:w-[44vw] md:max-w-[720px]" aria-label="Live agent browser">
      <header className="flex min-h-14 items-center gap-2 border-b border-line px-3">
        <Globe2 className="size-4 shrink-0 text-accent" />
        <div className="min-w-0 flex-1"><div className="truncate text-xs font-medium">{selected.currentUrl ?? "Browser starting…"}</div><div className="text-[10px] uppercase tracking-wide text-ink-dim">{selected.access} · {selected.status}</div></div>
        {sessions.length > 1 ? <select aria-label="Active browser session" value={selected.id} onChange={(event) => { setSelectedId(event.currentTarget.value); setTakeover(false); }} className="min-h-9 max-w-32 border border-line bg-panel px-2 text-xs">{sessions.map((session) => <option key={session.id} value={session.id}>{session.access}</option>)}</select> : null}
        <button type="button" aria-label={takeover ? "Disable browser takeover" : "Enable browser takeover"} aria-pressed={takeover} disabled={selected.access !== "interactive"} onClick={() => void setAdminTakeover(!takeover)} className={`flex size-10 items-center justify-center border ${takeover ? "border-accent bg-accent-soft text-accent" : "border-line text-ink-muted"} disabled:opacity-35`}><MousePointer2 className="size-4" /></button>
        {selected.currentUrl ? <a href={selected.currentUrl} target="_blank" rel="noreferrer" aria-label="Open current page in a new tab" className="flex size-10 items-center justify-center border border-line text-ink-muted"><ExternalLink className="size-4" /></a> : null}
        <button type="button" aria-label="Close browser session" onClick={async () => { await apiFetch(`/api/browser/sessions/${encodeURIComponent(selected.id)}/close`, { method: "POST" }); await loadSessions(); }} className="flex size-10 items-center justify-center border border-line text-ink-muted hover:text-red-400"><Power className="size-4" /></button>
        <button type="button" aria-label="Hide live browser" onClick={() => { void setAdminTakeover(false); setDismissedId(selected.id); }} className="flex size-10 items-center justify-center border border-line text-ink-muted"><X className="size-4" /></button>
      </header>
      <div
        className={`relative flex min-h-0 flex-1 items-center justify-center overflow-hidden bg-black ${takeover ? "outline outline-1 outline-inset outline-accent" : ""}`}
        tabIndex={takeover ? 0 : -1}
        onKeyDown={(event) => {
          if (!takeover || event.key === "Tab") return;
          event.preventDefault();
          sendInput({ kind: "keyboard", event: { type: "keyDown", key: event.key, code: event.code } });
        }}
      >
        {frame ? <img ref={imageRef} src={`data:image/jpeg;base64,${frame.data}`} alt="Live view of the agent browser" draggable={false} className={`max-h-full max-w-full object-contain ${takeover ? "cursor-crosshair" : ""}`} onPointerMove={(event) => pointerEvent("mouseMoved", event)} onPointerDown={(event) => { event.currentTarget.setPointerCapture(event.pointerId); pointerEvent("mousePressed", event); }} onPointerUp={(event) => pointerEvent("mouseReleased", event)} /> : <div className="flex items-center gap-2 text-sm text-white/60">{streaming ? <Loader2 className="size-4 animate-spin" /> : null}{streaming ? "Waiting for browser frames…" : "Browser preview unavailable"}</div>}
      </div>
      <footer className="flex min-h-10 items-center justify-between border-t border-line px-3 text-[11px] text-ink-dim"><span>{selected.tabs.length} {selected.tabs.length === 1 ? "tab" : "tabs"}</span><span>{takeover ? "Admin control enabled" : "Agent has control"}</span></footer>
    </aside>
  );
}