From 58187dca6492dd0faddd90fab1afd906b66e0e35 Mon Sep 17 00:00:00 2001 From: Matteo Benedetto Date: Wed, 12 Aug 2026 13:50:47 +0200 Subject: [PATCH] pi-firefox-bidi: estensione pi per WebDriver BiDi + delegazione agy --- .gitignore | 4 + README.md | 56 ++++++ extensions/firefox-bidi.ts | 337 +++++++++++++++++++++++++++++++++++++ package.json | 14 ++ 4 files changed, 411 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 extensions/firefox-bidi.ts create mode 100644 package.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..112cac0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +pylibs/ +.venv/ +*.log diff --git a/README.md b/README.md new file mode 100644 index 0000000..5d0bfbb --- /dev/null +++ b/README.md @@ -0,0 +1,56 @@ +# pi-firefox-bidi + +Estensione per **pi agent**: controllo bidirezionale di **Firefox** tramite +**WebDriver BiDi** (navigazione, screenshot allegati al contesto, eval JS, +eventi asincroni) e delegazione asincrona ad **Antigravity CLI** (`agy`). + +## Architettura + +``` +pi agent (questa estensione) ──HTTP JSON──► Browser Bridge (bridge.py) + └──WebSocket BiDi──► Firefox (ws://127.0.0.1:9222) +``` + +- **Firefox ESR 128+**: Remote Agent BiDi nativo con `--remote-debugging-port 9222` + (nessun geckodriver). Endpoint unico `ws://127.0.0.1:9222/session`, loopback only. +- **Browser Bridge** (`~/Dev/firefox-bidi/bridge/bridge.py`, Python + websockets): + possiede la sessione BiDi, espone HTTP JSON su `:8787` e un endpoint MCP minimale + (`/mcp`) per Antigravity CLI. +- **Script di avvio** (`~/Dev/firefox-bidi/start-firefox-bidi.sh`): + - `start` → profilo dedicato `bidi-profile` (automazione pulita) + - `start-main` → **profilo principale** (login/cookie intatti; chiude l'istanza + attiva se serve, lock esclusivo) + - `status` / `stop` / `foreground` + +## Tool registrati + +| Tool | Funzione | +|---|---| +| `browser_start` / `browser_stop` / `browser_status` | ciclo di vita Firefox + bridge | +| `browser_navigate(url)` | naviga e attende il carico | +| `browser_screenshot(path?, fullPage?)` | PNG su disco, **allegato come immagine al contesto** | +| `browser_eval(expression)` | `script.evaluate` | +| `browser_send(method, params)` | comando BiDi grezzo | +| `browser_events(action, module?)` | coda eventi asincroni (log, network, browsingContext) | +| `agy_delegate(prompt, conversation?, model?)` | pi → agy headless (`--output-format stream-json`) | +| `bridge_notify(event, payload)` | webhook locale `:8790` (agy/bridge → pi) | + +## Installazione + +```bash +pi install git:git.enne2.net/enne2/pi-firefox-bidi +# oppure in dev: pi -e /home/enne2/Dev/pi-firefox-bidi +``` + +Requisiti runtime (sulla macchina, non nel package): +- `firefox` (ESR 128+) e `python3` con `websockets` (`pip install --target websockets` se PEP 668 blocca `--user`) +- Variabili opzionali: `FIREFOX_BIDI_BRIDGE` (default `http://127.0.0.1:8787`), + `FIREFOX_BIDI_SCRIPT`, `FIREFOX_BIDI_BRIDGE_PY` (default `~/Dev/firefox-bidi/...`) + +## Sicurezza + +Il Remote Agent di Firefox ascolta solo su `127.0.0.1`; con `start-main` il +controllo BiDi espone anche i login reali del profilo: **non esporre mai la +porta 9222 fuori dalla macchina**. Il bridge MCP richiede token +(`FIREFOX_BIDI_TOKEN`). Evitare `--dangerously-skip-permissions` per +l'orchestrazione generale. diff --git a/extensions/firefox-bidi.ts b/extensions/firefox-bidi.ts new file mode 100644 index 0000000..20b3eec --- /dev/null +++ b/extensions/firefox-bidi.ts @@ -0,0 +1,337 @@ +/** + * firefox-bidi.ts — Estensione pi: controllo bidirezionale di Firefox + * via WebDriver BiDi, con screenshot e delegazione verso Antigravity CLI. + * + * Installazione (solo se autorizzata): + * cp firefox-bidi.ts ~/.pi/agent/extensions/ # poi /reload in pi + * + * Architettura: + * pi (questa estensione) ──HTTP JSON──► Browser Bridge (python) + * └──WebSocket BiDi──► Firefox (ws://127.0.0.1:9222) + * + * Il Bridge mantiene la sessione BiDi (id→Future, event loop) ed espone + * API HTTP semplici; questa estensione registra tool pi che le chiamano. + * La parte browser del bridge è in bridge/bridge.py (vedi DESIGN.md). + * + * Zero dipendenze npm: si usa solo fetch di Node 22 + child_process. + */ +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { Type } from "typebox"; +import { spawn, execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); + +const BRIDGE_URL = process.env.FIREFOX_BIDI_BRIDGE ?? "http://127.0.0.1:8787"; +const SCRIPT = + process.env.FIREFOX_BIDI_SCRIPT ?? "/home/enne2/Dev/firefox-bidi/start-firefox-bidi.sh"; +const BRIDGE_PY = + process.env.FIREFOX_BIDI_BRIDGE_PY ?? "/home/enne2/Dev/firefox-bidi/bridge/bridge.py"; + +let bridgeProcess: ReturnType | null = null; + +async function bridge(path: string, init?: RequestInit): Promise { + const res = await fetch(`${BRIDGE_URL}${path}`, { + headers: { "content-type": "application/json" }, + ...init, + }); + const body = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(`bridge ${path}: ${res.status} ${JSON.stringify(body)}`); + return body; +} + +async function ensureBridge() { + try { + await bridge("/status"); + return; + } catch { + /* bridge non attivo: avvio */ + } + await execFileAsync(SCRIPT, ["start"]); + bridgeProcess = spawn("python3", [BRIDGE_PY], { + stdio: "ignore", + detached: true, + }); + bridgeProcess.unref(); + // attende che il bridge risponda + for (let i = 0; i < 40; i++) { + try { + await bridge("/status"); + return; + } catch { + await new Promise((r) => setTimeout(r, 250)); + } + } + throw new Error("Bridge non raggiungibile dopo 10s"); +} + +export default function (pi: ExtensionAPI) { + // ------------------------------------------------------------------------- + // Ciclo di vita del browser + // ------------------------------------------------------------------------- + pi.registerTool({ + name: "browser_start", + label: "Browser Start", + description: + "Avvia Firefox con WebDriver BiDi (profilo dedicato, porta 9222) e il Browser Bridge.", + parameters: Type.Object({}), + async execute() { + await ensureBridge(); + const st = await bridge("/status"); + return { + content: [ + { type: "text", text: `Firefox BiDi attivo: ${JSON.stringify(st)}` }, + ], + details: {}, + }; + }, + }); + + pi.registerTool({ + name: "browser_stop", + label: "Browser Stop", + description: "Termina l'istanza Firefox BiDi (e il bridge).", + parameters: Type.Object({}), + async execute() { + if (bridgeProcess) { + bridgeProcess.kill(); + bridgeProcess = null; + } + await execFileAsync(SCRIPT, ["stop"]).catch(() => {}); + return { + content: [{ type: "text", text: "Firefox BiDi fermato." }], + details: {}, + }; + }, + }); + + pi.registerTool({ + name: "browser_status", + label: "Browser Status", + description: "Stato di Firefox BiDi e del bridge (sessioni, contesti aperti).", + parameters: Type.Object({}), + async execute() { + const st = await bridge("/status"); + return { content: [{ type: "text", text: JSON.stringify(st, null, 2) }], details: {} }; + }, + }); + + // ------------------------------------------------------------------------- + // Navigazione e comandi BiDi + // ------------------------------------------------------------------------- + pi.registerTool({ + name: "browser_navigate", + label: "Browser Navigate", + description: "Naviga il contesto principale verso un URL e attende il carico.", + parameters: Type.Object({ + url: Type.String({ description: "URL completo (es. https://example.com)" }), + }), + async execute(_id, params) { + await ensureBridge(); + const r = await bridge("/navigate", { + method: "POST", + body: JSON.stringify({ url: params.url }), + }); + return { + content: [ + { + type: "text", + text: `Navigazione OK: ${r.url} (context ${r.context})`, + }, + ], + details: r, + }; + }, + }); + + pi.registerTool({ + name: "browser_screenshot", + label: "Browser Screenshot", + description: + "Cattura uno screenshot (PNG) della pagina corrente e lo salva su disco. " + + "Il percorso viene restituito e l'immagine è allegata al contesto.", + parameters: Type.Object({ + path: Type.Optional( + Type.String({ description: "Percorso di salvataggio (default: /tmp/firefox-bidi/shot-.png)" }), + ), + fullPage: Type.Optional(Type.Boolean({ description: "Screenshot dell'intera pagina" })), + }), + async execute(_id, params) { + await ensureBridge(); + const r = await bridge("/screenshot", { + method: "POST", + body: JSON.stringify({ out: params.path, fullPage: params.fullPage ?? false }), + }); + return { + content: [ + { type: "text", text: `Screenshot salvato in ${r.path} (${r.bytes} bytes)` }, + { type: "image", image: r.path, mediaType: "image/png" }, + ], + details: r, + }; + }, + }); + + pi.registerTool({ + name: "browser_eval", + label: "Browser Eval", + description: + "Esegue JavaScript nella pagina (script.evaluate BiDi). Restituisce il valore JSON.", + parameters: Type.Object({ + expression: Type.String({ description: "Espressione JS da valutare" }), + awaitPromise: Type.Optional(Type.Boolean({ default: true })), + }), + async execute(_id, params) { + await ensureBridge(); + const r = await bridge("/eval", { + method: "POST", + body: JSON.stringify(params), + }); + return { + content: [ + { + type: "text", + text: `Result: ${JSON.stringify(r.result ?? r, null, 2)}`, + }, + ], + details: r, + }; + }, + }); + + pi.registerTool({ + name: "browser_send", + label: "Browser BiDi Send", + description: + "Invia un comando WebDriver BiDi grezzo (power users), es. browsingContext.captureScreenshot, " + + "session.subscribe, network.getCookies, log.read.", + parameters: Type.Object({ + method: Type.String({ description: "Nome metodo BiDi, es. browsingContext.captureScreenshot" }), + params: Type.Optional(Type.Record(Type.String(), Type.Any(), { default: {} })), + }), + async execute(_id, params) { + await ensureBridge(); + const r = await bridge("/cmd", { + method: "POST", + body: JSON.stringify(params), + }); + return { + content: [{ type: "text", text: JSON.stringify(r, null, 2) }], + details: r, + }; + }, + }); + + // ------------------------------------------------------------------------- + // Eventi asincroni (push browser → pi) + // ------------------------------------------------------------------------- + pi.registerTool({ + name: "browser_events", + label: "Browser Events", + description: + "Legge gli eventi BiDi accumulati dal bridge (log.entryAdded, network.responseCompleted, " + + "browsingContext.*, script.*). Opzionalmente sottoscrive un modulo prima.", + parameters: Type.Object({ + action: Type.Enum( + { list: "list", subscribe: "subscribe", drain: "drain" }, + { description: "list=eventi in coda, subscribe=iscrivi un modulo, drain=svuota la coda" }, + ), + module: Type.Optional( + Type.String({ description: "Modulo BiDi da sottoscrivere, es. log, network, browsingContext" }), + ), + clear: Type.Optional(Type.Boolean({ default: false })), + }), + async execute(_id, params) { + await ensureBridge(); + const r = await bridge("/events", { + method: "POST", + body: JSON.stringify(params), + }); + return { + content: [{ type: "text", text: JSON.stringify(r, null, 2) }], + details: r, + }; + }, + }); + + // ------------------------------------------------------------------------- + // Delegazione asincrona a Antigravity CLI (pi → agy) + // ------------------------------------------------------------------------- + pi.registerTool({ + name: "agy_delegate", + label: "Antigravity Delegate", + description: + "Delega un task a Antigravity CLI in modalità headless (stream-json NDJSON). " + + "Usa --conversation per continuare una conversazione agy esistente. " + + "L'output (init/step_update/result) viene restituito come testo.", + parameters: Type.Object({ + prompt: Type.String({ description: "Task per l'agente agy" }), + conversation: Type.Optional(Type.String({ description: "ID conversazione agy da riprendere" })), + model: Type.Optional(Type.String({ description: "Modello agy (es. Gemini 3.1 Pro)" })), + timeout: Type.Optional(Type.Number({ default: 300, description: "Timeout secondi" })), + }), + async execute(_id, params) { + const args = [ + "-p", params.prompt, + "--output-format", "stream-json", + "--print-timeout", `${params.timeout ?? 300}m`, + ]; + if (params.conversation) args.push("--conversation", params.conversation); + if (params.model) args.push("--model", params.model); + const { stdout, stderr } = await execFileAsync("agy", args, { + timeout: ((params.timeout ?? 300) + 30) * 1000, + maxBuffer: 20 * 1024 * 1024, + }); + // NDJSON → riepilogo compatto per il contesto pi + const lines = stdout.trim().split("\n").filter(Boolean).map((l) => { + try { return JSON.parse(l); } catch { return null; } + }); + const init = lines.find((e) => e?.type === "init"); + const result = lines.find((e) => e?.type === "result"); + const summary = { + conversation_id: init?.conversation_id ?? result?.conversation_id ?? params.conversation, + status: result?.status, + response: result?.response, + duration: result?.duration, + stderr: stderr.slice(0, 500), + }; + return { + content: [ + { + type: "text", + text: `agy → ${summary.status ?? "?"} | conversation=${summary.conversation_id}\n${summary.response ?? ""}${summary.stderr ? `\n[stderr] ${summary.stderr}` : ""}`, + }, + ], + details: summary, + }; + }, + }); + + // ------------------------------------------------------------------------- + // Notifica asincrona (agy/bridge → pi): webhook locale + // ------------------------------------------------------------------------- + pi.registerTool({ + name: "bridge_notify", + label: "Bridge Notify (webhook)", + description: + "Invia un evento al webhook locale di pi (http://127.0.0.1:8790/webhook): " + + "usato da agy o dal bridge per segnalare task completi, errori o eventi browser.", + parameters: Type.Object({ + event: Type.String({ description: "Nome evento, es. task.completed, browser.event" }), + payload: Type.Optional(Type.Record(Type.String(), Type.Any(), { default: {} })), + }), + async execute(_id, params) { + await fetch("http://127.0.0.1:8790/webhook", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ event: params.event, payload: params.payload ?? {} }), + }).catch((e) => { + throw new Error(`webhook: ${e.message}`); + }); + return { + content: [{ type: "text", text: "Evento inoltrato al webhook pi." }], + details: {}, + }; + }, + }); +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..806adff --- /dev/null +++ b/package.json @@ -0,0 +1,14 @@ +{ + "name": "pi-firefox-bidi", + "version": "0.1.0", + "description": "Estensione pi: controllo bidirezionale di Firefox via WebDriver BiDi (navigazione, screenshot, eval, eventi) + delegazione asincrona ad Antigravity CLI (agy).", + "keywords": ["pi-package", "firefox", "webdriver-bidi", "screenshot", "agy", "antigravity"], + "license": "MIT", + "peerDependencies": { + "@earendil-works/pi-coding-agent": "*", + "typebox": "*" + }, + "pi": { + "extensions": ["./extensions"] + } +}