/** * agy-pi — Google Antigravity CLI (agy) come subagent multimodale dentro pi. * * Espone un set completo di strumenti specializzati per generazione ed editing * di immagini, analisi di file, trascrizione audio, analisi video e gestione * conversazione. Ogni strumento costruisce un prompt ottimizzato secondo le * best practice di Gemini/Nano Banana (Keep+Change+Add+Render, un cambiamento * per turno, ri-attacco dell'immagine base, ecc.). * * L'agente (pi) decide quale strumento usare in base al task. */ import { execFile, execFileSync, spawn, spawnSync } from "node:child_process"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { promisify } from "node:util"; import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; import { Type } from "typebox"; import { type Api, type AssistantMessage, type AssistantMessageEventStream, calculateCost, type Context, createAssistantMessageEventStream, type Model, type SimpleStreamOptions, type ThinkingLevelMap, type ToolCall, } from "@earendil-works/pi-ai/compat"; const execFileAsync = promisify(execFile); // --------------------------------------------------------------------------- // Configurazione // --------------------------------------------------------------------------- const AGY_CHAT_DIR = path.join(os.homedir(), ".agy-chat"); const STATE_FILE = path.join(AGY_CHAT_DIR, "conversation_id"); const CONV_DIR = path.join(os.homedir(), ".gemini", "antigravity-cli", "conversations"); const DEFAULT_TIMEOUT_MS = 180_000; // 3 min const IMAGE_TIMEOUT_MS = 300_000; // 5 min const IMAGE_EXTENSIONS = new Set([".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff", ".svg"]); // --------------------------------------------------------------------------- // Configurazione persistente (~/.config/agy-pi/config.json) // --------------------------------------------------------------------------- const CONFIG_DIR = path.join(os.homedir(), ".config", "agy-pi"); const CONFIG_FILE = path.join(CONFIG_DIR, "config.json"); interface AgyConfig { geminiApiKey?: string; enne2ApiKey?: string; sttBackend?: string; // gemini | enne2 sttUrl?: string; sttModel?: string; sttMaxDuration?: number; // secondi ttsBackend?: string; // gemini | enne2 ttsNotify?: boolean; ttsModel?: string; agyBin?: string; agyDefaultModel?: string; agyTimeoutMs?: number; // Fase 2 — Context Injection contextInject?: boolean; // on|off contextTokens?: number; // token cap per il contesto iniettato webSearch?: string; // auto|on|off — Strada A: pi cerca + inietta vocalPlanningMode?: boolean; // Opzione 4: piano + conferma prima di eseguire sttDirectGemini?: boolean; // Forza l'uso di Gemini multimodale per l'audio anche con modelli non-Gemini (es. DeepSeek) voiceModel?: string; // Modello per l'elaborazione vocale diretta (default: gemini-3.7-flash-medium) } const CONFIG_DEFAULTS: AgyConfig = { sttBackend: "gemini", sttUrl: "https://ai.enne2.net", sttModel: "gemma4:E4B", sttMaxDuration: 120, ttsBackend: "gemini", ttsNotify: true, ttsModel: "gemini-2.5-flash-preview-tts", agyTimeoutMs: 180_000, contextInject: true, contextTokens: 1500, webSearch: "auto", vocalPlanningMode: true, sttDirectGemini: true, voiceModel: "gemini-3.7-flash-medium", }; function loadConfig(): AgyConfig { try { return { ...CONFIG_DEFAULTS, ...JSON.parse(fs.readFileSync(CONFIG_FILE, "utf8")) }; } catch { return { ...CONFIG_DEFAULTS }; } } function saveConfig(cfg: AgyConfig) { try { fs.mkdirSync(CONFIG_DIR, { recursive: true }); fs.writeFileSync(CONFIG_FILE, JSON.stringify(cfg, null, 2), { mode: 0o600 }); } catch { /* ignora */ } } // Legge una chiave: config file → env var → default function getConfig(key: keyof AgyConfig): string | undefined { const cfg = loadConfig(); const v = cfg[key]; if (v === undefined || v === "") return undefined; return String(v); } function setConfig(key: keyof AgyConfig, value: string) { const cfg = loadConfig(); const numKeys: (keyof AgyConfig)[] = ["sttMaxDuration", "agyTimeoutMs", "contextTokens"]; const boolKeys: (keyof AgyConfig)[] = ["ttsNotify", "contextInject", "vocalPlanningMode", "sttDirectGemini"]; if (numKeys.includes(key)) { (cfg as any)[key] = Number(value); } else if (boolKeys.includes(key)) { (cfg as any)[key] = value === "true" || value === "1" || value === "yes"; } else { (cfg as any)[key] = value; } saveConfig(cfg); } function resetConfig() { try { fs.rmSync(CONFIG_FILE, { force: true }); } catch { /* ignora */ } } // --------------------------------------------------------------------------- // Helper: trovare il binario agy // --------------------------------------------------------------------------- function findAgy(): string { const candidates = [ getConfig("agyBin"), process.env.AGY_BIN, path.join(os.homedir(), ".local", "bin", "agy"), "agy", ].filter(Boolean) as string[]; for (const c of candidates) { if (c === "agy") return c; if (fs.existsSync(c)) return c; } return "agy"; } // --------------------------------------------------------------------------- // Helper: ultimo conversation_id // --------------------------------------------------------------------------- function getLatestConversationId(): string | null { try { if (!fs.existsSync(CONV_DIR)) return null; const files = fs .readdirSync(CONV_DIR) .filter((f) => f.endsWith(".db")) .map((f) => path.join(CONV_DIR, f)) .sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs); if (files.length === 0) return null; return path.basename(files[0], ".db"); } catch { return null; } } function readState(): string | null { try { return fs.readFileSync(STATE_FILE, "utf8").trim() || null; } catch { return null; } } function writeState(id: string) { try { fs.mkdirSync(AGY_CHAT_DIR, { recursive: true }); fs.writeFileSync(STATE_FILE, id, "utf8"); } catch { /* ignora */ } } function resetState() { try { fs.rmSync(STATE_FILE, { force: true }); } catch { /* ignora */ } } // --------------------------------------------------------------------------- // Helper: eseguire agy // --------------------------------------------------------------------------- interface RunResult { output: string; error: string | null; exitCode: number; } async function runAgy( args: string[], timeoutMs: number, signal?: AbortSignal, ): Promise { const bin = findAgy(); // --output-format json: agy non scrive nulla su stdout quando è piped/redirected // (bug #76). Il formato json emette un oggetto con la risposta in .response. const fullArgs = [...args, "--output-format", "json"]; try { const { stdout, stderr } = await execFileAsync(bin, fullArgs, { timeout: timeoutMs, maxBuffer: 20 * 1024 * 1024, signal, env: { ...process.env, PATH: `${path.join(os.homedir(), ".local", "bin")}:${process.env.PATH ?? ""}` }, }); // estrai la risposta dal JSON let output = stdout; try { const parsed = JSON.parse(stdout); if (parsed && typeof parsed.response === "string") { output = parsed.response; } } catch { /* non-JSON: usa stdout grezzo */ } return { output, error: stderr || null, exitCode: 0 }; } catch (err: any) { const code = typeof err.code === "number" ? err.code : 1; return { output: err.stdout || "", error: err.stderr || err.message || String(err), exitCode: code, }; } } // --------------------------------------------------------------------------- // Helper: estrarre il percorso immagine dall'output // --------------------------------------------------------------------------- function extractImagePath(text: string): string | undefined { const m = text.match(/IMAGE_PATH:\s*(\S+)/i) || text.match(/(\/[^\s]+\.(?:png|jpe?g|webp|gif))/i); return m ? m[1] : undefined; } // --------------------------------------------------------------------------- // Helper: copiare l'immagine generata in una cartella di output // --------------------------------------------------------------------------- function copyImage(src: string | undefined, outputDir?: string): string | undefined { if (!src || !outputDir) return src; try { fs.mkdirSync(outputDir, { recursive: true }); const dest = path.join(outputDir, path.basename(src)); fs.copyFileSync(src, dest); return dest; } catch { return src; } } // Esegue uno script Python/OpenCV sull'immagine per metriche oggettive: // dimensioni, aspect ratio, luminosità, densità bordi e colori dominanti. // Restituisce JSON (stringa) o stringa vuota in caso di errore. async function runOpenCV(imagePath: string): Promise { const script = ` import cv2, json, sys, collections img = cv2.imread(sys.argv[1]) if img is None: print(json.dumps({"error": "cannot read image"})) sys.exit(1) h, w = img.shape[:2] gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) brightness = round(float(gray.mean()), 1) small = cv2.resize(img, (32, 32), interpolation=cv2.INTER_AREA) colors = [tuple(int(x) for x in p) for p in small.reshape(-1, 3)] counts = collections.Counter(colors) dominant = [{"rgb": list(c), "count": n} for c, n in counts.most_common(3)] edges = cv2.Canny(gray, 100, 200) edge_density = round(float(edges.mean() / 255.0), 3) print(json.dumps({ "width": w, "height": h, "aspect_ratio": round(w / h, 3), "brightness": brightness, "edge_density": edge_density, "dominant_colors": dominant })) `; try { const { stdout } = await execFileAsync("python3", ["-c", script, imagePath], { timeout: 15_000, }); return stdout.trim(); } catch { return ""; } } // --------------------------------------------------------------------------- // Helper: esecuzione comune di un tool agy // --------------------------------------------------------------------------- interface AgyExecOptions { prompt: string; mode?: "chat" | "image" | "analyze"; model?: string; effort?: "low" | "medium" | "high"; newConversation?: boolean; stateless?: boolean; // non continua la conversazione (ri-attacca l'immagine base) addDirs?: string[]; filePaths?: string[]; yolo?: boolean; timeoutMs?: number; outputDir?: string; signal?: AbortSignal; contextBlock?: string; // Fase 2: contesto iniettato prima della richiesta } // =========================================================================== // Fase 2 — Context Injection (pi → agy) // Best practice: tag XML, richiesta per ultima (anti lost-in-the-middle), // prefisso stabile/cache, token cap, memoria durevole automatica, niente segreti. // ========================================================================= const MEMORY_FILE = path.join(CONFIG_DIR, "memory.json"); interface DurableMemory { goal?: string; decisions: string[]; conventions: string[]; openQuestions: string[]; updatedAt: string; } function loadMemory(): DurableMemory { try { const raw = JSON.parse(fs.readFileSync(MEMORY_FILE, "utf8")); return { decisions: Array.isArray(raw.decisions) ? raw.decisions : [], conventions: Array.isArray(raw.conventions) ? raw.conventions : [], openQuestions: Array.isArray(raw.openQuestions) ? raw.openQuestions : [], goal: typeof raw.goal === "string" ? raw.goal : undefined, updatedAt: raw.updatedAt ?? "", }; } catch { return { decisions: [], conventions: [], openQuestions: [], updatedAt: "" }; } } function saveMemory(mem: DurableMemory) { try { fs.mkdirSync(CONFIG_DIR, { recursive: true }); fs.writeFileSync( MEMORY_FILE, JSON.stringify({ ...mem, updatedAt: new Date().toISOString() }, null, 2), { mode: 0o600 }, ); } catch { /* ignora */ } } // Auto-update: quando la conversazione pi cresce oltre la soglia, condensa i // punti chiave recenti in durable facts invece di rigirare tutto il transcript. function autoUpdateMemory(ctx: any, force = false) { try { const mem = loadMemory(); const entries = ctx?.sessionManager?.getEntries?.() ?? []; if (force || entries.length > 24) { const userTexts: string[] = []; for (const e of entries) { if (e.type === "message" && e.message?.role === "user") { const c = e.message.content; const text = typeof c === "string" ? c : Array.isArray(c) ? c.map((b: any) => b.text ?? "").join(" ") : ""; if (text && text.length > 20) userTexts.push(text.slice(0, 160)); } } if (!mem.goal && userTexts.length) mem.goal = userTexts[0]; // ultima decisione/azione chiave (ultimo user prompt significativo) if (userTexts.length > 1) { const last = userTexts[userTexts.length - 1]; const lastShort = last.length > 100 ? last.slice(0, 100) : last; if (!mem.decisions.includes(lastShort)) { mem.decisions.push(lastShort); if (mem.decisions.length > 8) mem.decisions.shift(); } } saveMemory(mem); } } catch { /* ignora */ } } function getGitInfo(cwd: string): { branch: string; dirty: boolean; modified: string[] } { try { const branch = execFileSync("git", ["-C", cwd, "rev-parse", "--abbrev-ref", "HEAD"], { encoding: "utf8", }) .trim() || "(nessun branch)"; const porcelain = execFileSync("git", ["-C", cwd, "status", "--porcelain"], { encoding: "utf8", }) .split("\n") .filter(Boolean); return { branch, dirty: porcelain.length > 0, modified: porcelain.slice(0, 8).map((l) => l.slice(0, 70)), }; } catch { return { branch: "(no git)", dirty: false, modified: [] }; } } // Estrae lo stato recente della conversazione pi (ultimi N messaggi utente) // e lo compatta — NON il transcript grezzo (lost-in-the-middle, token bloat). function extractSessionState(ctx: any): string { try { const entries = ctx?.sessionManager?.getEntries?.() ?? []; const userTexts: string[] = []; for (let i = entries.length - 1; i >= 0 && userTexts.length < 3; i--) { const e = entries[i]; if (e.type === "message" && e.message?.role === "user") { const c = e.message.content; const text = typeof c === "string" ? c : Array.isArray(c) ? c.map((b: any) => b.text ?? "").join(" ") : ""; if (text) userTexts.push(text.replace(/\s+/g, " ").trim().slice(0, 200)); } } return userTexts.reverse().join("\n"); } catch { return ""; } } // Cerca sul web tramite Gemini API (googleSearch grounding) e restituisce i // risultati iniettabili. È la Strada A: l'estensione cerca, poi inietta. async function webSearchGemini(query: string, signal?: AbortSignal): Promise { let key = getConfig("geminiApiKey") ?? process.env.GEMINI_API_KEY ?? ""; if (!key) { try { key = fs.readFileSync(path.join(AGY_CHAT_DIR, "gemini-key"), "utf8").trim(); } catch { /* ignora */ } } if (!key) return ""; const model = process.env.AGY_GEMINI_MODEL ?? "gemini-3.5-flash"; try { const res = await fetch( `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${encodeURIComponent(key)}`, { method: "POST", signal, headers: { "Content-Type": "application/json" }, body: JSON.stringify({ contents: [{ role: "user", parts: [{ text: query }] }], tools: [{ googleSearch: {} }], }), }, ); if (!res.ok) return ""; const data: any = await res.json(); const text = data?.candidates?.[0]?.content?.parts?.map((p: any) => p.text ?? "").join(" ") ?? ""; return text.trim().slice(0, 600); } catch { return ""; } } // Chiamata generica alla Gemini API (solo testo, nessun tool). Restituisce la // risposta completa oppure stringa vuota in caso di errore. async function geminiText(prompt: string, signal?: AbortSignal): Promise { let key = getConfig("geminiApiKey") ?? process.env.GEMINI_API_KEY ?? ""; if (!key) { try { key = fs.readFileSync(path.join(AGY_CHAT_DIR, "gemini-key"), "utf8").trim(); } catch { /* ignora */ } } if (!key) return ""; const model = process.env.AGY_GEMINI_MODEL ?? "gemini-3.5-flash"; try { const res = await fetch( `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${encodeURIComponent(key)}`, { method: "POST", signal, headers: { "Content-Type": "application/json" }, body: JSON.stringify({ contents: [{ role: "user", parts: [{ text: prompt }] }], }), }, ); if (!res.ok) return ""; const data: any = await res.json(); const text = data?.candidates?.[0]?.content?.parts?.map((p: any) => p.text ?? "").join(" ") ?? ""; return text.trim(); } catch { return ""; } } // Genera DINAMICAMENTE uno script Python/OpenCV specifico per i requisiti, // usando l'LLM. Lo script legge l'immagine da argv[1] e stampa su stdout un JSON // standard: {"pass": bool, "score": 0..1, "findings": [...], "errors": [...]}. async function generateOpenCVScript(requirements: string): Promise { const prompt = `Sei un esperto di computer vision con OpenCV. Scrivi UN SINGOLO script Python (solo codice, senza markdown) che verifichi un'immagine rispetto a questi requisiti:\n\n` + `REQUISITI: ${requirements}\n\n` + `Lo script:\n` + `- legge il percorso immagine da sys.argv[1]\n` + `- importa solo cv2, numpy, json, sys (e eventuali standard library)\n` + `- esegue verifiche OGGETTIVE pertinenti ai requisiti (es. intervalli di colore HSV, forme/contorni, simmetria, testo, dimensioni, orientamento, ecc.)\n` + `- stampa su stdout UN SOLO oggetto JSON: {"pass": true/false, "score": 0..1, "findings": [stringhe], "errors": [stringhe]}\n` + ` - pass: true se l'immagine rispetta i requisiti\n` + ` - score: grado di conformità da 0 a 1\n` + ` - findings: brevi note oggettive misurate (es. "colore dominante rosso RGB(200,30,30)")\n` + ` - errors: eventuali problemi di lettura/verifica\n` + `- NON deve fallire se l'immagine è leggibile: gestisci con try/except e metti l'errore in errors\n` + `- usa solo OpenCV, numpy e librerie standard (nessuna libreria esterna)\n\n` + `Restituisci SOLO il codice Python, nessun testo aggiuntivo, nessun blocco markdown.`; const code = await geminiText(prompt); // rimuovi eventuali fence markdown return code.replace(/```python|```/g, "").trim(); } // Esegue lo script OpenCV generato sull'immagine e restituisce il JSON su stdout. async function runOpenCVScript(script: string, imagePath: string): Promise { if (!script) return ""; try { const { stdout } = await execFileAsync("python3", ["-I", "-c", script, imagePath], { timeout: 20_000, cwd: os.tmpdir(), env: { PATH: process.env.PATH ?? "", HOME: os.homedir() }, }); return stdout.trim(); } catch { return ""; } } // Genera un edit deterministico opzionale. Lo script usa argv[1] come input e // argv[2] come output e deve stampare un singolo JSON con findings/errors. async function generateProgrammaticEditScript(requirements: string, editInstructions: string): Promise { const prompt = `Write ONE Python script (code only) that applies this deterministic image edit: ${editInstructions}.\n` + `The resulting image must still target these requirements: ${requirements}.\n` + `Contract: read input image from sys.argv[1], write output image to sys.argv[2], and print one JSON object ` + `{"findings": ["..."], "errors": ["..."]}. Use only cv2, numpy, PIL/Pillow, json, sys, math. ` + `Do not import or use os, pathlib, subprocess, socket, network, arbitrary file access, eval, exec, or dynamic imports. ` + `Do not access any path except argv[1] and argv[2]. Preserve quality and dimensions unless explicitly requested. ` + `Return only Python code, without markdown fences.`; return (await geminiText(prompt)).replace(/```python|```/g, "").trim(); } // Static rejection is deliberately conservative: generated/user scripts are // still run in an isolated temp cwd with a minimal environment and timeout. function validateProgrammaticEditScript(script: string): string | null { if (!script.trim()) return "Edit script is empty."; if (script.length > 30_000) return "Edit script exceeds the thirty-thousand character limit."; if (/\b(?:os|pathlib|subprocess|socket|requests|urllib|shutil|glob|asyncio|ctypes|pickle)\b/i.test(script)) return "Edit script uses a forbidden module or identifier."; if (/\b(?:open|eval|exec|compile|__import__|globals|locals|input|getattr|setattr|vars|dir)\s*\(/i.test(script) || /__/.test(script)) return "Edit script uses a forbidden operation."; if (/(?:['\"])(?:\/(?:[^'\"]+)|~\/|\.\.\/|[A-Za-z]:\\)/.test(script)) return "Edit script contains a filesystem path; use only sys.argv input/output paths."; const imports = [...script.matchAll(/(?:from|import)\s+([A-Za-z_][\w.]*)/g)].map((m) => m[1].split(".")[0]); const allowed = new Set(["cv2", "numpy", "np", "PIL", "Image", "json", "sys", "math"]); const forbiddenImport = imports.find((name) => !allowed.has(name)); return forbiddenImport ? `Edit script imports forbidden module: ${forbiddenImport}.` : null; } async function runProgrammaticEditScript(script: string, imagePath: string, outputPath: string): Promise<{ ok: boolean; report: string }> { const validationError = validateProgrammaticEditScript(script); if (validationError) return { ok: false, report: validationError }; try { // Prefer bubblewrap when available: no network, temporary /tmp, and a // read-only host view. Static validation remains necessary defense in depth. const useBubblewrap = fs.existsSync("/usr/bin/bwrap"); if (useBubblewrap) { fs.closeSync(fs.openSync(outputPath, "a")); } const command = useBubblewrap ? "/usr/bin/bwrap" : "python3"; const args = useBubblewrap ? ["--ro-bind", "/", "/", "--bind", outputPath, outputPath, "--dev", "/dev", "--proc", "/proc", "--unshare-net", "--chdir", os.tmpdir(), "--", "python3", "-I", "-c", script, imagePath, outputPath] : ["-I", "-c", script, imagePath, outputPath]; const { stdout, stderr } = await execFileAsync(command, args, { timeout: 20_000, cwd: os.tmpdir(), env: { PATH: process.env.PATH ?? "", HOME: os.homedir() }, maxBuffer: 2 * 1024 * 1024, }); if (!fs.existsSync(outputPath)) return { ok: false, report: "Edit script completed without producing the output image." }; return { ok: true, report: (stdout || stderr || "").trim() }; } catch (error: any) { return { ok: false, report: String(error?.stderr || error?.message || error).slice(0, 2000) }; } } // Cache anti-ridondanza: evita ricerche web ripetute sullo stesso topic in una // finestra breve (il multi-step reasoning lancia prompt simili in sequenza). let lastSearch = { topic: "", time: 0 }; // Euristica per webSearch=auto. Determina se il prompt beneficia di dati // aggiornati dalla ricerca web. Filtri negativi (immagini, codice locale, // comandi) per non sprecare chiamate API/token su task che non servono. function needsWebSearch(prompt: string, mode?: string): boolean { if (mode === "image" || mode === "analyze") return false; const ws = getConfig("webSearch") ?? "auto"; if (ws === "on") return true; if (ws === "off") return false; const p = prompt.trim().toLowerCase(); if (!p || p.length < 14) return false; // --- Filtri negativi: task che NON beneficiano della ricerca web --- // generazione / editing immagini if (/(genera|crea|disegna|produrr|dipin|realizz).{0,30}(immagine|logo|icona|poster|banner|ritratto|illustraz|foto|meme|sfondo)/i.test(p)) return false; // analisi locale / OCR / trascrizione / traduzione / voce if (/(analizza questo|analizza il|\bocr\b|trascri|leggi il file|traduci|riassumi il file)/.test(p)) return false; // task di codice puramente locale if (/^(scrivi|rifattorizza|correggi|implementa|aggiungi|rimuovi|crea|modifica|fix|refactor|aggiorna il file|genera il codice)/.test(p)) return false; // comandi / gestione conversazione if (/^(continua|ricordati|ignora|vai avanti|mostra|riassumi la conversazione|non)/.test(p)) return false; // --- Trigger positivi --- // segnali di recency / tempo corrente if (/(oggi|stamattina|adesso|al momento|quest'anno|del 20\d\d|nel 20\d\d|recent|ultim|appena|di recente|attuale|latest)/i.test(p)) return true; // versioni / package / tech aggiornati if (/(versione|version|release|changelog|release notes|npm|package|dipendenz|compatibil|lts|stabile|beta|deprecat|\beol\b|end of life|release date)/i.test(p)) return true; // confronti / alternative / raccomandazioni if (/(differenza|confronta|\bvs\b|alternativ|migliore|best|top|consigli|oppure)/i.test(p)) return true; // domande informative / definizioni if (/(cos'?è|che cos|chi è|quanto|quando|dove|perché|come funzion|cosa significa|definizion|significat|prezzo|costo|storia di)/i.test(p)) return true; // news / novità / fatti correnti if (/(news|notizie|novità|novita|ultime notizie|accadut|succes|eventi|mercato|azienda|lancio|annunci)/i.test(p)) return true; return false; } const estTokens = (s: string) => Math.ceil(s.length / 4); function truncToTokens(s: string, limit: number): string { if (estTokens(s) <= limit) return s; return s.slice(0, limit * 4); } // Assemblea del blocco di contesto con tag XML e token cap (~1500). // La richiesta utente va SEMPRE dopo (anti lost-in-the-middle). async function buildContextBlock( ctx: any, prompt: string, signal?: AbortSignal, mode?: string, ): Promise { if (getConfig("contextInject") === "off") return ""; const budget = Number(getConfig("contextTokens") ?? 1500) || 1500; const cwd = ctx?.sessionManager?.getCwd?.() ?? process.cwd(); const git = getGitInfo(cwd); const now = new Date().toISOString(); const os = `${process.platform} ${process.arch}`; const parts: { label: string; xml: string; body: string }[] = []; // --- --- const envBody = [ `cwd: ${cwd}`, `os: ${os}`, `time: ${now}`, `git_branch: ${git.branch}${git.dirty ? " (dirty)" : ""}`, git.modified.length ? `modified: ${git.modified.join(" | ")}` : "", ] .filter(Boolean) .join("\n"); parts.push({ label: "env", xml: "environment_snapshot", body: envBody }); // --- --- const sessionState = extractSessionState(ctx); if (sessionState) { parts.push({ label: "session", xml: "session_state", body: sessionState }); } // --- --- const mem = loadMemory(); const memLines = [ mem.goal ? `goal: ${mem.goal.slice(0, 120)}` : "", mem.decisions.length ? `decisions: ${mem.decisions.join(" | ")}` : "", mem.conventions.length ? `conventions: ${mem.conventions.join(" | ")}` : "", mem.openQuestions.length ? `open_questions: ${mem.openQuestions.join(" | ")}` : "", ].filter(Boolean); if (memLines.length) { parts.push({ label: "memory", xml: "durable_memory", body: memLines.join("\n") }); } // --- (Strada A) --- if (needsWebSearch(prompt, mode)) { // anti-ridondanza: non ricercare lo stesso topic entro 60s const topic = prompt.trim().toLowerCase().slice(0, 60); const now = Date.now(); if (topic !== lastSearch.topic || now - lastSearch.time > 60_000) { lastSearch = { topic, time: now }; const web = await webSearchGemini(prompt.slice(0, 300), signal); if (web) parts.push({ label: "web", xml: "web_context", body: web }); } } // Token cap: budget totale ~1500. Assegna in modo proporzionale. const blocks: string[] = []; let used = 0; const envBudget = Math.min(300, budget); const sessionBudget = Math.min(500, budget); const memBudget = Math.min(400, budget); const webBudget = Math.min(300, budget); const budgets: Record = { env: envBudget, session: sessionBudget, memory: memBudget, web: webBudget, }; for (const p of parts) { const cap = budgets[p.label] ?? 200; if (used >= budget) break; const body = truncToTokens(p.body, cap); if (!body) continue; blocks.push(`<${p.xml}> ${body} `); used += estTokens(body) + 6; } if (!blocks.length) return ""; return `[CONTEXT_AGENTE] Contesto ambiente e stato corrente per la risposta. Usa solo se pertinente; la richiesta dell'utente è sotto. ${blocks.join("\n\n")}`; } // Workaround: se un file da analizzare è FUORI dalla cartella di lavoro corrente, // lo copia in /tmp (cartella nativamente accessibile da agy) con un nome univoco // casuale e riscrive i riferimenti nel prompt. Previene errori di accesso/competenza // quando agy legge file esterni al workspace, senza sovrascrivere file esistenti. function stageExternalFiles( prompt: string, filePaths: string[], ): { prompt: string; filePaths: string[]; dirs: string[] } { const cwdBase = path.resolve(process.cwd()) + path.sep; const stagedPaths: string[] = []; let newPrompt = prompt; for (const f of filePaths) { if (!f) continue; const abs = path.resolve(f); // già sotto la cartella di lavoro corrente → nessuna copia necessaria if (abs.startsWith(cwdBase)) { stagedPaths.push(f); continue; } try { const ext = path.extname(abs); const unique = `agy_stage_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 10)}${ext}`; const dest = path.join(os.tmpdir(), unique); fs.copyFileSync(abs, dest); // riscrivi i riferimenti nel prompt (path originale e assoluto) newPrompt = newPrompt.split(f).join(dest).split(abs).join(dest); stagedPaths.push(dest); } catch { // se la copia fallisce, lascia il path originale (potrebbe comunque funzionare) stagedPaths.push(f); } } // /tmp è accessibile nativamente da agy → nessun --add-dir necessario return { prompt: newPrompt, filePaths: stagedPaths, dirs: [] }; } // Estrae dal briefing JSON prodotto dall'interpretazione vocale i campi per // l'orchestratore: trascrizione, prompt pulito, se serve ricerca web e suggerimenti. function extractVoiceBriefing(text: string): { transcript: string; cleanPrompt: string; needsSearch: boolean; searchHints: string[]; } { const result = { transcript: "", cleanPrompt: "", needsSearch: false, searchHints: [] as string[] }; try { const start = text.indexOf("{"); const end = text.lastIndexOf("}"); if (start >= 0 && end > start) { const obj = JSON.parse(text.slice(start, end + 1)); if (typeof obj.trascrizione_corretta === "string") result.transcript = obj.trascrizione_corretta.trim(); if (typeof obj.prompt_utente_pulito === "string") result.cleanPrompt = obj.prompt_utente_pulito.trim(); result.needsSearch = String(obj.ricerca_necessaria).toLowerCase() === "true"; if (Array.isArray(obj.suggerimenti_ricerca)) { result.searchHints = obj.suggerimenti_ricerca.map(String).filter(Boolean); } } } catch { /* JSON non parsato: si usa il testo grezzo come fallback */ } return result; } async function executeAgy(opts: AgyExecOptions) { // Staging dei file esterni (workaround accesso agy a file fuori dal workspace) const staged = stageExternalFiles(opts.prompt, opts.filePaths ?? []); const finalPrompt = opts.contextBlock ? `${opts.contextBlock}\n\n${staged.prompt}` : staged.prompt; const args: string[] = ["-p", finalPrompt]; // add-dir per i file (i file esterni sono stagizzati in /tmp, accessibile nativamente) const dirs = new Set(); for (const d of opts.addDirs ?? []) if (d) dirs.add(d); for (const f of staged.filePaths) if (f) dirs.add(path.dirname(f)); for (const d of dirs) args.push("--add-dir", d); // conversazione const useState = !opts.stateless && !opts.newConversation; let convId: string | null = null; if (opts.newConversation) convId = null; else if (opts.stateless) convId = null; else convId = readState(); if (convId) args.push("--conversation", convId); const model = opts.model ?? getConfig("agyDefaultModel"); if (model) args.push("--model", model); if (opts.effort) args.push("--effort", opts.effort); if (opts.yolo) args.push("--dangerously-skip-permissions"); const timeout = opts.timeoutMs ?? Number(getConfig("agyTimeoutMs") ?? DEFAULT_TIMEOUT_MS) ?? (opts.mode === "image" ? IMAGE_TIMEOUT_MS : DEFAULT_TIMEOUT_MS); const r = await runAgy(args, timeout, opts.signal); // aggiorna stato conversazione (solo se non stateless) if (r.exitCode === 0 && !opts.stateless) { const latest = getLatestConversationId(); if (latest) writeState(latest); } let text = r.output.trim(); if (r.error) { const err = r.error .split("\n") .filter((l) => !/logging before google\.Init/i.test(l)) .join("\n") .trim(); if (err && !text) text = err; } const imagePath = copyImage(extractImagePath(text), opts.outputDir); if (imagePath && opts.outputDir) { text += `\n\n[immagine copiata in: ${imagePath}]`; } return { text, imagePath, conversationId: convId, exitCode: r.exitCode, }; } // --------------------------------------------------------------------------- // Helper: costruire il suffisso "IMAGE_PATH" per i tool immagine // --------------------------------------------------------------------------- const IMG_SUFFIX = "\n\nAlla fine della risposta, scrivi su una riga esattamente: IMAGE_PATH: "; // --------------------------------------------------------------------------- // Registrazione microfono (F12) e trascrizione // --------------------------------------------------------------------------- const MAX_RECORD_MS = 120_000; // 2 min interface Recording { proc: ReturnType; file: string; startTime: number; timer?: NodeJS.Timeout; } let recording: Recording | null = null; function startRecording(ctx?: any): string { const maxDur = Number(getConfig("sttMaxDuration") ?? 120); const file = path.join(os.tmpdir(), `agy-rec-${Date.now()}.wav`); const proc = spawn( "ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", "-f", "pulse", "-i", "default", "-ac", "1", "-ar", "16000", "-t", String(maxDur), file], { stdio: "ignore" }, ); const startTime = Date.now(); let timer: NodeJS.Timeout | undefined; if (ctx) { const updateStatus = () => { const elapsedSec = Math.floor((Date.now() - startTime) / 1000); const elapsedMinStr = String(Math.floor(elapsedSec / 60)).padStart(2, "0"); const elapsedSecStr = String(elapsedSec % 60).padStart(2, "0"); const maxMinStr = String(Math.floor(maxDur / 60)).padStart(2, "0"); const maxSecStr = String(maxDur % 60).padStart(2, "0"); ctx.ui.setStatus( "agy-rec", `🔴 REGISTRAZIONE... ${elapsedMinStr}:${elapsedSecStr} / ${maxMinStr}:${maxSecStr} (Ctrl+Esc per annullare)`, ); }; updateStatus(); timer = setInterval(updateStatus, 500); } recording = { proc, file, startTime, timer }; // se ffmpeg termina da solo (timeout max durata), azzera lo stato e avvisa proc.on("exit", (code) => { if (recording && recording.proc === proc) { if (recording.timer) clearInterval(recording.timer); recording = null; playSound("timeout"); if (ctx) { ctx.ui.setStatus("agy-rec", ""); ctx.ui.notify("⏰ Tempo massimo di registrazione raggiunto", "warning"); } } }); return file; } function stopRecording(): Promise { return new Promise((resolve) => { if (!recording) return resolve(null); if (recording.timer) clearInterval(recording.timer); const { proc, file } = recording; recording = null; let done = false; const finish = () => { if (!done) { done = true; resolve(file); } }; proc.on("exit", finish); proc.kill("SIGINT"); // ffmpeg finalizza il file setTimeout(finish, 3000); // fallback }); } function cancelRecording(): Promise { return new Promise((resolve) => { if (!recording) return resolve(); if (recording.timer) clearInterval(recording.timer); const { proc, file } = recording; recording = null; let done = false; const cleanup = () => { if (!done) { done = true; try { if (fs.existsSync(file)) fs.unlinkSync(file); } catch { /* ignora */ } resolve(); } }; proc.on("exit", cleanup); proc.kill("SIGKILL"); setTimeout(cleanup, 1000); }); } // --------------------------------------------------------------------------- // Feedback sonori sci-fi (sintesi procedurale ffmpeg) // --------------------------------------------------------------------------- type SoundType = "start" | "stop" | "cancel" | "timeout" | "done"; // Espressioni aevalsrc per effetti fantascientifici const SOUNDS: Record = { // Power up: sweep ascendente 440→1760Hz start: { expr: "sin(2*PI*(440+1320*t/0.15)*t)*(1-t/0.15)", dur: "0.15" }, // Deactivate: sweep discendente 1200→300Hz stop: { expr: "sin(2*PI*(1200-900*t/0.18)*t)*(1-t/0.18)", dur: "0.18" }, // Error/Abort: modulato discendente a bassa freq cancel: { expr: "sin(2*PI*(350-200*t/0.35)*t)*exp(-2*t)*(1+0.3*sin(2*PI*8*t))", dur: "0.35" }, // Warning beacon: pulsazione bi-tono (radar) timeout: { expr: "if(lt(mod(t,0.5),0.15),sin(2*PI*700*t),if(lt(mod(t,0.5),0.35),0,sin(2*PI*500*t)))*0.8", dur: "1.2" }, // Success chime: arpeggio Do-Mi-Sol acuto done: { expr: "if(lt(t,0.08),sin(2*PI*523*t),if(lt(t,0.16),sin(2*PI*659*t),sin(2*PI*784*t)))*(1-0.3*t)", dur: "0.28" }, }; function playSound(type: SoundType) { const cfg = SOUNDS[type]; if (!cfg) return; const wav = path.join(os.tmpdir(), `agy-sound-${type}-${Date.now()}.wav`); // genera il tono sci-fi con ffmpeg (silenzioso) e riproduce con paplay/aplay/ffplay try { execFileAsync( "ffmpeg", [ "-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", `aevalsrc='${cfg.expr}':d=${cfg.dur}:s=44100`, wav, ], { timeout: 10_000 }, ) .then(() => { execFileAsync("paplay", [wav], { timeout: 5000 }).catch(() => { // fallback su aplay/ffplay se paplay non disponibile execFileAsync("aplay", ["-q", wav], { timeout: 5000 }).catch(() => { execFileAsync("ffplay", ["-nodisp", "-autoexit", wav], { timeout: 5000 }).catch(() => {}); }); }); // pulizia setTimeout(() => { try { fs.rmSync(wav, { force: true }); } catch { /* ignora */ } }, 3000); }) .catch(() => {}); } catch { /* ignora */ } } // Ottimizza l'audio: taglia il silenzio iniziale/finale e converte in WAV 16kHz mono async function optimizeAudio(input: string): Promise { const output = input.replace(/\.wav$/, "-opt.wav"); try { await execFileAsync( "ffmpeg", [ "-hide_banner", "-loglevel", "error", "-y", "-i", input, "-af", "silenceremove=start_periods=1:start_threshold=-50dB:start_silence=0.5,areverse,silenceremove=start_periods=1:start_threshold=-50dB:start_silence=0.5,areverse", "-ac", "1", "-ar", "16000", output, ], { timeout: 30_000 }, ); return output; } catch { return input; // fallback al file originale } } // Trascrizione affidabile e veloce via Gemini API diretta // (agy CLI non supporta audio; la API supporta audio/wav nativamente) // Rileva se l'audio contiene parlato reale (non solo silenzio) // Ritorna true se c'è segnale, false se è silenzio function audioHasSpeech(file: string): boolean { try { const res = spawnSync( "ffmpeg", ["-hide_banner", "-i", file, "-af", "volumedetect", "-f", "null", "-"], { timeout: 30_000, encoding: "utf8" }, ); const stderr = res.stderr || ""; const maxMatch = stderr.match(/max_volume: ([\-0-9.]+) dB/); const meanMatch = stderr.match(/mean_volume: ([\-0-9.]+) dB/); if (maxMatch) { const max = parseFloat(maxMatch[1]); // max < -35dB ≈ silenzio quasi totale if (max < -35) return false; } if (meanMatch) { const mean = parseFloat(meanMatch[1]); if (mean < -45) return false; } return true; } catch { return true; // se non possiamo verificare, assumiamo che ci sia parlato } } // Risultato trascrizione con dettaglio errore interface TranscriptResult { text: string; error?: string; } // Trascrizione affidabile e veloce via Gemini API diretta // (agy CLI non supporta audio; la API supporta audio/mpeg nativamente) async function transcribeWithGeminiAPI(file: string): Promise { // key da config file o env var let key = getConfig("geminiApiKey") ?? process.env.GEMINI_API_KEY ?? ""; if (!key) { try { key = fs.readFileSync(path.join(AGY_CHAT_DIR, "gemini-key"), "utf8").trim(); } catch { /* ignora */ } } if (!key) return { text: "", error: "Key Gemini mancante (imposta con /agy:config set geminiApiKey )" }; // rileva silenzio prima di chiamare l'API (evita allucinazioni su audio vuoto) if (!audioHasSpeech(file)) { return { text: "", error: "Nessun parlato rilevato nell'audio (silenzio o volume troppo basso)" }; } const model = process.env.AGY_GEMINI_MODEL ?? "gemini-3.5-flash"; try { // Comprimi in Opus/OGG (audio/ogg) — molto più efficiente dell'MP3 per il parlato, // riduce il payload di ~60% evitando HTTP 413 su registrazioni lunghe let audioFile = file; let mimeType = "audio/wav"; if (file.toLowerCase().endsWith(".wav")) { const ogg = file.replace(/\.wav$/i, ".ogg"); try { await execFileAsync( "ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", "-i", file, "-ac", "1", "-ar", "16000", "-c:a", "libopus", "-b:a", "16k", ogg], { timeout: 60_000 }, ); audioFile = ogg; mimeType = "audio/ogg"; } catch (e: any) { return { text: "", error: `Conversione Opus fallita: ${e.message}` }; } } const b64 = fs.readFileSync(audioFile).toString("base64"); const body = { contents: [ { parts: [ { text: "Trascrivi fedelmente il contenuto di questo audio in italiano. Restituisci SOLO la trascrizione testuale." }, { inline_data: { mime_type: mimeType, data: b64 } }, ], }, ], }; // retry su errori transitori (429, 500, 503) let lastErr = ""; for (let attempt = 0; attempt < 3; attempt++) { const res = await fetch( `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${key}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), signal: AbortSignal.timeout(120_000), }, ); if (res.ok) { const data: any = await res.json(); const text = data?.candidates?.[0]?.content?.parts?.[0]?.text?.trim() ?? ""; if (!text) { return { text: "", error: "La Gemini API ha restituito una risposta vuota" }; } return { text }; } lastErr = `HTTP ${res.status}`; const errBody = await res.text().catch(() => ""); try { const e = JSON.parse(errBody); lastErr = `${lastErr}: ${e?.error?.message ?? ""}`.trim(); } catch { /* ignora */ } if (res.status !== 429 && res.status !== 500 && res.status !== 503) break; await new Promise((r) => setTimeout(r, 1500 * (attempt + 1))); } return { text: "", error: lastErr || "Errore API Gemini sconosciuto" }; } catch (e: any) { return { text: "", error: `Errore API Gemini: ${e?.message ?? String(e)}` }; } } function extractText(content: any): string { if (typeof content === "string") return content; if (Array.isArray(content)) { return content .filter((p: any) => p && p.type === "text" && typeof p.text === "string") .map((p: any) => p.text) .join(" "); } return ""; } function getConversationContext(ctx: any, maxEntries = 8): string { try { const entries = ctx.sessionManager.getEntries(); const recent = entries.slice(-maxEntries); const lines: string[] = []; for (const e of recent) { if (e.type !== "message" || !e.message) continue; const text = extractText(e.message.content); if (!text) continue; const who = e.message.role === "user" ? "Utente" : e.message.role === "assistant" ? "Assistente" : "Strumento"; lines.push(`${who}: ${text}`); } return lines.join("\n"); } catch { return ""; } } // =========================================================================== // Opzione 4 — Workflow vocale a 2 fasi: overlay TUI con trascrizione + piano // e conferma utente prima dell'esecuzione (Enter esegui, Esc annulla, E modifica, // Spazio testo letterale nell'editor senza inviare, F12 registra di nuovo). // Riusa l'infrastruttura overlay di /agy:key e /agy:status. // ========================================================================= interface VoicePlanDecision { action: "send" | "cancel" | "record" | "literal"; text: string; } async function showVoicePlanOverlay( ctx: ExtensionContext, transcript: string, plan: string, ): Promise { const { Container, Text, matchesKey, Key } = await import("@earendil-works/pi-tui"); const { DynamicBorder } = await import("@earendil-works/pi-coding-agent"); return ctx.ui.custom( (tui, theme, _keybindings, done) => { let text = plan; let editing = false; const container = new Container(); const render = () => { container.clear(); container.addChild(new DynamicBorder((s: string) => theme.fg("accent", s))); container.addChild( new Text(theme.fg("accent", theme.bold("🎙️ Conferma vocale")), 1, 1), ); container.addChild(new Text("", 0, 0)); container.addChild(new Text(theme.fg("dim", "Trascrizione:"), 1, 0)); container.addChild(new Text(theme.fg("text", transcript.slice(0, 250)), 1, 0)); container.addChild(new Text("", 0, 0)); container.addChild(new Text(theme.fg("accent", "📋 Piano proposto:"), 1, 0)); container.addChild(new Text(theme.fg("text", text), 1, 0)); container.addChild(new Text("", 0, 0)); container.addChild( new Text( theme.fg( "dim", editing ? "✏️ Modifica: digitando cambia il testo • Enter applica • Esc annulla" : "Enter esegui • Esc annulla • E modifica • Spazio testo letterale • F12 registra di nuovo", ), 1, 0, ), ); container.addChild(new DynamicBorder((s: string) => theme.fg("accent", s))); }; render(); return { render: (w) => { render(); return container.render(w); }, invalidate: () => container.invalidate(), handleInput: (data) => { if (editing) { if (matchesKey(data, Key.enter) || matchesKey(data, Key.escape)) { editing = false; tui.requestRender(); return; } if (matchesKey(data, Key.backspace)) { text = text.slice(0, -1); tui.requestRender(); return; } if (data.length === 1 && data.charCodeAt(0) >= 32) { text += data; tui.requestRender(); } return; } if (matchesKey(data, Key.enter)) { done({ action: "send", text }); return; } if (matchesKey(data, Key.escape)) { done({ action: "cancel", text }); return; } if (matchesKey(data, "e") || data === "E") { editing = true; tui.requestRender(); return; } // Spazio: chiude l'overlay e inserisce la TRASCRIZIONE LETTERALE // nell'editor, senza inviare il prompt proposto. if (matchesKey(data, Key.space)) { done({ action: "literal", text }); return; } if (matchesKey(data, Key.f12)) { done({ action: "record", text }); return; } }, }; }, { overlay: true, overlayOptions: { width: "60%", minWidth: 60, anchor: "center" }, }, ); } // Trascrizione via server locale ai.enne2.net (gemma4:E4B supporta audio) async function transcribeWithEnne2(file: string): Promise { const baseUrl = getConfig("sttUrl") ?? "https://ai.enne2.net"; const model = getConfig("sttModel") ?? "gemma4:E4B"; const apiKey = getConfig("enne2ApiKey") ?? process.env.ENNE2_API_KEY ?? ""; // rileva silenzio if (!audioHasSpeech(file)) { return { text: "", error: "Nessun parlato rilevato nell'audio (silenzio o volume troppo basso)" }; } try { // Comprimi in MP3 per evitare HTTP 413 su audio lunghi (come il backend gemini) let audioFile = file; let audioFormat = "wav"; if (file.toLowerCase().endsWith(".wav")) { const mp3 = file.replace(/\.wav$/i, ".mp3"); try { await execFileAsync( "ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", "-i", file, "-ac", "1", "-ar", "16000", "-c:a", "libmp3lame", "-q:a", "5", mp3], { timeout: 60_000 }, ); audioFile = mp3; audioFormat = "mp3"; } catch { /* fallback al WAV */ } } const b64 = fs.readFileSync(audioFile).toString("base64"); const body = { model, messages: [ { role: "user", content: [ { type: "text", text: "Trascrivi fedelmente il parlato in questo audio. Rispondi solo con la trascrizione." }, { type: "input_audio", input_audio: { data: b64, format: audioFormat } }, ], }, ], max_tokens: 2000, }; const headers: Record = { "Content-Type": "application/json" }; if (apiKey) headers["Authorization"] = `Bearer ${apiKey}`; const res = await fetch(`${baseUrl}/v1/chat/completions`, { method: "POST", headers, body: JSON.stringify(body), signal: AbortSignal.timeout(120_000), }); if (!res.ok) { return { text: "", error: `Errore server enne2: HTTP ${res.status}` }; } const data: any = await res.json(); const text = data?.choices?.[0]?.message?.content?.trim() ?? ""; return { text }; } catch (e: any) { return { text: "", error: `Errore server enne2: ${e?.message ?? String(e)}` }; } } // Dispatcher: sceglie il backend di trascrizione (gemini | enne2) async function transcribeAudio(file: string): Promise { const backend = getConfig("sttBackend") ?? "gemini"; if (backend === "enne2" || backend === "local") { return transcribeWithEnne2(file); } return transcribeWithGeminiAPI(file); } // --------------------------------------------------------------------------- // TTS via Gemini API (nessun engine esterno) // --------------------------------------------------------------------------- async function ttsSpeak(text: string, outputDir?: string): Promise { let key = getConfig("geminiApiKey") ?? process.env.GEMINI_API_KEY ?? ""; if (!key) { try { key = fs.readFileSync(path.join(AGY_CHAT_DIR, "gemini-key"), "utf8").trim(); } catch { /* ignora */ } } if (!key) return null; const model = getConfig("ttsModel") ?? "gemini-2.5-flash-preview-tts"; try { const body = { contents: [{ parts: [{ text }] }], generationConfig: { responseModalities: ["AUDIO"] }, }; const res = await fetch( `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${key}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), signal: AbortSignal.timeout(60_000), }, ); if (!res.ok) return null; const data: any = await res.json(); const parts = data?.candidates?.[0]?.content?.parts ?? []; const audioPart = parts.find((p: any) => p.inlineData); if (!audioPart) return null; // salva PCM raw (16-bit, 24kHz) e converti in WAV const pcmFile = path.join(os.tmpdir(), `tts-${Date.now()}.pcm`); fs.writeFileSync(pcmFile, Buffer.from(audioPart.inlineData.data, "base64")); const wavFile = pcmFile.replace(".pcm", ".wav"); await execFileAsync( "ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", "-f", "s16le", "-ar", "24000", "-ac", "1", "-i", pcmFile, wavFile], { timeout: 30_000 }, ); fs.rmSync(pcmFile, { force: true }); // copia in outputDir se richiesto let finalFile = wavFile; if (outputDir) { try { fs.mkdirSync(outputDir, { recursive: true }); const dest = path.join(outputDir, `tts-${Date.now()}.wav`); fs.copyFileSync(wavFile, dest); finalFile = dest; } catch { /* ignora */ } } // riproduci (fire-and-forget) execFileAsync("paplay", [finalFile], { timeout: 60_000 }).catch(() => {}); return finalFile; } catch { return null; } } // --------------------------------------------------------------------------- // Helper: onUpdate nel formato corretto (oggetto con content, non stringa) // --------------------------------------------------------------------------- function toolUpdate(onUpdate: any, text: string) { onUpdate?.({ content: [{ type: "text", text }] }); } // ========================================================================= // Client diretto Antigravity — protocollo v1internal (cloudcode-pa) // Parla direttamente con i server di inferenza di Antigravity usando il // token OAuth dell'account (nessun subprocess agy). Endpoint, envelope e // flusso OAuth sono stati reverse-engineered e documentati pubblicamente // (opencode-antigravity-auth, antigravity-proxy, torana-edge). // // USO CONSERVATIVO: quota per-modello (retrieveUserQuota/fetchAvailableModels), // richieste serializzate (una alla volta), retry limitati (1 refresh + 1 retry // su 401/403). L'uso automatizzato massiccio fa scattare il re-auth di Google // e può portare a ban dell'account — vedi ToS Antigravity/Gemini. // ========================================================================= const ANTG_TOKEN_FILE = path.join(os.homedir(), ".gemini", "antigravity-cli", "antigravity-oauth-token"); const ANTG_OAUTH_URL = "https://oauth2.googleapis.com/token"; const ANTG_HOSTS = ["https://daily-cloudcode-pa.googleapis.com", "https://cloudcode-pa.googleapis.com"]; const ANTG_CLIENT_ID = "1071006060591-tmhssin2h21lcre235vtolojh4g403ep.apps.googleusercontent.com"; const ANTG_CLIENT_SECRET = "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"; const ANTG_UA = "antigravity/cli/1.1.17"; const ANTG_API_CLIENT = "google-cloud-sdk vscode_cloudshelleditor/0.1"; const ANTG_METADATA = JSON.stringify({ ideType: "ANTIGRAVITY", platform: "LINUX", pluginType: "GEMINI" }); interface AntgModelInfo { backend: string; thinkingLevel?: "low" | "medium" | "high"; } const ANTG_MODEL_MAP: Record = { "gemini-3.7-flash": { backend: "gemini-3.7-flash-medium" }, "gemini-3.7-flash-medium": { backend: "gemini-3.7-flash-medium" }, "gemini-3.7-flash-low": { backend: "gemini-3.7-flash-low" }, "gemini-3.7-flash-high": { backend: "gemini-3.7-flash-high" }, "gemini-3.5-flash": { backend: "gemini-3.5-flash-low" }, "gemini-3.6-flash-medium": { backend: "gemini-3.6-flash-medium" }, "gemini-3.6-flash-high": { backend: "gemini-3.6-flash-high" }, "gemini-3.1-pro": { backend: "gemini-3.1-pro-low", thinkingLevel: "low" }, "gemini-3.1-pro-high": { backend: "gemini-3.1-pro-high", thinkingLevel: "high" }, "claude-sonnet-4.6": { backend: "claude-sonnet-4-6" }, "claude-opus-4.6": { backend: "claude-opus-4-6-thinking" }, "gpt-oss-120b": { backend: "gpt-oss-120b-medium" }, }; const ANTG_DEFAULT_MODEL = "gemini-3.7-flash-medium"; let antgToken: { access: string; expiresAtMs: number; refresh: string } | null = null; let antgProject: { pid: string; base: string } | null = null; let antgQueue: Promise = Promise.resolve(); // serializzazione richieste function antgParseExpiryMs(s: string | undefined): number { if (!s) return 0; if (typeof s === "number") return s * 1000; // RFC3339 con nanosecondi: tronca la frazione ai ms per Date.parse const m = String(s).match(/^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(\.\d+)?(Z|[+-]\d{2}:\d{2})?$/); if (!m) return 0; const ms = Date.parse(m[1] + (m[2] ? m[2].slice(0, 4) : "") + (m[3] || "Z")); return Number.isNaN(ms) ? 0 : ms; } function antgReadTokenFile(): any { try { return JSON.parse(fs.readFileSync(ANTG_TOKEN_FILE, "utf8")); } catch { return null; } } function antgWriteTokenFile(data: any) { try { const tmp = ANTG_TOKEN_FILE + ".tmp"; fs.writeFileSync(tmp, JSON.stringify(data, null, 2), { mode: 0o600 }); fs.renameSync(tmp, ANTG_TOKEN_FILE); } catch { /* ignora */ } } async function antgRefreshToken(refreshToken: string): Promise<{ access: string; expiresAtMs: number; refresh: string }> { const resp = await fetch(ANTG_OAUTH_URL, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ client_id: ANTG_CLIENT_ID, client_secret: ANTG_CLIENT_SECRET, refresh_token: refreshToken, grant_type: "refresh_token", }), signal: AbortSignal.timeout(30_000), }); const payload: any = await resp.json().catch(() => ({})); if (!resp.ok) throw new Error(`OAuth refresh fallito (HTTP ${resp.status}): ${JSON.stringify(payload).slice(0, 200)}`); const expiresIn = Number(payload.expires_in ?? 3600); return { access: payload.access_token, expiresAtMs: Date.now() + expiresIn * 1000, refresh: payload.refresh_token || refreshToken, }; } async function antgGetAccessToken(): Promise { if (antgToken && antgToken.expiresAtMs > Date.now() + 120_000) return antgToken.access; const data = antgReadTokenFile(); if (!data?.token?.refresh_token) { throw new Error("Nessun token OAuth Antigravity: avvia `agy` almeno una volta per autenticarti con l'account."); } const tok = data.token; const expMs = antgParseExpiryMs(tok.expiry); if (tok.access_token && expMs > Date.now() + 120_000) { antgToken = { access: tok.access_token, expiresAtMs: expMs, refresh: tok.refresh_token }; return tok.access_token; } const fresh = await antgRefreshToken(tok.refresh_token); data.token = { ...tok, access_token: fresh.access, token_type: "Bearer", expiry: new Date(fresh.expiresAtMs).toISOString().replace(/\.\d{3}Z$/, ".000000Z"), }; antgWriteTokenFile(data); antgToken = fresh; return fresh.access; } function antgDeepFind(obj: any, key: string): any { if (obj && typeof obj === "object") { if (key in obj) return obj[key]; for (const v of Object.values(obj)) { const r = antgDeepFind(v, key); if (r != null) return r; } } return undefined; } function antgHeaders(token: string, stream = false): Record { return { Authorization: `Bearer ${token}`, "Content-Type": "application/json", "User-Agent": ANTG_UA, "X-Goog-Api-Client": ANTG_API_CLIENT, "Client-Metadata": ANTG_METADATA, ...(stream ? { Accept: "text/event-stream" } : {}), }; } async function antgLoadCodeAssist(token: string, base: string): Promise { const resp = await fetch(`${base}/v1internal:loadCodeAssist`, { method: "POST", headers: antgHeaders(token), body: "{}", signal: AbortSignal.timeout(60_000), }); const payload: any = await resp.json().catch(() => ({})); if (!resp.ok) throw new Error(`loadCodeAssist HTTP ${resp.status}`); const pid = antgDeepFind(payload, "cloudaicompanionProject") ?? antgDeepFind(payload, "cloudaicompanion_project"); if (!pid) throw new Error("loadCodeAssist senza project id"); return String(pid); } async function antgGetProject(): Promise<{ pid: string; base: string }> { if (antgProject) return antgProject; const token = await antgGetAccessToken(); let lastErr = ""; for (const base of ANTG_HOSTS) { try { antgProject = { pid: await antgLoadCodeAssist(token, base), base }; return antgProject; } catch (e: any) { lastErr = e.message; } } throw new Error(`Discovery project fallito: ${lastErr}`); } interface AntgGenerateOpts { prompt?: string; parts?: any[]; model?: string; system?: string; maxOutputTokens?: number; temperature?: number; thinking?: "low" | "medium" | "high" | "off"; stream?: boolean; } function antgResolveModel(friendly: string | undefined): AntgModelInfo { if (friendly && ANTG_MODEL_MAP[friendly]) return ANTG_MODEL_MAP[friendly]; if (friendly) return { backend: friendly }; return ANTG_MODEL_MAP[ANTG_DEFAULT_MODEL]!; } async function antgReadStream(resp: any): Promise<{ text: string; details: any }> { const reader = resp.body?.getReader(); if (!reader) throw new Error("Nessun body streaming"); const decoder = new TextDecoder(); let buf = "", text = "", finish = "", usage: any, modelVersion = "", responseId = "", events = 0; while (true) { const { done, value } = await reader.read(); if (done) break; // il framing SSE usa CRLF: normalizza a \n buf += decoder.decode(value, { stream: true }).replace(/\r\n/g, "\n"); let i; while ((i = buf.indexOf("\n\n")) >= 0) { const chunk = buf.slice(0, i); buf = buf.slice(i + 2); for (const line of chunk.split("\n")) { if (!line.startsWith("data:")) continue; const d = line.slice(5).trim(); if (!d) continue; events++; try { const o = JSON.parse(d); const inner = o.response ?? o; for (const pt of inner.candidates?.[0]?.content?.parts ?? []) if (pt.text) text += pt.text; if (inner.candidates?.[0]?.finishReason) finish = inner.candidates[0].finishReason; if (inner.usageMetadata) usage = inner.usageMetadata; if (inner.modelVersion) modelVersion = inner.modelVersion; if (inner.responseId) responseId = inner.responseId; } catch { /* evento non-JSON: ignora */ } } } } return { text: text.trim(), details: { finishReason: finish || undefined, usage, modelVersion, responseId, events }, }; } async function antgGenerate(opts: AntgGenerateOpts): Promise<{ text: string; details: any }> { // serializza: una richiesta alla volta (uso conservativo del canale account) const run = antgQueue.then(async () => { const { pid, base } = await antgGetProject(); const mi = antgResolveModel(opts.model); const requestId = `agent-${crypto.randomUUID().replace(/-/g, "")}`; const gc: any = { maxOutputTokens: opts.maxOutputTokens ?? 8192, temperature: opts.temperature ?? 0.4, }; if (mi.thinkingLevel) gc.thinkingConfig = { thinkingLevel: mi.thinkingLevel }; if (opts.thinking && opts.thinking !== "off") gc.thinkingConfig = { thinkingLevel: opts.thinking }; const userParts = opts.parts ?? (opts.prompt ? [{ text: opts.prompt }] : []); const request: any = { contents: [{ role: "user", parts: userParts }], generationConfig: gc, }; if (opts.system) request.systemInstruction = { parts: [{ text: opts.system }] }; const envelope = { project: pid, model: mi.backend, request, requestType: "agent", userAgent: "antigravity", requestId, }; const doCall = async (token: string) => { if (opts.stream !== false) { const resp = await fetch(`${base}/v1internal:streamGenerateContent?alt=sse`, { method: "POST", headers: antgHeaders(token, true), body: JSON.stringify(envelope), signal: AbortSignal.timeout(300_000), }); if (!resp.ok) throw new Error(`streamGenerateContent HTTP ${resp.status}`); return await antgReadStream(resp); } const resp = await fetch(`${base}/v1internal:generateContent`, { method: "POST", headers: antgHeaders(token), body: JSON.stringify(envelope), signal: AbortSignal.timeout(180_000), }); const raw = await resp.text(); if (!resp.ok) throw new Error(`generateContent HTTP ${resp.status}: ${raw.slice(0, 200)}`); const p = JSON.parse(raw); const inner = p.response ?? p; return { text: (inner.candidates?.[0]?.content?.parts ?? []).map((x: any) => x.text ?? "").join("").trim(), details: { finishReason: inner.candidates?.[0]?.finishReason, usage: inner.usageMetadata, modelVersion: inner.modelVersion, responseId: inner.responseId, }, }; }; try { return await doCall(await antgGetAccessToken()); } catch (e: any) { // 401/403 → un solo refresh + retry; niente loop if (/401|403|UNAUTHENTICATED/.test(String(e.message))) { antgToken = null; return await doCall(await antgGetAccessToken()); } throw e; } }); antgQueue = run.catch(() => undefined); return run as Promise<{ text: string; details: any }>; } // --------------------------------------------------------------------------- // Pipeline Multimodale Diretta (Gemini): invio diretto dell'audio senza STT // --------------------------------------------------------------------------- function isGeminiModel(model: any): boolean { if (!model) return true; // se non specificato, default ad Antigravity/Gemini const id = (model.id || "").toLowerCase(); const provider = (model.provider || "").toLowerCase(); if (id.startsWith("gemini-") || id.includes("gemini")) return true; if (provider === "google" || provider === "google-vertex") return true; if (provider === "antigravity") { if (id.startsWith("claude-") || id.startsWith("gpt-oss")) return false; return true; } return false; } interface AudioInterpretationResult { transcript: string; cleanPrompt: string; needsSearch: boolean; searchHints: string[]; rawText: string; } async function interpretAudioDirectGemini( audioFile: string, editorText: string, context: string, targetModel?: string, ): Promise { if (!audioHasSpeech(audioFile)) { throw new Error("Nessun parlato rilevato nell'audio (silenzio o volume troppo basso)"); } let sendFile = audioFile; let mimeType = "audio/wav"; if (audioFile.toLowerCase().endsWith(".wav")) { const ogg = audioFile.replace(/\.wav$/i, ".ogg"); try { await execFileAsync( "ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", "-i", audioFile, "-ac", "1", "-ar", "16000", "-c:a", "libopus", "-b:a", "16k", ogg], { timeout: 60_000 }, ); sendFile = ogg; mimeType = "audio/ogg"; } catch { /* fallback su wav */ } } const b64 = fs.readFileSync(sendFile).toString("base64"); const promptInstructions = `[CONTESTO INTERNO — COMUNICAZIONE TRA AGENTI]\n` + `Sei un analista tecnico/middleware per un agente AI orchestratore. Ascolta la traccia audio allegata (e l'eventuale testo dell'editor) e produci un briefing strutturato per l'orchestratore. NON rispondere all'utente: il tuo output sarà letto SOLO dall'orchestratore, che poi eseguirà le azioni.` + (editorText ? `\n\nTesto scritto dall'utente nel campo di input (da combinare con la voce):\n${editorText}` : "") + `\n\nContesto della conversazione precedente:\n${context || "(nessuno)"}` + `\n\nRestituisci UN SOLO oggetto JSON (nessun markdown aggiuntivo tranne il blocco json) con questi campi esatti:` + `\n{"trascrizione_corretta":"...","intent_analisi":"...","note_per_agent":"...","azioni_raccomandate":["..."],"prompt_utente_pulito":"...","ricerca_necessaria":true/false,"suggerimenti_ricerca":["..."]}` + `\nRegole:` + `\n- trascrizione_corretta: la trascrizione fedele e completa delle parole pronunciate nell'audio in italiano.` + `\n- prompt_utente_pulito: la richiesta rielaborata e pulita che l'orchestratore userà come prompt effettivo.` + `\n- NON eseguire alcuno strumento o azione; solo analisi, trascrizione e briefing in JSON.` + `\n- ricerca_necessaria=true se serve verificare best practices, documentazione o librerie aggiornate.` + `\n- suggerimenti_ricerca: termini chiave per la ricerca web se necessaria.`; const parts = [ { text: promptInstructions }, { inlineData: { mimeType, data: b64 } }, ]; const model = targetModel && targetModel.startsWith("gemini-") ? targetModel : "gemini-3.7-flash-medium"; try { const res = await antgGenerate({ parts, model, maxOutputTokens: 3000, thinking: "low", stream: true, }); const rawText = res.text.trim(); const briefing = extractVoiceBriefing(rawText); return { transcript: briefing.transcript || briefing.cleanPrompt || rawText, cleanPrompt: briefing.cleanPrompt || rawText, needsSearch: briefing.needsSearch, searchHints: briefing.searchHints, rawText, }; } catch (err: any) { // Fallback su Gemini API diretta se Antigravity account token fallisce let key = getConfig("geminiApiKey") ?? process.env.GEMINI_API_KEY ?? ""; if (!key) { try { key = fs.readFileSync(path.join(AGY_CHAT_DIR, "gemini-key"), "utf8").trim(); } catch { /* ignora */ } } if (key) { const apiModel = process.env.AGY_GEMINI_MODEL ?? "gemini-3.5-flash"; const body = { contents: [{ parts: [{ text: promptInstructions }, { inline_data: { mime_type: mimeType, data: b64 } }] }], }; const res = await fetch( `https://generativelanguage.googleapis.com/v1beta/models/${apiModel}:generateContent?key=${key}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), signal: AbortSignal.timeout(120_000), }, ); if (res.ok) { const data: any = await res.json(); const rawText = data?.candidates?.[0]?.content?.parts?.[0]?.text?.trim() ?? ""; const briefing = extractVoiceBriefing(rawText); return { transcript: briefing.transcript || briefing.cleanPrompt || rawText, cleanPrompt: briefing.cleanPrompt || rawText, needsSearch: briefing.needsSearch, searchHints: briefing.searchHints, rawText, }; } } throw err; } } // ========================================================================= // Provider "antigravity" — modelli del gateway come provider pi nativo // Registrato con pi.registerProvider() + streamSimple: appare nel selettore // modelli (e in `pi --list-models`). Riusa il client v1internal qui sopra. // NOTA: canale account — quota per-modello e rischio ToS/ban reale se usato // come default con traffico continuo. Preferire per uso selettivo. // ========================================================================= interface AntgProviderModelDef { id: string; name: string; reasoning: boolean; images: boolean; thinkingLevelMap?: ThinkingLevelMap; contextWindow: number; maxTokens: number; } const ANTG_THINK_MAP: ThinkingLevelMap = { off: null, minimal: "low", low: "low", medium: "medium", high: "high", xhigh: "high", max: "high", }; const ANTG_PROVIDER_MODELS: AntgProviderModelDef[] = [ { id: "gemini-3.7-flash-low", name: "Gemini 3.7 Flash (Low)", reasoning: true, images: true, thinkingLevelMap: ANTG_THINK_MAP, contextWindow: 1048576, maxTokens: 65536 }, { id: "gemini-3.7-flash-medium", name: "Gemini 3.7 Flash (Medium)", reasoning: true, images: true, thinkingLevelMap: ANTG_THINK_MAP, contextWindow: 1048576, maxTokens: 65536 }, { id: "gemini-3.7-flash-high", name: "Gemini 3.7 Flash (High)", reasoning: true, images: true, thinkingLevelMap: ANTG_THINK_MAP, contextWindow: 1048576, maxTokens: 65536 }, { id: "gemini-3.5-flash-low", name: "Gemini 3.5 Flash (Medium)", reasoning: true, images: true, thinkingLevelMap: ANTG_THINK_MAP, contextWindow: 1048576, maxTokens: 65536 }, { id: "gemini-3.6-flash-medium", name: "Gemini 3.6 Flash (Medium)", reasoning: true, images: true, thinkingLevelMap: ANTG_THINK_MAP, contextWindow: 1048576, maxTokens: 65536 }, { id: "gemini-3.6-flash-high", name: "Gemini 3.6 Flash (High)", reasoning: true, images: true, thinkingLevelMap: ANTG_THINK_MAP, contextWindow: 1048576, maxTokens: 65536 }, { id: "gemini-3.1-pro-low", name: "Gemini 3.1 Pro (Low)", reasoning: true, images: true, thinkingLevelMap: ANTG_THINK_MAP, contextWindow: 1048576, maxTokens: 65536 }, { id: "gemini-3.1-pro-high", name: "Gemini 3.1 Pro (High)", reasoning: true, images: true, thinkingLevelMap: ANTG_THINK_MAP, contextWindow: 1048576, maxTokens: 65536 }, { id: "claude-sonnet-4-6", name: "Claude Sonnet 4.6 (Thinking)", reasoning: true, images: true, thinkingLevelMap: { off: null }, contextWindow: 1000000, maxTokens: 64000 }, { id: "claude-opus-4-6-thinking", name: "Claude Opus 4.6 (Thinking)", reasoning: true, images: true, thinkingLevelMap: { off: null }, contextWindow: 1000000, maxTokens: 64000 }, { id: "gpt-oss-120b-medium", name: "GPT-OSS 120B (Medium)", reasoning: false, images: false, contextWindow: 262144, maxTokens: 32768 }, ]; // Modelli attualmente registrati nel provider (statici all'avvio, aggiornabili // con /agy:refresh-models dal catalogo vivo fetchAvailableModels). let antgRegisteredModels: AntgProviderModelDef[] = ANTG_PROVIDER_MODELS; // Configurazione del provider "antigravity" (riusata all'avvio e al refresh). function antgProviderConfig(models: AntgProviderModelDef[]) { return { name: "Antigravity (account)", baseUrl: ANTG_HOSTS[0], apiKey: "antigravity", api: "antigravity", models: models.map((m) => { const input: ("text" | "image")[] = m.images ? ["text", "image"] : ["text"]; return { id: m.id, name: m.name, reasoning: m.reasoning, thinkingLevelMap: m.thinkingLevelMap, input, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: m.contextWindow, maxTokens: m.maxTokens, }; }), streamSimple: streamAntigravity, }; } // Scarica il catalogo modelli vivo dall'account e lo filtra: niente modelli // interni (displayName vuoto, chat_*, tab_*), niente placeholder, niente // gemini-2.5-* (ritirati → HTTP 429). Restituisce la lista per il provider. async function antgFetchCatalog(token: string, base: string): Promise { const resp = await fetch(`${base}/v1internal:fetchAvailableModels`, { method: "POST", headers: antgHeaders(token), body: "{}", signal: AbortSignal.timeout(60_000), }); const raw = await resp.text(); if (!resp.ok) throw new Error(`fetchAvailableModels HTTP ${resp.status}: ${raw.slice(0, 200)}`); const payload = JSON.parse(raw); const catalog: Record = payload.models ?? {}; const out: AntgProviderModelDef[] = []; for (const [backendId, info] of Object.entries(catalog)) { const m = info as any; const name = m.displayName; if (!name || typeof name !== "string" || !name.trim()) continue; // interni/autocomplete if (m.isInternal) continue; if (backendId.startsWith("chat_") || backendId.startsWith("tab_")) continue; if (backendId.includes("MODEL_PLACEHOLDER")) continue; if (backendId.startsWith("gemini-2.5-")) continue; // ritirati (429) const ctx = Number(m.maxTokens); if (!ctx) continue; // modelli senza contesto (es. gemini-3.1-flash-image, generazione immagini) const reasoning = m.supportsThinking === true; const images = m.supportsImages === true; const thinkingLevelMap = reasoning ? (backendId.startsWith("gemini-") ? ANTG_THINK_MAP : { off: null }) : undefined; out.push({ id: backendId, name, reasoning, images, thinkingLevelMap, contextWindow: ctx, maxTokens: Number(m.maxOutputTokens) || 65536, }); } out.sort((a, b) => a.id.localeCompare(b.id)); return out; } // Campi accettati dal gateway (validazione protobuf stretta — i campi ignoti // tipo $defs/$ref/$schema causano HTTP 400 INVALID_ARGUMENT). Perplexity + // riproduzione locale confermano: solo type/properties/required/items/enum/… const ANTG_SCHEMA_ALLOWED = new Set(["type", "description", "properties", "required", "items", "enum", "format", "nullable", "minimum", "maximum"]); function antgToGeminiParameters(tschema: any): any { try { const root = JSON.parse(JSON.stringify(tschema)); const visit = (node: any): any => { if (Array.isArray(node)) return node.map(visit); if (node === null || typeof node !== "object") return node; const out: any = {}; for (const [k, v] of Object.entries(node)) { if (!ANTG_SCHEMA_ALLOWED.has(k)) continue; if (k === "properties" && v && typeof v === "object" && !Array.isArray(v)) { out.properties = Object.fromEntries(Object.entries(v as any).map(([n, c]) => [n, visit(c)])); } else if (k === "items") { out.items = visit(v); } else { out[k] = visit(v); } } return out; }; return visit(root); } catch { return { type: "object", properties: {} }; } } function antgAbortSignal(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal { if (!signal) return AbortSignal.timeout(timeoutMs); const c = new AbortController(); const t = setTimeout(() => c.abort(), timeoutMs); const onAbort = () => c.abort(signal.reason); signal.addEventListener("abort", onAbort, { once: true }); c.signal.addEventListener("abort", () => { clearTimeout(t); signal.removeEventListener("abort", onAbort); }, { once: true }); return c.signal; } function antgWithQueue(fn: () => Promise): Promise { const p = antgQueue.then(() => fn()); antgQueue = p.catch(() => undefined); return p; } function isValidGeminiThoughtSignature(sig: any): sig is string { if (typeof sig !== "string" || !sig.trim()) return false; // I payload di reasoning di altri provider possono essere stringhe JSON come {"id":"rs_..."} // o etichette come "reasoning_content" / "thought" che Google rifiuta con Base64 decoding failed. if (sig.startsWith("{") || sig.startsWith("[") || sig === "reasoning_content" || sig === "thought" || sig.length < 32) { return false; } if (!/^[A-Za-z0-9+/=_-]+$/.test(sig)) return false; try { const buf = Buffer.from(sig, "base64"); if (buf.length < 24) return false; // Le firme crittografiche Gemini contengono byte binari (non solo testo ASCII leggibile) const str = buf.toString("utf8"); if (/^[a-zA-Z0-9_ -]+$/.test(str)) return false; return true; } catch { return false; } } function antgBuildGeminiRequest(model: Model, context: Context, options?: SimpleStreamOptions): any { const contents: any[] = []; const push = (role: "user" | "model", part: any) => { const last = contents[contents.length - 1]; if (last && last.role === role) last.parts.push(part); else contents.push({ role, parts: [part] }); }; const signedToolCallIds = new Set(); for (const msg of context.messages) { if (msg.role === "user") { const items = typeof msg.content === "string" ? [{ type: "text", text: msg.content }] : msg.content; for (const b of items as any[]) { if (b.type === "text") push("user", { text: b.text }); else if (b.type === "image") push("user", { inlineData: { mimeType: b.mimeType, data: b.data } }); } } else if (msg.role === "assistant") { const isSame = msg.provider === model.provider && msg.model === model.id; const parts: any[] = []; for (const b of msg.content) { if (b.type === "text") { const rawSig = b.textSignature || (isSame ? (b as any).thoughtSignature : undefined); const sig = isValidGeminiThoughtSignature(rawSig) ? rawSig : undefined; if ((!b.text || !b.text.trim()) && !sig) continue; parts.push({ text: b.text, ...(sig ? { thoughtSignature: sig } : {}) }); } else if (b.type === "thinking") { const rawSig = b.thinkingSignature || (isSame ? (b as any).thoughtSignature : undefined); const sig = isValidGeminiThoughtSignature(rawSig) ? rawSig : undefined; if ((!b.thinking || !b.thinking.trim()) && !sig) continue; parts.push({ thought: true, text: b.thinking, ...(sig ? { thoughtSignature: sig } : {}) }); } else if (b.type === "toolCall") { const rawSig = b.thoughtSignature || (b as any).thought_signature; const sig = isValidGeminiThoughtSignature(rawSig) ? rawSig : undefined; if (sig) { if (b.id) signedToolCallIds.add(b.id); if (b.name) signedToolCallIds.add(b.name); parts.push({ functionCall: { name: b.name, args: b.arguments ?? {} }, thoughtSignature: sig, }); } else { // Chiamata non firmata (cross-model o da provider terzo come DeepSeek): Google Gemini 2.5/3.x // rigetta con HTTP 400 i functionCall privi di valida Base64 thought_signature. // La serializziamo come testo per mantenere il contesto senza causare l'errore 400. parts.push({ text: `[Tool Call: ${b.name}]\nArgs: ${JSON.stringify(b.arguments ?? {})}`, }); } } } for (const p of parts) push("model", p); } else if (msg.role === "toolResult") { const items = typeof msg.content === "string" ? [{ type: "text", text: msg.content }] : msg.content; const text = items.map((c: any) => (c.type === "text" ? c.text : "")).join("\n"); const isSigned = (msg.toolCallId && signedToolCallIds.has(msg.toolCallId)) || signedToolCallIds.has(msg.toolName); if (isSigned) { push("user", { functionResponse: { name: msg.toolName, response: { result: text, isError: !!msg.isError } } }); } else { push("user", { text: `[Tool Result: ${msg.toolName}]\n${text}` }); } } } const gc: any = { maxOutputTokens: model.maxTokens || 8192 }; if (model.id.startsWith("gemini-") && model.reasoning) { const level = options?.reasoning ?? "low"; const antgLevel = level === "low" || level === "minimal" ? "low" : level === "medium" ? "medium" : "high"; gc.thinkingConfig = { thinkingLevel: antgLevel, includeThoughts: true }; } const request: any = { contents, generationConfig: gc }; if (context.systemPrompt) request.systemInstruction = { parts: [{ text: context.systemPrompt }] }; if (context.tools?.length) { request.tools = [{ functionDeclarations: context.tools.map((t) => ({ name: t.name, description: t.description, parameters: antgToGeminiParameters(t.parameters), })), }]; } return request; } async function antgConsumeSse(resp: any, model: Model, output: AssistantMessage, stream: AssistantMessageEventStream) { const reader = resp.body?.getReader(); if (!reader) throw new Error("Nessun body streaming"); const decoder = new TextDecoder(); let buf = "", finishReason = "", usageMeta: any; const blocks = output.content; let currentBlock: any = null; const blockIndex = () => blocks.length - 1; const endCurrent = () => { if (!currentBlock) return; if (currentBlock.type === "text") stream.push({ type: "text_end", contentIndex: blockIndex(), content: currentBlock.text, partial: output }); else stream.push({ type: "thinking_end", contentIndex: blockIndex(), content: currentBlock.thinking, partial: output }); currentBlock = null; }; let toolCallCounter = 0; while (true) { const { done, value } = await reader.read(); if (done) break; // il framing SSE usa CRLF: normalizza a \n buf += decoder.decode(value, { stream: true }).replace(/\r\n/g, "\n"); let i; while ((i = buf.indexOf("\n\n")) >= 0) { const chunk = buf.slice(0, i); buf = buf.slice(i + 2); for (const line of chunk.split("\n")) { if (!line.startsWith("data:")) continue; const d = line.slice(5).trim(); if (!d) continue; let o: any; try { o = JSON.parse(d); } catch { continue; } const inner = o.response ?? o; const candidate = inner.candidates?.[0]; if (candidate?.content?.parts) { for (const part of candidate.content.parts) { const sig = part.thoughtSignature || part.thought_signature || candidate.content?.thoughtSignature || candidate.content?.thought_signature; if (part.text !== undefined) { const isThinking = part.thought === true; if (!currentBlock || (isThinking && currentBlock.type !== "thinking") || (!isThinking && currentBlock.type !== "text")) { endCurrent(); if (isThinking) { currentBlock = { type: "thinking", thinking: "", thinkingSignature: sig }; blocks.push(currentBlock); stream.push({ type: "thinking_start", contentIndex: blockIndex(), partial: output }); } else { currentBlock = { type: "text", text: "", textSignature: sig }; blocks.push(currentBlock); stream.push({ type: "text_start", contentIndex: blockIndex(), partial: output }); } } if (currentBlock.type === "thinking") { currentBlock.thinking += part.text; if (sig) currentBlock.thinkingSignature = sig; stream.push({ type: "thinking_delta", contentIndex: blockIndex(), delta: part.text, partial: output }); } else { currentBlock.text += part.text; if (sig) currentBlock.textSignature = sig; stream.push({ type: "text_delta", contentIndex: blockIndex(), delta: part.text, partial: output }); } } if (part.functionCall) { endCurrent(); const tc: ToolCall = { type: "toolCall", id: part.functionCall.id || `${part.functionCall.name}_${Date.now()}_${++toolCallCounter}`, name: part.functionCall.name || "", arguments: part.functionCall.args ?? {}, ...(sig ? { thoughtSignature: sig } : {}), }; blocks.push(tc); const ci = blockIndex(); stream.push({ type: "toolcall_start", contentIndex: ci, partial: output }); stream.push({ type: "toolcall_end", contentIndex: ci, toolCall: tc, partial: output }); } } } if (inner.usageMetadata) usageMeta = inner.usageMetadata; if (inner.candidates?.[0]?.finishReason) finishReason = inner.candidates[0].finishReason; if (inner.responseId && !output.responseId) output.responseId = inner.responseId; } } } endCurrent(); if (usageMeta) { output.usage.input = usageMeta.promptTokenCount ?? 0; output.usage.output = (usageMeta.candidatesTokenCount ?? 0) + (usageMeta.thoughtsTokenCount ?? 0); output.usage.reasoning = usageMeta.thoughtsTokenCount ?? 0; output.usage.totalTokens = usageMeta.totalTokenCount ?? (output.usage.input + output.usage.output); output.usage.cost = calculateCost(model, output.usage); } output.rawStopReason = finishReason || undefined; output.stopReason = blocks.some((b) => b.type === "toolCall") ? "toolUse" : finishReason === "MAX_TOKENS" ? "length" : "stop"; } function streamAntigravity(model: Model, context: Context, options?: SimpleStreamOptions): AssistantMessageEventStream { const stream = createAssistantMessageEventStream(); const output: AssistantMessage = { role: "assistant", content: [], api: model.api, provider: model.provider, model: model.id, usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, stopReason: "pending", timestamp: Date.now(), }; (async () => { try { stream.push({ type: "start", partial: output }); await antgWithQueue(async () => { const { pid, base } = await antgGetProject(); const token = await antgGetAccessToken(); const envelope = { project: pid, model: model.id, request: antgBuildGeminiRequest(model, context, options), requestType: "agent", userAgent: "antigravity", requestId: `agent-${crypto.randomUUID().replace(/-/g, "")}`, }; const signal = antgAbortSignal(options?.signal, 300_000); const call = (tok: string) => fetch(`${base}/v1internal:streamGenerateContent?alt=sse`, { method: "POST", headers: antgHeaders(tok, true), body: JSON.stringify(envelope), signal, }); let resp = await call(token); if (resp.status === 401 || resp.status === 403) { antgToken = null; resp = await call(await antgGetAccessToken()); } if (!resp.ok) { const bodyText = await resp.text().catch(() => ""); throw new Error(`Antigravity HTTP ${resp.status}: ${bodyText.slice(0, 300)}`); } await antgConsumeSse(resp, model, output, stream); }); if (output.stopReason === "pending") throw new Error("Provider stream terminato senza stop reason"); if (output.stopReason === "error" || output.stopReason === "aborted") throw new Error(output.errorMessage || "Errore sconosciuto"); stream.push({ type: "done", reason: output.stopReason as "stop" | "length" | "toolUse" | "deferred", message: output }); stream.end(); } catch (error) { output.stopReason = options?.signal?.aborted ? "aborted" : "error"; output.errorMessage = error instanceof Error ? error.message : String(error); stream.push({ type: "error", reason: output.stopReason as "aborted" | "error", error: output }); stream.end(); } })(); return stream; } // --------------------------------------------------------------------------- // Estensione // --------------------------------------------------------------------------- export default function agyExtension(pi: ExtensionAPI) { // ========================================================================= // TOOL: agy — generico (chat / image / analyze) // ========================================================================= pi.registerTool({ name: "agy", label: "agy (Antigravity subagent)", description: "Use Google Antigravity CLI for text-only multi-turn reasoning or general chat. " + "Do NOT use for image generation/editing, image analysis, audio transcription, video analysis, or text-to-speech: " + "use the dedicated agy_* tool instead.", parameters: Type.Object({ prompt: Type.String({ description: "Il task/prompt da dare a agy." }), mode: Type.Optional( Type.Union([Type.Literal("chat"), Type.Literal("image"), Type.Literal("analyze")], { description: "chat only (default); image/analyze are legacy modes—use dedicated tools.", }), ), model: Type.Optional(Type.String({ description: "Modello agy (es. 'Gemini 3.1 Pro (High)')." })), effort: Type.Optional( Type.Union([Type.Literal("low"), Type.Literal("medium"), Type.Literal("high")]), ), newConversation: Type.Optional(Type.Boolean({ description: "true per forzare una nuova conversazione." })), addDir: Type.Optional(Type.String({ description: "Cartella da aggiungere al workspace agy." })), filePath: Type.Optional(Type.String({ description: "Optional attachment for legacy use; prefer the dedicated image, audio, or video tool." })), yolo: Type.Optional(Type.Boolean({ description: "true per --dangerously-skip-permissions." })), injectContext: Type.Optional( Type.Boolean({ description: "Inietta contesto ambiente/sessione/web nel prompt (default true)." }), ), webSearch: Type.Optional( Type.Boolean({ description: "Forza/non forzare la ricerca web (override di webSearch config)." }), ), }), async execute(toolCallId, params, signal, onUpdate, ctx) { const p = params as any; toolUpdate(onUpdate, `agy: ${p.mode === "image" ? "generazione immagine" : "elaborazione"}...`); // Fase 2: costruzione contesto iniettato (ambiente + sessione + memoria + web) const wantInject = p.injectContext ?? true; const prevWs = getConfig("webSearch"); if (typeof p.webSearch === "boolean") setConfig("webSearch", p.webSearch ? "on" : "off"); const contextBlock = wantInject ? await buildContextBlock(ctx, p.prompt, signal, p.mode) : ""; if (typeof p.webSearch === "boolean" && prevWs) setConfig("webSearch", prevWs); const res = await executeAgy({ prompt: p.prompt, mode: p.mode, model: p.model, effort: p.effort, newConversation: p.newConversation, addDirs: p.addDir ? [p.addDir] : [], filePaths: p.filePath ? [p.filePath] : [], yolo: p.yolo, signal, contextBlock, }); return { content: [{ type: "text", text: res.text }], details: { conversationId: res.conversationId, exitCode: res.exitCode, imagePath: res.imagePath, model: p.model }, }; }, }); // ========================================================================= // TOOL: agy_generate — generazione immagine strutturata // ========================================================================= pi.registerTool({ name: "agy_generate", label: "agy generate image", description: "Generate a new image from a structured subject, action, location, composition, style, lighting, and format. " + "Use ONLY for image generation from scratch, not editing or image analysis.", parameters: Type.Object({ subject: Type.String({ description: "Soggetto: chi/cosa è nell'immagine. Sii specifico." }), action: Type.Optional(Type.String({ description: "Azione: cosa sta succedendo." })), location: Type.Optional(Type.String({ description: "Luogo/contesto/sfondo." })), composition: Type.Optional( Type.String({ description: "Composizione: inquadratura (es. 'close-up', 'wide shot', 'low-angle')." }), ), style: Type.Optional(Type.String({ description: "Stile: estetica (es. 'fotorealistico', 'watercolor', 'film noir')." })), lighting: Type.Optional(Type.String({ description: "Illuminazione (es. 'golden hour', 'softbox', 'neon')." })), aspectRatio: Type.Optional( Type.String({ description: "Formato (es. '1:1', '16:9', '9:16', '4:3', '21:9')." }), ), text: Type.Optional(Type.String({ description: "Testo da includere nell'immagine (tra virgolette)." })), negative: Type.Optional(Type.String({ description: "Cosa evitare, in framing positivo (es. 'nessun testo')." })), model: Type.Optional(Type.String({ description: "Modello agy." })), outputDir: Type.Optional(Type.String({ description: "Cartella dove copiare l'immagine generata." })), }), async execute(toolCallId, params, signal, onUpdate, ctx) { const p = params as any; const parts = [`Genera un'immagine: ${p.subject}`]; if (p.action) parts.push(`Azione: ${p.action}`); if (p.location) parts.push(`Luogo/contesto: ${p.location}`); if (p.composition) parts.push(`Composizione: ${p.composition}`); if (p.style) parts.push(`Stile: ${p.style}`); if (p.lighting) parts.push(`Illuminazione: ${p.lighting}`); if (p.aspectRatio) parts.push(`Formato/aspect ratio: ${p.aspectRatio}`); if (p.text) parts.push(`Includi il testo "${p.text}" nell'immagine.`); if (p.negative) parts.push(`Evita: ${p.negative}.`); const prompt = parts.join(". ") + IMG_SUFFIX; toolUpdate(onUpdate, "agy_generate: generazione immagine..."); const res = await executeAgy({ prompt, mode: "image", model: p.model, stateless: true, outputDir: p.outputDir, signal, }); return { content: [{ type: "text", text: res.text }], details: { imagePath: res.imagePath, exitCode: res.exitCode }, }; }, }); // ========================================================================= // TOOL: agy_edit — editing controllato (Keep + Change + Add + Render) // ========================================================================= pi.registerTool({ name: "agy_edit", label: "agy edit image", description: "Edit an existing image using Keep, Change, Add, and Render. " + "Use ONLY for controlled image edits; use agy_inpaint for changing one specific region.", parameters: Type.Object({ baseImage: Type.String({ description: "Percorso dell'immagine base da modificare." }), keep: Type.String({ description: "Cosa mantenere invariato (es. 'soggetto, posa, illuminazione, composizione')." }), change: Type.String({ description: "Cosa cambiare (es. 'il colore del divano in blu navy')." }), add: Type.Optional(Type.String({ description: "Cosa aggiungere (es. 'un vaso sul tavolo')." })), render: Type.Optional( Type.String({ description: "Target di resa (es. 'foto premium', 'stile editoriale', 'formato 4:5')." }), ), preserveAspectRatio: Type.Optional( Type.Boolean({ description: "true per non cambiare l'aspect ratio dell'input." }), ), model: Type.Optional(Type.String({ description: "Modello agy." })), outputDir: Type.Optional(Type.String({ description: "Cartella dove copiare l'immagine risultante." })), }), async execute(toolCallId, params, signal, onUpdate, ctx) { const p = params as any; const keep = p.preserveAspectRatio ? `${p.keep}. Non cambiare l'aspect ratio dell'immagine di input.` : p.keep; let prompt = `Usando l'immagine al percorso ${p.baseImage} come base, mantieni ${keep} invariato. Cambia ${p.change}.`; if (p.add) prompt += ` Aggiungi ${p.add}.`; if (p.render) prompt += ` Render come ${p.render}.`; prompt += IMG_SUFFIX; toolUpdate(onUpdate, "agy_edit: modifica immagine..."); const res = await executeAgy({ prompt, mode: "image", model: p.model, stateless: true, filePaths: [p.baseImage], outputDir: p.outputDir, yolo: true, signal, }); return { content: [{ type: "text", text: res.text }], details: { imagePath: res.imagePath, baseImage: p.baseImage, exitCode: res.exitCode }, }; }, }); // ========================================================================= // TOOL: agy_inpaint — editing di una zona specifica (semantic masking) // ========================================================================= pi.registerTool({ name: "agy_inpaint", label: "agy inpaint (edit zona specifica)", description: "Change ONLY one specified region or element of an image and preserve everything else. " + "Use for localized inpainting; use agy_edit for broader controlled edits.", parameters: Type.Object({ baseImage: Type.String({ description: "Percorso dell'immagine base." }), target: Type.String({ description: "L'elemento specifico da modificare (es. 'la maglietta del soggetto')." }), replacement: Type.String({ description: "La nuova descrizione dell'elemento (es. 'una maglietta rossa')." }), keepRest: Type.Optional( Type.String({ description: "Cosa mantenere identico (default: tutto il resto)." }), ), model: Type.Optional(Type.String({ description: "Modello agy." })), outputDir: Type.Optional(Type.String({ description: "Cartella dove copiare l'immagine risultante." })), }), async execute(toolCallId, params, signal, onUpdate, ctx) { const p = params as any; const keep = p.keepRest ?? "tutto il resto"; const prompt = `Usando l'immagine al percorso ${p.baseImage}, cambia SOLO ${p.target} in ${p.replacement}. ` + `Mantieni ${keep} esattamente identico, preservando stile, illuminazione e composizione originali.` + IMG_SUFFIX; toolUpdate(onUpdate, "agy_inpaint: modifica zona specifica..."); const res = await executeAgy({ prompt, mode: "image", model: p.model, stateless: true, filePaths: [p.baseImage], outputDir: p.outputDir, yolo: true, signal, }); return { content: [{ type: "text", text: res.text }], details: { imagePath: res.imagePath, baseImage: p.baseImage, exitCode: res.exitCode }, }; }, }); // ========================================================================= // TOOL: agy_style_transfer — applica uno stile preservando il contenuto // ========================================================================= pi.registerTool({ name: "agy_style_transfer", label: "agy style transfer", description: "Apply an artistic style to an existing image while preserving its content and composition. " + "Use ONLY for style transfer, not general image edits or analysis.", parameters: Type.Object({ baseImage: Type.String({ description: "Percorso dell'immagine base." }), style: Type.String({ description: "Lo stile da applicare (es. 'pittura Van Gogh', 'architectural drawing', 'film noir')." }), preserve: Type.Optional( Type.String({ description: "Cosa preservare (default: 'la composizione originale')." }), ), model: Type.Optional(Type.String({ description: "Modello agy." })), outputDir: Type.Optional(Type.String({ description: "Cartella dove copiare l'immagine risultante." })), }), async execute(toolCallId, params, signal, onUpdate, ctx) { const p = params as any; const preserve = p.preserve ?? "la composizione originale"; const prompt = `Usando l'immagine al percorso ${p.baseImage}, trasforma il contenuto nello stile di ${p.style}. ` + `Preserva ${preserve} ma renderizzala con lo stile richiesto.` + IMG_SUFFIX; toolUpdate(onUpdate, "agy_style_transfer: applica stile..."); const res = await executeAgy({ prompt, mode: "image", model: p.model, stateless: true, filePaths: [p.baseImage], outputDir: p.outputDir, yolo: true, signal, }); return { content: [{ type: "text", text: res.text }], details: { imagePath: res.imagePath, baseImage: p.baseImage, exitCode: res.exitCode }, }; }, }); // ========================================================================= // TOOL: agy_compose — combina più immagini // ========================================================================= pi.registerTool({ name: "agy_compose", label: "agy compose (combina immagini)", description: "Combine two or more existing images into one new composition. " + "Use ONLY for multi-image fusion; use agy_edit for editing one image.", parameters: Type.Object({ images: Type.Array(Type.String({ description: "Percorsi delle immagini da combinare." }), { description: "Lista di percorsi immagine (fino a ~6-14).", }), instruction: Type.String({ description: "Istruzione di fusione: cosa prendere da ciascuna immagine e come combinarle.", }), model: Type.Optional(Type.String({ description: "Modello agy." })), outputDir: Type.Optional(Type.String({ description: "Cartella dove copiare l'immagine risultante." })), }), async execute(toolCallId, params, signal, onUpdate, ctx) { const p = params as any; const refs = (p.images as string[]).map((img, i) => `Immagine ${i + 1}: ${img}`).join("\n"); const prompt = `Combina le seguenti immagini in una nuova composizione:\n${refs}\n\n` + `Istruzione: ${p.instruction}. Specifica il ruolo di ciascuna immagine.` + IMG_SUFFIX; toolUpdate(onUpdate, "agy_compose: combina immagini..."); const res = await executeAgy({ prompt, mode: "image", model: p.model, stateless: true, filePaths: p.images, outputDir: p.outputDir, yolo: true, signal, }); return { content: [{ type: "text", text: res.text }], details: { imagePath: res.imagePath, images: p.images, exitCode: res.exitCode }, }; }, }); // ========================================================================= // TOOL: agy_character — consistenza personaggio // ========================================================================= pi.registerTool({ name: "agy_character", label: "agy character consistency", description: "Generate or edit an image while preserving a character or object's identity from a reference image. " + "Use ONLY when identity consistency is required.", parameters: Type.Object({ referenceImage: Type.String({ description: "Percorso dell'immagine di riferimento del personaggio." }), name: Type.String({ description: "Nome/token del personaggio (es. 'Maya-giacca-blu')." }), features: Type.String({ description: "Caratteristiche immutabili da preservare (es. 'cicatrice sopracciglio sinistro')." }), task: Type.String({ description: "Cosa fare con il personaggio (es. 'mettilo in una scena notturna')." }), model: Type.Optional(Type.String({ description: "Modello agy." })), outputDir: Type.Optional(Type.String({ description: "Cartella dove copiare l'immagine risultante." })), }), async execute(toolCallId, params, signal, onUpdate, ctx) { const p = params as any; const prompt = `Usando l'immagine al percorso ${p.referenceImage} come riferimento del personaggio '${p.name}', ` + `${p.task}. Mantieni le caratteristiche del personaggio identiche: ${p.features}. ` + `Usa il token '${p.name}' per riferirti al personaggio.` + IMG_SUFFIX; toolUpdate(onUpdate, "agy_character: mantieni consistenza personaggio..."); const res = await executeAgy({ prompt, mode: "image", model: p.model, stateless: true, filePaths: [p.referenceImage], outputDir: p.outputDir, yolo: true, signal, }); return { content: [{ type: "text", text: res.text }], details: { imagePath: res.imagePath, referenceImage: p.referenceImage, exitCode: res.exitCode }, }; }, }); // ========================================================================= // TOOL: agy_analyze — fallback per analisi immagini quando Pi è text-only // ========================================================================= pi.registerTool({ name: "agy_analyze", label: "agy analyze image fallback", description: "Analyze a local image with Gemini ONLY as a fallback when the current Pi model cannot inspect images natively. " + "Use for image understanding or OCR. Do NOT use for audio (use agy_transcribe), video (use agy_video), " + "image generation/editing, or images the current model can already see.", parameters: Type.Object({ filePath: Type.String({ description: "Local image path only: PNG, JPG, JPEG, GIF, WebP, BMP, TIFF, or SVG. Never audio or video." }), question: Type.Optional(Type.String({ description: "Specific question about the image." })), model: Type.Optional(Type.String({ description: "Gemini model used for fallback analysis." })), yolo: Type.Optional(Type.Boolean({ description: "true to pass --dangerously-skip-permissions; use only when explicitly required." })), }), async execute(toolCallId, params, signal, onUpdate, ctx) { const p = params as any; const filePath = String(p.filePath ?? "").trim(); const ext = path.extname(filePath).toLowerCase(); if (!IMAGE_EXTENSIONS.has(ext)) { return { content: [{ type: "text", text: "agy_analyze accepts images only. Use agy_transcribe for audio or agy_video for video." }], details: { error: "unsupported_media_type", filePath }, isError: true, }; } if (ctx.model?.input?.includes("image")) { return { content: [{ type: "text", text: "The current Pi model supports native image input; agy_analyze is not needed." }], details: { error: "native_vision_available", model: ctx.model.id }, isError: true, }; } const prompt = p.question ? `Analyze the image at ${filePath}. ${p.question}` : `Analyze the image at ${filePath} and describe it in detail.`; toolUpdate(onUpdate, "agy_analyze: analisi immagine..."); const res = await executeAgy({ prompt, mode: "analyze", model: p.model, stateless: true, filePaths: [filePath], yolo: p.yolo, signal, }); return { content: [{ type: "text", text: res.text }], details: { filePath, exitCode: res.exitCode }, }; }, }); // ========================================================================= // TOOL: agy_create_verified — generazione iterativa con verifica // Loop self-contained: genera immagine → analizza con visione → verifica // OpenCV → rigenera se non rispetta i requisiti (fino a maxIterations). // ========================================================================= pi.registerTool({ name: "agy_create_verified", label: "agy create verified image", description: "Generate an image, optionally apply a deterministic Python image edit, inspect it, and regenerate it until the requirements pass or the iteration limit is reached. " + "Use ONLY when iterative visual verification or deterministic correction is required; use agy_generate for one-shot generation.", parameters: Type.Object({ requirements: Type.String({ description: "Requisiti precisi che l'immagine deve soddisfare." }), outputDir: Type.Optional(Type.String({ description: "Cartella dove salvare l'immagine finale." })), maxIterations: Type.Optional(Type.Number({ description: "Maximum iterations (default 3, max 5)." })), useOpenCV: Type.Optional(Type.Boolean({ description: "Run objective OpenCV checks (default true)." })), editInstructions: Type.Optional(Type.String({ description: "Optional deterministic edit to apply each iteration before verification; use precise, measurable instructions." })), editScript: Type.Optional(Type.String({ description: "Optional Python script for a deterministic edit. Must read argv[1], write argv[2], and use only the allowlisted image libraries." })), model: Type.Optional(Type.String({ description: "Antigravity model." })), }), async execute(toolCallId, params, signal, onUpdate, ctx) { const p = params as any; const requirements = String(p.requirements ?? "").trim(); const maxIter = Math.min(Math.max(Number(p.maxIterations ?? 3) || 3, 1), 5); const useCV = p.useOpenCV ?? true; const editInstructions = String(p.editInstructions ?? "").trim(); const suppliedEditScript = String(p.editScript ?? "").trim(); if (editInstructions && suppliedEditScript) { return { content: [{ type: "text", text: "Provide either editInstructions or editScript, not both." }], details: { error: "conflicting_edit_inputs" }, isError: true, }; } toolUpdate(onUpdate, "agy_create_verified: avvio loop di creazione verificata..."); let currentImage: string | undefined; let lastIssues = ""; const report: { iteration: number; pass: boolean; imagePath: string; verdict: string; opencv: string; programmaticEdit: string; }[] = []; let finalText = ""; // Genera DINAMICAMENTE lo script OpenCV specifico per i requisiti (una volta, // i requisiti sono fissi) e lo riusa in ogni iterazione del loop. let cvScript = ""; if (useCV) { toolUpdate(onUpdate, "agy_create_verified: generazione script OpenCV dinamico..."); cvScript = await generateOpenCVScript(requirements); } let editScript = suppliedEditScript; if (editInstructions) { toolUpdate(onUpdate, "agy_create_verified: generazione edit deterministico..."); editScript = await generateProgrammaticEditScript(requirements, editInstructions); } const editValidationError = editScript ? validateProgrammaticEditScript(editScript) : null; if (editValidationError) { return { content: [{ type: "text", text: `Edit programmatico rifiutato: ${editValidationError}` }], details: { error: "invalid_programmatic_edit", validation: editValidationError }, isError: true, }; } for (let iter = 1; iter <= maxIter; iter++) { toolUpdate( onUpdate, `agy_create_verified: iterazione ${iter}/${maxIter} (${iter === 1 ? "generazione" : "rigenerazione"})...`, ); // 1) Genera (o rigenera) con prompt di correzione se non è la prima const genPrompt = currentImage ? `L'immagine precedente (${currentImage}) non rispetta i requisiti per questi motivi: ${lastIssues}. ` + `Rigenera/rettifica l'immagine per soddisfare esattamente: ${requirements}.` : `Genera un'immagine che soddisfi esattamente questi requisiti: ${requirements}.`; const genRes = await executeAgy({ prompt: genPrompt + IMG_SUFFIX, mode: "image", model: p.model, stateless: true, filePaths: currentImage ? [currentImage] : [], outputDir: p.outputDir, signal, }); const imgPath = genRes.imagePath; if (!imgPath) { finalText = `Generazione fallita all'iterazione ${iter}: nessuna immagine prodotta.\n${genRes.text}`; break; } currentImage = imgPath; let programmaticEdit = ""; let editPassed = true; if (editScript) { const editRoot = p.outputDir ? path.resolve(String(p.outputDir)) : os.tmpdir(); try { fs.mkdirSync(editRoot, { recursive: true }); } catch { /* il runner segnalerà l'errore */ } const editedPath = path.join(editRoot, `agy-verified-edit-${process.pid}-${iter}-${Math.random().toString(36).slice(2)}.png`); toolUpdate(onUpdate, `agy_create_verified: edit programmatico ${iter}/${maxIter}...`); const editResult = await runProgrammaticEditScript(editScript, imgPath, editedPath); programmaticEdit = editResult.report; editPassed = editResult.ok; if (editPassed) currentImage = editedPath; else lastIssues = `Edit programmatico fallito: ${programmaticEdit}`; } const candidateImage = currentImage!; // 2) Analisi visione (giudice): PASS/FAIL + problemi rispetto ai requisiti const judgePrompt = `Analizza il file al percorso ${candidateImage}. Requisiti richiesti: ${requirements}. ` + `Verifica se l'immagine li rispetta. Rispondi iniziando con \"PASS:\" o \"FAIL:\", ` + `poi elenca sinteticamente (max 3 punti) le deviazioni rispetto ai requisiti in caso di FAIL.`; const judgeRes = await executeAgy({ prompt: judgePrompt, mode: "analyze", model: p.model, stateless: true, filePaths: [candidateImage], yolo: true, signal, }); const verdict = judgeRes.text.trim(); const visionPass = /^\s*PASS:?/i.test(verdict); lastIssues = verdict; // 3) Verifica OpenCV DINAMICA (script generato dall'LLM per i requisiti) let cvMetrics = ""; let ocvPass: boolean | null = null; if (useCV && cvScript) { const out = await runOpenCVScript(cvScript, candidateImage); cvMetrics = out; try { const parsed = JSON.parse(out); ocvPass = typeof parsed.pass === "boolean" ? parsed.pass : null; if (typeof parsed.score === "number") cvMetrics += `\nscore: ${parsed.score}`; if (Array.isArray(parsed.findings) && parsed.findings.length) cvMetrics += `\nfindings: ${parsed.findings.join("; ")}`; } catch { ocvPass = null; } // Se OpenCV rileva non conformità, le aggiunge alle issue per guidare la rigenerazione if (ocvPass === false) { lastIssues = `${verdict}\nVerifica OpenCV: ${cvMetrics}`; } } // Conforme solo se visione PASS e (se disponibile) anche OpenCV PASS const pass = editPassed && visionPass && (ocvPass === null || ocvPass === true); report.push({ iteration: iter, pass, imagePath: candidateImage, verdict, opencv: cvMetrics, programmaticEdit }); if (pass) { finalText = `✅ Immagine verificata dopo ${iter} iterazione/i.\nPercorso: ${candidateImage}` + (cvMetrics ? `\nMetriche OpenCV: ${cvMetrics}` : "") + `\nVerdetto visione:\n${verdict}`; break; } if (iter === maxIter) { finalText = `⚠️ Requisiti non soddisfatti dopo ${maxIter} iterazioni.\nPercorso: ${candidateImage}` + (cvMetrics ? `\nMetriche OpenCV: ${cvMetrics}` : "") + `\nUltimo verdetto visione:\n${verdict}`; } } return { content: [{ type: "text", text: finalText || "Nessun output." }], details: { iterations: report.length, imagePath: currentImage, report, }, }; }, }); // ========================================================================= // TOOL: agy_transcribe — trascrizione audio (via Gemini API diretta) // ========================================================================= pi.registerTool({ name: "agy_transcribe", label: "agy transcribe audio", description: "Transcribe speech or other audio into text with the direct Gemini API. " + "Use ONLY for audio; use agy_analyze for images and agy_video for video.", parameters: Type.Object({ filePath: Type.String({ description: "Percorso del file audio (wav, mp3, m4a, ecc.)." }), language: Type.Optional(Type.String({ description: "Lingua del contenuto (es. 'italiano', 'english')." })), model: Type.Optional(Type.String({ description: "Modello Gemini (default: gemini-3.5-flash)." })), }), async execute(toolCallId, params, signal, onUpdate, ctx) { const p = params as any; toolUpdate(onUpdate, "agy_transcribe: trascrizione audio..."); const tr = await transcribeAudio(p.filePath); if (!tr.text) { return { content: [{ type: "text", text: `Trascrizione fallita: ${tr.error ?? "vuota"}` }], details: { filePath: p.filePath, error: tr.error }, isError: true, }; } const transcript = tr.text; return { content: [{ type: "text", text: transcript }], details: { filePath: p.filePath, model: p.model ?? "gemini-3.5-flash" }, }; }, }); // ========================================================================= // TOOL: agy_video — analisi video // ========================================================================= pi.registerTool({ name: "agy_video", label: "agy analyze video", description: "Analyze a video: scenes, content, codec, resolution, and audio tracks. " + "Use ONLY for video; use agy_analyze for images and agy_transcribe for audio-only files.", parameters: Type.Object({ filePath: Type.String({ description: "Percorso del file video (mp4, mov, ecc.)." }), question: Type.Optional(Type.String({ description: "Domanda specifica sul video." })), model: Type.Optional(Type.String({ description: "Modello agy." })), }), async execute(toolCallId, params, signal, onUpdate, ctx) { const p = params as any; const prompt = p.question ? `Analizza il video al percorso ${p.filePath}. ${p.question}` : `Analizza il video al percorso ${p.filePath}: descrivi cosa mostra, codec, risoluzione e se contiene audio.`; toolUpdate(onUpdate, "agy_video: analisi video..."); const res = await executeAgy({ prompt, mode: "analyze", model: p.model, stateless: true, filePaths: [p.filePath], yolo: true, signal, }); return { content: [{ type: "text", text: res.text }], details: { filePath: p.filePath, exitCode: res.exitCode }, }; }, }); // ========================================================================= // TOOL: agy_tts — text-to-speech via Gemini API // ========================================================================= pi.registerTool({ name: "agy_tts", label: "agy TTS (text to speech)", description: "Convert text to an audio file with Gemini TTS. Use ONLY when an audio file or playback is explicitly requested; " + "for normal voice output use the canonical tts_speak tool.", parameters: Type.Object({ text: Type.String({ description: "Il testo da pronunciare." }), play: Type.Optional(Type.Boolean({ description: "true per riprodurre l'audio (default: true)." })), outputDir: Type.Optional(Type.String({ description: "Cartella dove salvare il file audio." })), }), async execute(toolCallId, params, signal, onUpdate, ctx) { const p = params as any; toolUpdate(onUpdate, "agy_tts: sintesi vocale..."); return ttsSpeak(p.text, p.outputDir).then((file) => { if (!file) { return { content: [{ type: "text", text: "TTS fallito: key Gemini mancante o errore API." }], details: {}, isError: true, }; } return { content: [{ type: "text", text: `Audio TTS generato: ${file}` }], details: { audioFile: file }, }; }); }, }); // ========================================================================= // TOOL: agy_models — elenca i modelli disponibili // ========================================================================= pi.registerTool({ name: "agy_models", label: "agy list models", description: "Elenca i modelli disponibili per agy (Gemini, Claude, ecc.).", parameters: Type.Object({}), async execute(toolCallId, params, signal) { const r = await runAgy(["models"], 30_000, signal); return { content: [{ type: "text", text: r.output.trim() || r.error || "(nessun output)" }], details: { exitCode: r.exitCode }, }; }, }); // ========================================================================= // TOOL: agy_conversation — gestione stato conversazione // ========================================================================= pi.registerTool({ name: "agy_conversation", label: "agy conversation state", description: "Administrative control for agy conversation state: show the current ID, list stored conversations, or reset state. " + "Use only when the user explicitly requests conversation-state management.", parameters: Type.Object({ action: Type.Union( [Type.Literal("id"), Type.Literal("list"), Type.Literal("reset")], { description: "id | list | reset" }, ), }), async execute(toolCallId, params) { const action = (params as { action: string }).action; if (action === "id") { return { content: [{ type: "text", text: readState() ?? "(nessuna conversazione attiva)" }], details: {} }; } if (action === "reset") { resetState(); return { content: [{ type: "text", text: "Stato conversazione azzerato." }], details: {} }; } try { if (!fs.existsSync(CONV_DIR)) { return { content: [{ type: "text", text: "(nessuna conversazione trovata)" }], details: {} }; } const files = fs .readdirSync(CONV_DIR) .filter((f) => f.endsWith(".db")) .map((f) => path.join(CONV_DIR, f)) .sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs) .slice(0, 15); const lines = files.map((f) => { const id = path.basename(f, ".db"); const mtime = new Date(fs.statSync(f).mtimeMs).toISOString().replace("T", " ").slice(0, 19); return `${id}\t${mtime}`; }); return { content: [{ type: "text", text: lines.join("\n") || "(nessuna conversazione trovata)" }], details: {}, }; } catch (e: any) { return { content: [{ type: "text", text: `Errore: ${e.message}` }], details: {}, isError: true }; } }, }); // ========================================================================= // TOOL: antigravity_chat — client diretto protocollo Antigravity (account) // ========================================================================= pi.registerTool({ name: "antigravity_chat", label: "antigravity direct protocol chat", description: "Send one text or multimodal request directly to Antigravity's cloudcode-pa gateway, without the agy subprocess. " + "Use ONLY when direct gateway access is explicitly needed; for normal delegation use agy. " + "Respect per-model quota and send requests serially; do not use for batch automation.", parameters: Type.Object({ prompt: Type.String({ description: "Il messaggio da inviare al modello" }), model: Type.Optional( Type.String({ description: "Modello (default gemini-3.5-flash): gemini-3.5-flash | gemini-3.6-flash-medium | gemini-3.6-flash-high | gemini-3.1-pro | gemini-3.1-pro-high | claude-sonnet-4.6 | claude-opus-4.6 | gpt-oss-120b", }), ), system: Type.Optional(Type.String({ description: "System instruction opzionale" })), maxOutputTokens: Type.Optional(Type.Number({ description: "Token massimi di output (default 8192)" })), temperature: Type.Optional(Type.Number({ description: "Temperatura 0-2 (default 0.4)" })), thinking: Type.Optional( Type.String({ description: "Livello thinking per Gemini: low | medium | high | off (default: auto in base al modello)", }), ), stream: Type.Optional(Type.Boolean({ description: "Streaming SSE (default true)" })), }), async execute(toolCallId, params) { const p = params as any; try { const res = await antgGenerate({ prompt: p.prompt, model: p.model, system: p.system, maxOutputTokens: p.maxOutputTokens, temperature: p.temperature, thinking: p.thinking, stream: p.stream !== false, }); return { content: [{ type: "text", text: res.text || "(nessun testo nella risposta)" }], details: { ...res.details, model: p.model ?? ANTG_DEFAULT_MODEL, project: antgProject?.pid ?? null }, }; } catch (e: any) { return { content: [{ type: "text", text: `Errore Antigravity: ${e.message}` }], details: {}, isError: true, }; } }, }); // ========================================================================= // PROVIDER: antigravity — modelli del gateway nel selettore modelli // ========================================================================= pi.registerProvider("antigravity", antgProviderConfig(antgRegisteredModels)); // ========================================================================= // Comando: /agy:refresh-models — aggiorna l'elenco modelli del provider dal // catalogo vivo dell'account (fetchAvailableModels) e ri-registra il provider. // ========================================================================= pi.registerCommand("agy:refresh-models", { description: "Aggiorna l'elenco modelli del provider Antigravity dal catalogo vivo dell'account (fetchAvailableModels) e ri-registra il provider (selettore modelli).", handler: async (_args, ctx) => { ctx.ui.setStatus("agy:refresh-models", "Aggiornamento modelli Antigravity..."); try { const { pid, base } = await antgGetProject(); const token = await antgGetAccessToken(); const models = await antgFetchCatalog(token, base); if (!models.length) throw new Error("Il catalogo non ha restituito modelli utilizzabili"); antgRegisteredModels = models; pi.registerProvider("antigravity", antgProviderConfig(models)); ctx.ui.setStatus("agy:refresh-models", ""); const lines = models.map((m) => ` ${m.id} — ${m.name}${m.reasoning ? " (thinking)" : ""}`).join("\n"); ctx.ui.notify( `✅ Provider antigravity aggiornato: ${models.length} modelli (project ${pid})\n${lines}`, "info", ); } catch (e: any) { ctx.ui.setStatus("agy:refresh-models", ""); ctx.ui.notify(`Errore aggiornamento modelli: ${e.message}`, "error"); } }, }); // ========================================================================= // Comandi interattivi // ========================================================================= pi.registerCommand("agy", { description: "Invia un prompt a agy (subagent Antigravity). Uso: /agy ", handler: async (args, ctx) => { if (!args?.trim()) { ctx.ui.notify("Uso: /agy ", "error"); return; } ctx.ui.setStatus("agy", "agy: elaborazione..."); const res = await executeAgy({ prompt: args.trim() }); ctx.ui.setStatus("agy", ""); ctx.ui.notify(res.text || "(nessun output)", "info"); }, }); pi.registerCommand("agy:new", { description: "Forza una nuova conversazione agy", handler: async (_args, ctx) => { resetState(); ctx.ui.notify("Nuova conversazione agy pronta.", "info"); }, }); pi.registerCommand("agy:list", { description: "Elenca le conversazioni agy", handler: async (_args, ctx) => { const id = readState(); ctx.ui.notify(`Conversazione corrente: ${id ?? "(nessuna)"}`, "info"); }, }); pi.registerCommand("agy:reset", { description: "Azzera lo stato conversazione agy", handler: async (_args, ctx) => { resetState(); ctx.ui.notify("Stato conversazione azzerato.", "info"); }, }); // ========================================================================= // Registrazione microfono (F12) + trascrizione via Gemini + prompt su pi // ========================================================================= async function handleRecordToggle(ctx: any) { if (!recording) { startRecording(ctx); playSound("start"); ctx.ui.notify("🎙️ Registrazione avviata (F12 per fermare, Ctrl+Esc per annullare)", "info"); return; } playSound("stop"); ctx.ui.setStatus("agy-rec", "⏹️ Finalizzazione..."); const file = await stopRecording(); if (!file) { ctx.ui.setStatus("agy-rec", ""); ctx.ui.notify("Nessuna registrazione attiva", "warning"); return; } ctx.ui.notify("Registrazione fermata, ottimizzazione audio...", "info"); const optimized = await optimizeAudio(file); const editorText = (ctx.ui.getEditorText?.() ?? "").trim(); const context = getConversationContext(ctx); let transcript = ""; let finalText = ""; let needsSearch = false; let searchHints: string[] = []; const activeModel = ctx.model; const directGeminiConfig = getConfig("sttDirectGemini"); // Usa la pipeline multimodale diretta con Gemini se abilitata (default: true) o se il modello di chat è Gemini const useDirectMultimodal = directGeminiConfig !== "false" ? true : isGeminiModel(activeModel); if (useDirectMultimodal) { // ========================================================================= // Pipeline Multimodale Diretta (Gemini): l'audio viene inviato direttamente // a Gemini (gateway Antigravity / Gemini API) senza passare per un STT separato, // anche se il modello attivo in sessione è DeepSeek, Claude o un modello locale. // ========================================================================= ctx.ui.notify("Interpretazione vocale diretta con Gemini...", "info"); try { const voiceModelName = getConfig("voiceModel") || (isGeminiModel(activeModel) ? activeModel?.id : "gemini-3.7-flash-medium"); const directRes = await interpretAudioDirectGemini( optimized, editorText, context, voiceModelName, ); transcript = directRes.transcript || directRes.cleanPrompt; finalText = directRes.cleanPrompt; needsSearch = directRes.needsSearch; searchHints = directRes.searchHints; } catch (err: any) { ctx.ui.notify(`Elaborazione diretta fallita: ${err.message}. Fallback su trascrizione STT...`, "warning"); } } // Fallback o modello non-Gemini: pipeline a 2 passaggi (STT + Briefing) if (!finalText) { ctx.ui.notify("Trascrizione in corso...", "info"); const tr = await transcribeAudio(optimized); if (!tr.text) { ctx.ui.setStatus("agy-rec", ""); ctx.ui.notify(`Trascrizione fallita: ${tr.error ?? "vuota"}`, "error"); playSound("cancel"); return; } transcript = tr.text; ctx.ui.notify("Interpretazione con Gemini...", "info"); const res = await executeAgy({ prompt: `[CONTESTO INTERNO — COMUNICAZIONE TRA AGENTI]\n` + `Sei un analista tecnico/middleware per un agente AI orchestratore. NON rispondere all'utente: il tuo output sarà letto SOLO dall'orchestratore, che poi risponderà all'utente.` + `\n\nAnalizza la richiesta vocale (e l'eventuale testo dell'editor) e produci un briefing strutturato per l'orchestratore.` + `\n\nTrascrizione vocale:\n${transcript}` + (editorText ? `\n\nTesto scritto dall'utente nel campo di input (da combinare con la voce):\n${editorText}` : "") + `\n\nContesto della conversazione:\n${context || "(nessuno)"}` + `\n\nRestituisci UN SOLO oggetto JSON (nessun testo aggiuntivo) con questi campi:` + `\n{"trascrizione_corretta":"...","intent_analisi":"...","note_per_agent":"...","azioni_raccomandate":["..."],"prompt_utente_pulito":"...","ricerca_necessaria":true/false,"suggerimenti_ricerca":["..."]}` + `\nRegole:` + `\n- NON eseguire alcuno strumento o azione; solo analisi e briefing.` + `\n- Nessun saluto o testo rivolto all'utente: solo JSON tecnico per l'orchestratore.` + `\n- prompt_utente_pulito = la richiesta rielaborata che l'orchestratore userà come prompt verso l'utente.` + `\n- ricerca_necessaria=true se serve verificare best practices, versioni, documentazione o dati aggiornati.` + `\n- suggerimenti_ricerca: se ricerca_necessaria, indica all'orchestratore di usare la ricerca web (Perplexity) e su cosa.`, stateless: true, model: "Gemini 3.6 Flash (Medium)", yolo: true, }); const briefing = extractVoiceBriefing(res.text); finalText = briefing.cleanPrompt || res.text.trim() || transcript; needsSearch = briefing.needsSearch; searchHints = briefing.searchHints; } ctx.ui.setStatus("agy-rec", ""); // Guida per l'agente successivo: usa la ricerca web/Perplexity se necessario if (needsSearch) { const hints = searchHints.length ? ` Suggerimenti: ${searchHints.join("; ")}` : ""; finalText += `\n\n[Nota per l'agente: per rispondere correttamente, usa la ricerca web (Perplexity) per verificare best practices/versioni/documentazione aggiornate.${hints}]`; } // Il testo dell'editor è stato consumato: lo svuota per evitare reinvii duplicati. if (editorText) { try { ctx.ui.setEditorText?.(""); } catch { /* ignora */ } } // Opzione 4: piano + conferma in overlay TUI prima di eseguire (se abilitata) const planningMode = getConfig("vocalPlanningMode") ?? "true"; if (planningMode !== "false") { const decision = await showVoicePlanOverlay(ctx, transcript, finalText); if (decision.action === "cancel") { ctx.ui.notify("Vocale annullato — nessuna azione eseguita.", "info"); playSound("cancel"); return; } if (decision.action === "record") { ctx.ui.notify("Registra di nuovo con F12.", "info"); return; } if (decision.action === "literal") { // Non invia il piano proposto: inserisce la dettatura letterale // nell'area del prompt (ripristinando l'eventuale testo editor consumato). const literal = editorText ? `${editorText}\n\n${transcript}` : transcript; try { ctx.ui.pasteToEditor?.(literal); } catch { /* ignora */ } ctx.ui.notify("Trascrizione letterale inserita nel prompt (non inviata)", "info"); playSound("done"); return; } finalText = decision.text; } // Inserisci il risultato come prompt su pi if (ctx.isIdle()) { pi.sendUserMessage(finalText); } else { pi.sendUserMessage(finalText, { deliverAs: "followUp" }); } ctx.ui.notify("✅ Richiesta vocale inviata come prompt a pi", "info"); playSound("done"); // notifica vocale (config ttsNotify o AGY_TTS_NOTIFY=0 per disattivare) const ttsNotify = getConfig("ttsNotify") ?? "true"; if (ttsNotify !== "false" && process.env.AGY_TTS_NOTIFY !== "0") { ttsSpeak("Trascrizione completata e inviata.").catch(() => {}); } } pi.registerShortcut("f12", { description: "Avvia/ferma registrazione microfono (max 2 min) e trascrive via Gemini", handler: async (ctx) => { await handleRecordToggle(ctx); }, }); pi.registerShortcut("ctrl+escape", { description: "Annulla la registrazione microfono in corso", handler: async (ctx) => { if (recording) { await cancelRecording(); ctx.ui.setStatus("agy-rec", ""); ctx.ui.notify("❌ Registrazione annullata", "info"); playSound("cancel"); } }, }); pi.registerCommand("agy:record", { description: "Avvia/ferma registrazione microfono (come F12)", handler: async (_args, ctx) => { await handleRecordToggle(ctx); }, }); pi.registerCommand("agy:record:stop", { description: "Ferma la registrazione microfono in corso", handler: async (_args, ctx) => { if (!recording) { ctx.ui.notify("Nessuna registrazione in corso", "warning"); return; } await handleRecordToggle(ctx); }, }); pi.registerCommand("agy:record:cancel", { description: "Annulla la registrazione microfono in corso", handler: async (_args, ctx) => { if (!recording) { ctx.ui.notify("Nessuna registrazione in corso da annullare", "warning"); return; } await cancelRecording(); ctx.ui.setStatus("agy-rec", ""); ctx.ui.notify("❌ Registrazione annullata", "info"); playSound("cancel"); }, }); pi.registerCommand("agy:speak", { description: "Pronuncia un testo con TTS Gemini. Uso: /agy:speak ", handler: async (args, ctx) => { if (!args?.trim()) { ctx.ui.notify("Uso: /agy:speak ", "error"); return; } ctx.ui.setStatus("agy-tts", "🔊 sintesi vocale..."); const file = await ttsSpeak(args.trim()); ctx.ui.setStatus("agy-tts", ""); if (file) ctx.ui.notify(`🔊 Audio: ${file}`, "info"); else ctx.ui.notify("TTS fallito (key Gemini mancante?)", "error"); }, }); pi.registerCommand("agy:vocal", { description: "Attiva/disattiva il feedback vocale TTS. Uso: /agy:vocal [on|off|status]", handler: async (args, ctx) => { const arg = (args ?? "").trim().toLowerCase(); const current = getConfig("ttsNotify") ?? "true"; if (arg === "on") { setConfig("ttsNotify", "true"); ctx.ui.notify("🔊 Feedback vocale ATTIVATO", "info"); ttsSpeak("Feedback vocale attivato.").catch(() => {}); return; } if (arg === "off") { setConfig("ttsNotify", "false"); ctx.ui.notify("🔇 Feedback vocale DISATTIVATO", "info"); return; } if (arg === "status" || !arg) { const on = current !== "false"; ctx.ui.notify(on ? "🔊 Feedback vocale: ATTIVO" : "🔇 Feedback vocale: DISATTIVO", "info"); return; } // toggle if (current === "false") { setConfig("ttsNotify", "true"); ctx.ui.notify("🔊 Feedback vocale ATTIVATO", "info"); ttsSpeak("Feedback vocale attivato.").catch(() => {}); } else { setConfig("ttsNotify", "false"); ctx.ui.notify("🔇 Feedback vocale DISATTIVATO", "info"); } }, }); // ========================================================================= // Comando: /agy:config — gestione configurazione persistente // ========================================================================= const CONFIG_KEYS: { key: keyof AgyConfig; desc: string }[] = [ { key: "geminiApiKey", desc: "Chiave API Google Gemini (STT/TTS)" }, { key: "enne2ApiKey", desc: "Token per il server proxy ai.enne2.net (opzionale)" }, { key: "sttBackend", desc: "Backend trascrizione: gemini | enne2" }, { key: "sttUrl", desc: "URL base backend enne2" }, { key: "sttModel", desc: "Modello STT backend enne2" }, { key: "sttMaxDuration", desc: "Durata max registrazione (secondi)" }, { key: "ttsBackend", desc: "Backend TTS: gemini | enne2" }, { key: "ttsNotify", desc: "Notifiche vocali automatiche: true | false" }, { key: "ttsModel", desc: "Modello TTS Gemini" }, { key: "agyBin", desc: "Path del binario agy" }, { key: "agyDefaultModel", desc: "Modello predefinito per le chiamate agy" }, { key: "agyTimeoutMs", desc: "Timeout esecuzione agy (ms)" }, { key: "contextInject", desc: "Iniezione contesto nel prompt agy: true | false" }, { key: "contextTokens", desc: "Token cap per il contesto iniettato (default 1500)" }, { key: "webSearch", desc: "Ricerca web Strada A: auto | on | off" }, { key: "vocalPlanningMode", desc: "Opzione 4: piano+conferma dopo il vocale (true|false)" }, { key: "sttDirectGemini", desc: "Interpretazione vocale diretta Gemini (true|false, default true)" }, { key: "voiceModel", desc: "Modello Gemini per l'audio (default gemini-3.7-flash-medium)" }, ]; pi.registerCommand("agy:config", { description: "Gestisce la configurazione dell'estensione. Uso: /agy:config [get|set|reset] [chiave] [valore]", handler: async (args, ctx) => { const parts = (args ?? "").trim().split(/\s+/); const action = parts[0] ?? ""; const key = parts[1] as keyof AgyConfig | undefined; const value = parts.slice(2).join(" "); // /agy:config — elenca tutto if (!action) { const cfg = loadConfig(); const lines = CONFIG_KEYS.map(({ key: k, desc }) => { const v = (cfg as any)[k]; const masked = k === "geminiApiKey" || k === "enne2ApiKey" ? v ? `${String(v).slice(0, 4)}...${String(v).slice(-4)}` : "(non impostata)" : v ?? "(non impostata)"; return `${k} = ${masked} — ${desc}`; }); ctx.ui.notify(`Config agy-pi (${CONFIG_FILE}):\n${lines.join("\n")}`, "info"); return; } // /agy:config get if (action === "get" && key) { const v = getConfig(key); ctx.ui.notify(`${key} = ${v ?? "(non impostata)"}`, "info"); return; } // /agy:config set if (action === "set" && key && value) { if (!CONFIG_KEYS.some((c) => c.key === key)) { ctx.ui.notify(`Chiave sconosciuta: ${key}`, "error"); return; } setConfig(key, value); ctx.ui.notify(`✅ ${key} impostato. File: ${CONFIG_FILE}`, "info"); return; } // /agy:config reset if (action === "reset") { resetConfig(); ctx.ui.notify("Configurazione azzerata (default ripristinati).", "info"); return; } ctx.ui.notify( "Uso: /agy:config | /agy:config get | /agy:config set | /agy:config reset", "warning", ); }, }); // ========================================================================= // Comando: /agy:key — dialog overlay TUI per inserire/modificare la chiave // API Gemini in modo interattivo (campo mascherato, Enter conferma, Esc // annulla). Usa i componenti TUI di pi (overlay in sovrimpressione). // ========================================================================= pi.registerCommand("agy:key", { description: "Apre un dialog overlay per inserire/modificare la chiave API Gemini (campo mascherato)", handler: async (_args, ctx) => { // Import dinamico: se pi-tui non fosse disponibile, fallisce solo questo // comando senza rompere il caricamento dell'intera estensione. const { Container, Text, matchesKey, Key, } = await import("@earendil-works/pi-tui"); const { DynamicBorder } = await import("@earendil-works/pi-coding-agent"); const result = await ctx.ui.custom( (tui, theme, _keybindings, done) => { let value = ""; const currentKey = getConfig("geminiApiKey") ?? ""; const maskCurrent = currentKey ? `${currentKey.slice(0, 4)}...${currentKey.slice(-4)}` : "(non impostata)"; const mask = (v: string) => "•".repeat(v.length); const container = new Container(); const renderDialog = () => { container.clear(); container.addChild(new DynamicBorder((s: string) => theme.fg("accent", s))); container.addChild( new Text(theme.fg("accent", theme.bold("🔑 Chiave API Gemini")), 1, 1), ); container.addChild( new Text(theme.fg("dim", `Attuale: ${maskCurrent}`), 1, 0), ); container.addChild(new Text("", 0, 0)); const field = value ? mask(value) : "(vuota)"; container.addChild( new Text( theme.fg("text", "Nuova chiave: ") + theme.fg("warning", field), 1, 0, (s) => theme.bg("toolPendingBg", s), ), ); container.addChild(new Text("", 0, 0)); container.addChild( new Text( theme.fg("dim", "Digita la chiave • Enter conferma • Esc annulla"), 1, 0, ), ); container.addChild(new DynamicBorder((s: string) => theme.fg("accent", s))); }; renderDialog(); return { render: (w) => { renderDialog(); return container.render(w); }, invalidate: () => container.invalidate(), handleInput: (data) => { if (matchesKey(data, Key.enter)) { const trimmed = value.trim(); if (trimmed) { setConfig("geminiApiKey", trimmed); // Aggiorna anche ~/.agy-chat/gemini-key per coerenza con gli script TTS/STT try { fs.mkdirSync(AGY_CHAT_DIR, { recursive: true }); fs.writeFileSync( path.join(AGY_CHAT_DIR, "gemini-key"), trimmed, { mode: 0o600 }, ); } catch { /* ignora */ } done(trimmed); } return; } if (matchesKey(data, Key.escape)) { done(null); return; } if (matchesKey(data, Key.backspace)) { value = value.slice(0, -1); tui.requestRender(); return; } // Caratteri stampabili permessi per una chiave API (A-Z a-z 0-9 _ . -) if (data.length === 1 && data.charCodeAt(0) >= 32) { if (/^[A-Za-z0-9._-]+$/.test(data)) { value += data; } tui.requestRender(); } }, }; }, { overlay: true, overlayOptions: { width: "50%", minWidth: 54, anchor: "center", }, }, ); if (result) { ctx.ui.notify( `✅ Chiave API Gemini aggiornata: ${result.slice(0, 4)}...${result.slice(-4)}`, "info", ); try { playSound("done"); } catch { /* ignora */ } } else { ctx.ui.notify("Chiave non modificata.", "info"); } }, }); // ========================================================================= // Comando: /agy:status — diagnostica estensione in un overlay TUI // Mostra: binario agy, versione, chiave Gemini, modello attivo, ultimo // errore dal log, stato config. Chiuso con Enter o Esc. // ========================================================================= pi.registerCommand("agy:status", { description: "Mostra lo stato dell'estensione (agi, chiave, modello, ultimo errore)", handler: async (_args, ctx) => { const { Container, Text, matchesKey, Key } = await import("@earendil-works/pi-tui"); const { DynamicBorder } = await import("@earendil-works/pi-coding-agent"); // ---- raccolta diagnostica ---- const agyBin = findAgy(); const agyExists = agyBin === "agy" ? fs.existsSync("/home/enne2/.local/bin/agy") : fs.existsSync(agyBin); let agyVersion = "(sconosciuta)"; if (agyExists) { try { agyVersion = ( await execFileAsync(agyBin, ["--version"], { timeout: 8000, env: { ...process.env, NO_COLOR: "1" }, }) ).stdout.trim() || "(sconosciuta)"; } catch { agyVersion = "(errore esecuzione)"; } } const cfgKey = getConfig("geminiApiKey") ?? ""; const fileKey = (() => { try { const f = path.join(AGY_CHAT_DIR, "gemini-key"); return fs.existsSync(f) ? fs.readFileSync(f, "utf8").trim() : ""; } catch { return ""; } })(); const key = cfgKey || fileKey; const keyStatus = key ? "✅ valida" : "❌ mancante (usa /agy:key)"; const keyMask = key ? `${key.slice(0, 4)}...${key.slice(-4)}` : "—"; const model = getConfig("agyDefaultModel") || "(default agy)"; const sttBackend = getConfig("sttBackend") ?? "gemini"; const ttsNotify = getConfig("ttsNotify") ?? "true"; // ultimo errore dal log agy (ultima riga con 'ERROR'/'denied'/'quota') let lastError = "(nessuno)"; try { const logDir = path.join(os.homedir(), ".gemini", "antigravity-cli", "log"); if (fs.existsSync(logDir)) { const logs = fs .readdirSync(logDir) .map((f) => path.join(logDir, f)) .sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs); const logFile = logs[0]; if (logFile) { const lines = fs.readFileSync(logFile, "utf8").split("\n"); const meaningful = [...lines] .reverse() .filter((l) => /error|denied|quota|failed|timed out/i.test(l)) .filter((l) => !/^ERROR: logging before google\.Init:/i.test(l.trim())); if (meaningful.length) { const raw = meaningful[0]; // estrai il messaggio dopo il pattern glog: "go:file.go:NN] msg" oppure dopo ":" const m = raw.match(/\][\s]*\s*\s*(.*)$/) || raw.match(/:\s*(.*)$/i); lastError = (m ? m[1] : raw).replace(/^ERROR:|^WARN:/i, "").trim().slice(0, 90) || "(vedi log)"; } } } } catch { lastError = "(non disponibile)"; } // ---- overlay TUI ---- await ctx.ui.custom( (tui, theme, _keybindings, done) => { const container = new Container(); const render = () => { container.clear(); container.addChild(new DynamicBorder((s: string) => theme.fg("accent", s))); container.addChild( new Text(theme.fg("accent", theme.bold("📊 Stato agy-pi")), 1, 1), ); container.addChild(new Text("", 0, 0)); const line = (label: string, v: string, color: "success" | "error" | "text" | "muted" | "accent" | "warning" | "dim") => container.addChild( new Text(theme.fg("dim", `${label}: `) + theme.fg(color, v), 1, 0), ); line("Binario agy", agyExists ? `trovato ${agyBin}` : "❌ NON trovato", agyExists ? "success" : "error"); line("Versione", agyVersion, "text"); line("Chiave Gemini", keyStatus, key ? "success" : "error"); line("Chiave (mask)", keyMask, "muted"); line("Modello attivo", model, "accent"); line("Backend STT", sttBackend, "text"); line("Notifiche TTS", ttsNotify, "text"); container.addChild(new Text("", 0, 0)); container.addChild( new Text(theme.fg("warning", "Ultimo errore:") + " " + theme.fg("text", lastError), 1, 0), ); container.addChild(new Text("", 0, 0)); container.addChild( new Text(theme.fg("dim", "Enter o Esc per chiudere"), 1, 0), ); container.addChild(new DynamicBorder((s: string) => theme.fg("accent", s))); }; render(); return { render: (w) => { render(); return container.render(w); }, invalidate: () => container.invalidate(), handleInput: (data) => { if (matchesKey(data, Key.enter) || matchesKey(data, Key.escape)) done(); }, }; }, { overlay: true, overlayOptions: { width: "55%", minWidth: 56, anchor: "center" }, }, ); }, }); }