feat: add multimodal audio analysis

2026-09-08 11:11:06 +02:00
parent 4a88084b2b
commit 4920c4c0ba
4 changed files with 202 additions and 1 deletions
@@ -42,6 +42,8 @@ const CONV_DIR = path.join(os.homedir(), ".gemini", "antigravity-cli", "conversa
 const DEFAULT_TIMEOUT_MS = 180_000; // 3 min
 const IMAGE_TIMEOUT_MS = 300_000; // 5 min
 const IMAGE_EXTENSIONS = new Set([".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff", ".svg"]);
+const AUDIO_EXTENSIONS = new Set([".wav", ".mp3", ".m4a", ".aac", ".flac", ".ogg", ".opus", ".webm"]);
+const AUDIO_MAX_BYTES = 25 * 1024 * 1024;

 // ---------------------------------------------------------------------------
 // Configurazione persistente (~/.config/agy-pi/config.json)
@@ -983,6 +985,155 @@ function toolUpdate(onUpdate: any, text: string) {
 	onUpdate?.({ content: [{ type: "text", text }] });
 }

+interface AudioLocalMetadata {
+	fileName: string;
+	durationSeconds: number | null;
+	codec: string | null;
+	container: string | null;
+	sampleRateHz: number | null;
+	channels: number | null;
+	bitDepth: number | null;
+	peakDbfs: number | null;
+	meanDbfs: number | null;
+	silenceIntervals: Array<{ startSeconds: number; endSeconds?: number }>;
+}
+
+function audioMimeType(file: string): string {
+	const ext = path.extname(file).toLowerCase();
+	return {
+		".wav": "audio/wav",
+		".mp3": "audio/mpeg",
+		".m4a": "audio/mp4",
+		".aac": "audio/aac",
+		".flac": "audio/flac",
+		".ogg": "audio/ogg",
+		".opus": "audio/opus",
+		".webm": "audio/webm",
+	}[ext] ?? "application/octet-stream";
+}
+
+async function inspectAudioLocally(file: string): Promise<AudioLocalMetadata> {
+	const probe = await execFileAsync(
+		"ffprobe",
+		[
+			"-v", "error",
+			"-show_entries",
+			"format=format_name,duration:stream=codec_name,codec_type,sample_rate,channels,bits_per_sample",
+			"-of", "json",
+			file,
+		],
+		{ timeout: 30_000, maxBuffer: 1_000_000 },
+	);
+	const parsed = JSON.parse(probe.stdout || "{}");
+	const stream = (parsed.streams ?? []).find((item: any) => item.codec_type === "audio") ?? parsed.streams?.[0] ?? {};
+	const numberOrNull = (value: unknown): number | null => {
+		const number = Number(value);
+		return Number.isFinite(number) ? number : null;
+	};
+
+	let peakDbfs: number | null = null;
+	let meanDbfs: number | null = null;
+	let silenceIntervals: Array<{ startSeconds: number; endSeconds?: number }> = [];
+	try {
+		const measured = await execFileAsync(
+			"ffmpeg",
+			["-hide_banner", "-i", file, "-af", "volumedetect,silencedetect=noise=-50dB:d=0.02", "-f", "null", "-"],
+			{ timeout: 60_000, maxBuffer: 2_000_000 },
+		);
+		const stderr = measured.stderr || "";
+		const peakMatch = stderr.match(/max_volume:\s*(-?[0-9.]+|-inf)\s*dB/);
+		const meanMatch = stderr.match(/mean_volume:\s*(-?[0-9.]+|-inf)\s*dB/);
+		if (peakMatch && peakMatch[1] !== "-inf") peakDbfs = Number(peakMatch[1]);
+		if (meanMatch && meanMatch[1] !== "-inf") meanDbfs = Number(meanMatch[1]);
+		const starts = [...stderr.matchAll(/silence_start:\s*([0-9.]+)/g)].map((match) => Number(match[1]));
+		const ends = [...stderr.matchAll(/silence_end:\s*([0-9.]+)/g)].map((match) => Number(match[1]));
+		silenceIntervals = starts.map((start, index) => ({ startSeconds: start, endSeconds: ends[index] }));
+	} catch {
+		// ffprobe è la misura minima; l'analisi Gemini può comunque procedere.
+	}
+
+	return {
+		fileName: path.basename(file),
+		durationSeconds: numberOrNull(parsed.format?.duration),
+		codec: stream.codec_name ?? null,
+		container: parsed.format?.format_name ?? null,
+		sampleRateHz: numberOrNull(stream.sample_rate),
+		channels: numberOrNull(stream.channels),
+		bitDepth: numberOrNull(stream.bits_per_sample),
+		peakDbfs,
+		meanDbfs,
+		silenceIntervals,
+	};
+}
+
+async function analyzeAudioWithAntigravity(
+	file: string,
+	question: string | undefined,
+	modelName: string | undefined,
+	outputDir: string | undefined,
+	signal?: AbortSignal,
+): Promise<{ text: string; metadata: AudioLocalMetadata; waveformPath?: string; reportPath?: string; model: string }> {
+	const requestedModel = modelName || getConfig("voiceModel") || ANTG_DEFAULT_MODEL;
+	if (!requestedModel.startsWith("gemini-")) {
+		throw new Error("L'analisi audio multimodale richiede un modello Gemini Antigravity, non un modello testuale alternativo.");
+	}
+	if (signal?.aborted) throw new Error("Analisi audio annullata");
+	const size = fs.statSync(file).size;
+	if (size > AUDIO_MAX_BYTES) throw new Error(`File audio troppo grande: ${size} byte (limite ${AUDIO_MAX_BYTES})`);
+
+	const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "agy-audio-analysis-"));
+	try {
+		const metadata = await inspectAudioLocally(file);
+		const waveform = path.join(workDir, "waveform.png");
+		await execFileAsync(
+			"ffmpeg",
+			[
+				"-hide_banner", "-loglevel", "error", "-y", "-i", file,
+				"-filter_complex", "showwavespic=s=1600x420:split_channels=0:colors=0x4f7cff",
+				"-frames:v", "1", waveform,
+			],
+			{ timeout: 60_000, maxBuffer: 1_000_000 },
+		);
+		if (signal?.aborted) throw new Error("Analisi audio annullata");
+
+		const instructions =
+			"Analizza ascoltando il file audio e osservando anche la waveform PNG allegata. " +
+			"Rispondi in italiano con una descrizione percettiva prudente, timbro/materiale probabile, " +
+			"attacco, transienti, corpo, decadimento, silenzi, clipping e artefatti. " +
+			"Se l'utente chiede una valutazione, separa ciò che osservi dai metadati locali da ciò che inferisci dall'ascolto. " +
+			"Non inventare parlato o dettagli non udibili. " +
+			(question ? `Richiesta specifica: ${question}\n\n` : "") +
+			`Metadati misurati localmente:\n${JSON.stringify(metadata, null, 2)}`;
+		const response = await antgGenerate({
+			parts: [
+				{ text: instructions },
+				{ inlineData: { mimeType: audioMimeType(file), data: fs.readFileSync(file).toString("base64") } },
+				{ inlineData: { mimeType: "image/png", data: fs.readFileSync(waveform).toString("base64") } },
+			],
+			model: requestedModel,
+			maxOutputTokens: 5000,
+			temperature: 0.2,
+			thinking: "low",
+			stream: true,
+		});
+
+		let waveformPath: string | undefined;
+		let reportPath: string | undefined;
+		if (outputDir) {
+			const destination = path.resolve(outputDir);
+			fs.mkdirSync(destination, { recursive: true });
+			const stem = path.basename(file, path.extname(file)).replace(/[^a-zA-Z0-9._-]+/g, "_");
+			waveformPath = path.join(destination, `${stem}.waveform.png`);
+			reportPath = path.join(destination, `${stem}.analysis.json`);
+			fs.copyFileSync(waveform, waveformPath);
+			fs.writeFileSync(reportPath, JSON.stringify({ filePath: file, model: requestedModel, metadata, analysis: response.text }, null, 2));
+		}
+		return { text: response.text, metadata, waveformPath, reportPath, model: requestedModel };
+	} finally {
+		try { fs.rmSync(workDir, { recursive: true, force: true }); } catch { /* ignora */ }
+	}
+}
+
 // Trascrizione audio esclusivamente tramite il gateway Antigravity OAuth.
 // La CLI agy non accetta file audio, ma cloudcode-pa supporta inlineData
 // audio con i modelli Gemini multimodali dell'account Antigravity.
@@ -2250,6 +2401,45 @@ export default function agyExtension(pi: ExtensionAPI) {
 		},
 	});

+	// =========================================================================
+	// TOOL: agy_analyze_audio — audio + waveform + metadati + Gemini
+	// =========================================================================
+	pi.registerTool({
+		name: "agy_analyze_audio",
+		label: "agy analyze audio multimodale",
+		description:
+			"Analyze a local audio file together with a generated waveform and measured metadata through Gemini over Antigravity OAuth. " +
+			"Use for sound effects, music, recordings, timbre, transients, silence and clipping. " +
+			"It does not assume speech and does not use an external Gemini API key.",
+		parameters: Type.Object({
+			filePath: Type.String({ description: "Percorso del file audio (WAV, MP3, M4A, FLAC, OGG, OPUS o WEBM)." }),
+			question: Type.Optional(Type.String({ description: "Domanda specifica sull'audio o sul suo uso." })),
+			model: Type.Optional(Type.String({ description: "Modello Gemini Antigravity (default: voiceModel configurato)." })),
+			outputDir: Type.Optional(Type.String({ description: "Cartella opzionale dove conservare waveform PNG e report JSON; l'audio non viene copiato." })),
+		}),
+		async execute(toolCallId, params, signal, onUpdate, ctx) {
+			const p = params as any;
+			const filePath = path.resolve(ctx.cwd, String(p.filePath ?? "").replace(/^@/, ""));
+			const ext = path.extname(filePath).toLowerCase();
+			if (!AUDIO_EXTENSIONS.has(ext)) throw new Error(`Formato audio non supportato: ${ext || "senza estensione"}`);
+			if (!fs.existsSync(filePath) || !fs.statSync(filePath).isFile()) throw new Error(`File audio non trovato: ${filePath}`);
+			toolUpdate(onUpdate, "agy_analyze_audio: waveform e metadati locali...");
+			const result = await analyzeAudioWithAntigravity(filePath, p.question, p.model, p.outputDir, signal);
+			toolUpdate(onUpdate, "agy_analyze_audio: analisi Gemini completata");
+			const local = JSON.stringify(result.metadata, null, 2);
+			return {
+				content: [{ type: "text", text: `${result.text}\n\nMetadati misurati localmente:\n${local}` }],
+				details: {
+					filePath,
+					model: result.model,
+					metadata: result.metadata,
+					waveformPath: result.waveformPath,
+					reportPath: result.reportPath,
+				},
+			};
+		},
+	});
+
 	// =========================================================================
 	// TOOL: agy_video — analisi video
 	// =========================================================================