351 lines
14 KiB
TypeScript
351 lines
14 KiB
TypeScript
/**
|
|
* kde-automation — Estensione pi per automazione desktop KDE Plasma 6 (Wayland)
|
|
*
|
|
* Fornisce tool nativi per:
|
|
* - Screenshot (spectacle CLI)
|
|
* - Input simulato (ydotool: click, type, mousemove)
|
|
* - Gestione finestre (KWin scripting via D-Bus)
|
|
* - Mostra Desktop (KWin D-Bus / KGlobalAccel, compatibile Wayland)
|
|
* - Ispezione UI (AT-SPI via python-dbus)
|
|
*
|
|
* Include la prassi di verifica UI come promptGuidelines e una skill /skill:kde-verify.
|
|
*/
|
|
|
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
import { Type } from "typebox";
|
|
import { execFile, spawn } from "node:child_process";
|
|
import { promisify } from "node:util";
|
|
import { existsSync, writeFileSync, mkdirSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { homedir } from "node:os";
|
|
|
|
const exec = promisify(execFile);
|
|
const SOCKET = "/run/user/1000/.ydotool_socket";
|
|
const UID = 1000;
|
|
const TMP = "/tmp";
|
|
|
|
// ---------- helpers ----------
|
|
|
|
async function run(cmd: string, args: string[], env?: Record<string, string>): Promise<string> {
|
|
const { stdout, stderr } = await exec(cmd, args, { env: { ...process.env, ...env } });
|
|
return stdout + (stderr || "");
|
|
}
|
|
|
|
/** Avvia ydotoold (idempotente) se il socket non esiste. Richiede sudoers NOPASSWD per ydotoold. */
|
|
async function ensureDaemon(): Promise<string> {
|
|
if (existsSync(SOCKET)) return "ydotoold attivo (socket presente)";
|
|
try {
|
|
await run("pgrep", ["-x", "ydotoold"]);
|
|
return "ydotoold in esecuzione ma socket mancante: riavviare con sudo ydotoold -p " + SOCKET + " -P 0666 -o " + UID + ":" + UID;
|
|
} catch {
|
|
// non in esecuzione: avvia
|
|
}
|
|
try {
|
|
const child = spawn("sudo", ["-n", "ydotoold", "-p", SOCKET, "-P", "0666", "-o", `${UID}:${UID}`], {
|
|
detached: true,
|
|
stdio: "ignore",
|
|
});
|
|
child.unref();
|
|
await new Promise((r) => setTimeout(r, 1200));
|
|
if (existsSync(SOCKET)) return "ydotoold avviato (socket creato)";
|
|
return "ydotoold avviato ma socket non ancora presente";
|
|
} catch (e) {
|
|
return "ERRORE avvio ydotoold: " + String(e);
|
|
}
|
|
}
|
|
|
|
/** Attiva una finestra per resourceClass via KWin scripting. */
|
|
async function activateWindow(resourceClass: string): Promise<string> {
|
|
const script = `const target = ${JSON.stringify(resourceClass)};
|
|
const clients = workspace.windowList();
|
|
let found = false;
|
|
for (const c of clients) {
|
|
if (c.resourceClass === target) {
|
|
workspace.activeWindow = c;
|
|
found = true;
|
|
break;
|
|
}
|
|
}
|
|
print(found ? "ok" : "notfound");`;
|
|
const scriptPath = join(TMP, "kwin-activate.js");
|
|
writeFileSync(scriptPath, script);
|
|
await run("qdbus6", ["org.kde.KWin", "/Scripting", "org.kde.kwin.Scripting.loadScript", scriptPath]);
|
|
await run("qdbus6", ["org.kde.KWin", "/Scripting", "org.kde.kwin.Scripting.start"]);
|
|
return "script KWin eseguito per " + resourceClass;
|
|
}
|
|
|
|
/** Legge lo stato Mostra Desktop esposto da KWin. */
|
|
async function readShowDesktopState(): Promise<string> {
|
|
return (await run("qdbus6", [
|
|
"org.kde.KWin",
|
|
"/KWin",
|
|
"org.freedesktop.DBus.Properties.Get",
|
|
"org.kde.KWin",
|
|
"showingDesktop",
|
|
])).trim();
|
|
}
|
|
|
|
/** Attende lo stato desiderato, perché showDesktop è Q_NOREPLY e asincrono. */
|
|
async function waitShowDesktopState(expected?: string): Promise<string> {
|
|
let state = await readShowDesktopState();
|
|
for (let i = 0; expected && state !== expected && i < 10; i++) {
|
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
state = await readShowDesktopState();
|
|
}
|
|
return state;
|
|
}
|
|
|
|
/** Imposta o alterna la modalità Mostra Desktop via API D-Bus native Wayland. */
|
|
async function setShowDesktop(action: string): Promise<string> {
|
|
if (action !== "show" && action !== "hide" && action !== "toggle") {
|
|
throw new Error(`azione non valida: ${action}; usare show, hide oppure toggle`);
|
|
}
|
|
|
|
let expected: string | undefined;
|
|
if (action === "toggle") {
|
|
const before = await readShowDesktopState();
|
|
expected = before === "true" ? "false" : before === "false" ? "true" : undefined;
|
|
// KGlobalAccel esegue l'azione registrata da KWin senza simulare input.
|
|
await run("qdbus6", [
|
|
"org.kde.kglobalaccel",
|
|
"/component/kwin",
|
|
"org.kde.kglobalaccel.Component.invokeShortcut",
|
|
"Show Desktop",
|
|
]);
|
|
} else {
|
|
expected = action === "show" ? "true" : "false";
|
|
// dbus-send tipizza esplicitamente il booleano; qdbus6 su questa build
|
|
// lo interpreta in modo non affidabile se passato come stringa "true".
|
|
await run("dbus-send", [
|
|
"--session",
|
|
"--dest=org.kde.KWin",
|
|
"--type=method_call",
|
|
"/KWin",
|
|
"org.kde.KWin.showDesktop",
|
|
`boolean:${action === "show"}`,
|
|
]);
|
|
}
|
|
|
|
const state = await waitShowDesktopState(expected);
|
|
return `Mostra Desktop: azione=${action}; showingDesktop=${state}`;
|
|
}
|
|
|
|
/** Ispeziona l'albero AT-SPI (applicazioni + primi children). */
|
|
async function inspectUI(depth: number): Promise<string> {
|
|
const py = `
|
|
import dbus, sys
|
|
bus = dbus.bus.BusConnection("unix:path=/run/user/1000/at-spi/bus_1")
|
|
root = bus.get_object("org.a11y.atspi.Registry", "/org/a11y/atspi/accessible/root")
|
|
acc = dbus.Interface(root, "org.a11y.atspi.Accessible")
|
|
def nm(o):
|
|
try: return str(o.Get("org.a11y.atspi.Accessible", "Name", dbus_interface="org.freedesktop.DBus.Properties")) or "?"
|
|
except Exception: return "?"
|
|
out = []
|
|
for ch in acc.GetChildren():
|
|
try:
|
|
o = bus.get_object(ch[0], ch[1]); a = dbus.Interface(o, "org.a11y.atspi.Accessible")
|
|
out.append("APP: %s | %s" % (nm(a), a.GetRoleName()))
|
|
if ${depth} > 0:
|
|
for k in a.GetChildren()[:${depth}]:
|
|
try:
|
|
ko = bus.get_object(k[0], k[1]); ka = dbus.Interface(ko, "org.a11y.atspi.Accessible")
|
|
out.append(" - %s | %s" % (nm(ka), ka.GetRoleName()))
|
|
except Exception: pass
|
|
except Exception: pass
|
|
print("\\n".join(out))
|
|
`;
|
|
const scriptPath = join(TMP, "atspi-inspect.py");
|
|
writeFileSync(scriptPath, py);
|
|
return run("python3", [scriptPath]);
|
|
}
|
|
|
|
// ---------- estensione ----------
|
|
|
|
export default function kdeAutomation(pi: ExtensionAPI) {
|
|
// --- promptGuidelines: prassi di verifica UI KDE ---
|
|
pi.on("before_agent_start", async (event) => {
|
|
const guidelines = event.systemPromptOptions.promptGuidelines ?? [];
|
|
const kdeGuidelines = [
|
|
"KDE AUTOMATION (desktop Plasma 6 Wayland): per verifiche UI usa i tool kde_* (kde_screenshot, kde_click, kde_type, kde_mousemove, kde_window_activate, kde_ui_inspect, kde_show_desktop).",
|
|
" - Prassi di verifica: 1) kde_screenshot per catturare lo stato; 2) analizza l'immagine (agy_analyze se serve visione); 3) azione (kde_click/kde_type/kde_window_activate); 4) kde_screenshot di verifica e confronta.",
|
|
" - ydotool usa la mappa tastiera US: con layout IT i caratteri speciali (trattini, @, ecc.) vanno specificati con keycode o attenzione.",
|
|
" - Su Wayland non esiste iniezione input nativa: ydotool (via /dev/uinput) è il metodo; il daemon si avvia da solo (sudoers NOPASSWD configurato).",
|
|
" - Per finestre: kde_window_activate con resourceClass (es. org.kde.konsole, org.kde.dolphin).",
|
|
" - Per Mostra Desktop usa kde_show_desktop: show/hide sono deterministici, toggle invoca KGlobalAccel; evitare xdotool/ydotool per questa azione.",
|
|
" - Dettagli completi: skill /skill:kde-verify e memoria qmem (project host-enne2-gk35).",
|
|
];
|
|
return {
|
|
systemPrompt: event.systemPrompt,
|
|
systemPromptOptions: {
|
|
...event.systemPromptOptions,
|
|
promptGuidelines: [...guidelines, ...kdeGuidelines],
|
|
},
|
|
};
|
|
});
|
|
|
|
// --- skill kde-verify ---
|
|
pi.on("resources_discover", async () => {
|
|
return {
|
|
skillPaths: [join(homedir(), ".pi/agent/extensions/kde-automation/skills")],
|
|
};
|
|
});
|
|
|
|
// --- comando /kde-verify ---
|
|
pi.registerCommand("kde-verify", {
|
|
description: "Mostra la prassi di verifica UI KDE (screenshot + input + finestre)",
|
|
handler: async (_args, ctx) => {
|
|
const text = [
|
|
"PRASSI VERIFICA UI KDE (Plasma 6 Wayland):",
|
|
"1. kde_screenshot (mode=fullscreen|active|window|region, output=/tmp/x.png)",
|
|
"2. Analizza l'immagine (read o agy_analyze per visione)",
|
|
"3. Azione: kde_window_activate (resourceClass) / kde_click (x,y,button) / kde_type (text)",
|
|
"4. kde_screenshot di verifica e confronta con lo stato atteso",
|
|
"5. Per nascondere/ripristinare tutte le finestre: kde_show_desktop (show|hide|toggle)",
|
|
"Nota: ydotool usa mappa US (layout IT: caratteri speciali diversi); per Show Desktop usare D-Bus.",
|
|
"Daemon: si avvia da solo via sudoers NOPASSWD (ydotoold).",
|
|
].join("\n");
|
|
ctx.ui.notify(text, "info");
|
|
},
|
|
});
|
|
|
|
// --- tool: kde_screenshot ---
|
|
pi.registerTool({
|
|
name: "kde_screenshot",
|
|
label: "KDE Screenshot",
|
|
description:
|
|
"Cattura uno screenshot del desktop KDE con Spectacle (nativo). mode: fullscreen (intero desktop), active (finestra attiva), window (finestra sotto cursore), region (selezione). output: percorso file PNG. delay: millisecondi di attesa. copy: copia in clipboard invece di salvare.",
|
|
parameters: Type.Object({
|
|
mode: Type.Optional(Type.String({ description: "fullscreen | active | window | region (default fullscreen)" })),
|
|
output: Type.Optional(Type.String({ description: "Percorso file PNG (default /tmp/kde-shot-<ts>.png)" })),
|
|
delay: Type.Optional(Type.Number({ description: "Delay in millisecondi prima della cattura" })),
|
|
copy: Type.Optional(Type.Boolean({ description: "Copia in clipboard invece di salvare su file" })),
|
|
}),
|
|
async execute(_id, params) {
|
|
const mode = params.mode ?? "fullscreen";
|
|
const out = params.output ?? `/tmp/kde-shot-${Date.now()}.png`;
|
|
const args = ["-b"];
|
|
if (mode === "active") args.push("-a");
|
|
else if (mode === "window") args.push("-u");
|
|
else if (mode === "region") args.push("-r");
|
|
else args.push("-f");
|
|
if (params.delay) args.push("-d", String(params.delay));
|
|
if (params.copy) args.push("-c");
|
|
else args.push("-o", out);
|
|
await run("spectacle", args);
|
|
return {
|
|
content: [{ type: "text", text: params.copy ? "Screenshot copiato in clipboard" : `Screenshot salvato: ${out}` }],
|
|
details: { output: params.copy ? undefined : out },
|
|
};
|
|
},
|
|
});
|
|
|
|
// --- tool: kde_show_desktop ---
|
|
pi.registerTool({
|
|
name: "kde_show_desktop",
|
|
label: "KDE Show Desktop",
|
|
description:
|
|
"Mostra o nasconde tutte le finestre con l'API D-Bus nativa di KWin su Plasma Wayland. action=show nasconde le finestre, hide le ripristina, toggle invoca l'azione globale KWin. Restituisce e verifica la proprietà showingDesktop.",
|
|
parameters: Type.Object({
|
|
action: Type.Optional(Type.String({ description: "show | hide | toggle (default show)" })),
|
|
}),
|
|
async execute(_id, params) {
|
|
const action = params.action ?? "show";
|
|
const result = await setShowDesktop(action);
|
|
return { content: [{ type: "text", text: result }], details: { action } };
|
|
},
|
|
});
|
|
|
|
// --- tool: kde_click ---
|
|
pi.registerTool({
|
|
name: "kde_click",
|
|
label: "KDE Click",
|
|
description:
|
|
"Simula un clic del mouse via ydotool. button: 1=sinistro, 2=centro, 3=destro. Se x e y sono forniti, prima muove il mouse a quelle coordinate. Richiede il daemon ydotoold (avvio automatico).",
|
|
parameters: Type.Object({
|
|
button: Type.Optional(Type.Number({ description: "1=sinistro (default), 2=centro, 3=destro" })),
|
|
x: Type.Optional(Type.Number({ description: "Coordinata X (opzionale, muove il mouse prima del clic)" })),
|
|
y: Type.Optional(Type.Number({ description: "Coordinata Y (opzionale)" })),
|
|
}),
|
|
async execute(_id, params) {
|
|
const daemon = await ensureDaemon();
|
|
const env = { YDOTOOL_SOCKET: SOCKET };
|
|
let log = daemon + "\n";
|
|
if (params.x !== undefined && params.y !== undefined) {
|
|
await run("ydotool", ["mousemove", String(params.x), String(params.y)], env);
|
|
log += `mouse -> ${params.x},${params.y}\n`;
|
|
}
|
|
await run("ydotool", ["click", String(params.button ?? 1)], env);
|
|
log += `click ${params.button ?? 1}`;
|
|
return { content: [{ type: "text", text: log }], details: {} };
|
|
},
|
|
});
|
|
|
|
// --- tool: kde_type ---
|
|
pi.registerTool({
|
|
name: "kde_type",
|
|
label: "KDE Type",
|
|
description:
|
|
"Digita testo via ydotool nella finestra attiva. ATTENZIONE: ydotool usa la mappa tastiera US — con layout IT i caratteri speciali (trattini, @, ecc.) possono differire. Per testo semplice ASCII va bene.",
|
|
parameters: Type.Object({
|
|
text: Type.String({ description: "Testo da digitare" }),
|
|
}),
|
|
async execute(_id, params) {
|
|
const daemon = await ensureDaemon();
|
|
await run("ydotool", ["type", params.text], { YDOTOOL_SOCKET: SOCKET });
|
|
return {
|
|
content: [{ type: "text", text: `${daemon}\nTesto digitato: "${params.text}"` }],
|
|
details: {},
|
|
};
|
|
},
|
|
});
|
|
|
|
// --- tool: kde_mousemove ---
|
|
pi.registerTool({
|
|
name: "kde_mousemove",
|
|
label: "KDE Mouse Move",
|
|
description: "Sposta il cursore del mouse alle coordinate (x, y) via ydotool.",
|
|
parameters: Type.Object({
|
|
x: Type.Number({ description: "Coordinata X" }),
|
|
y: Type.Number({ description: "Coordinata Y" }),
|
|
}),
|
|
async execute(_id, params) {
|
|
const daemon = await ensureDaemon();
|
|
await run("ydotool", ["mousemove", String(params.x), String(params.y)], { YDOTOOL_SOCKET: SOCKET });
|
|
return {
|
|
content: [{ type: "text", text: `${daemon}\nMouse -> ${params.x},${params.y}` }],
|
|
details: {},
|
|
};
|
|
},
|
|
});
|
|
|
|
// --- tool: kde_window_activate ---
|
|
pi.registerTool({
|
|
name: "kde_window_activate",
|
|
label: "KDE Window Activate",
|
|
description:
|
|
"Attiva (porta in primo piano) la finestra con la resourceClass specificata via KWin scripting. Esempi: org.kde.konsole, org.kde.dolphin, org.kde.plasma-systemmonitor, firefox.",
|
|
parameters: Type.Object({
|
|
resourceClass: Type.String({ description: "resourceClass della finestra (es. org.kde.konsole)" }),
|
|
}),
|
|
async execute(_id, params) {
|
|
const result = await activateWindow(params.resourceClass);
|
|
return { content: [{ type: "text", text: result }], details: {} };
|
|
},
|
|
});
|
|
|
|
// --- tool: kde_ui_inspect ---
|
|
pi.registerTool({
|
|
name: "kde_ui_inspect",
|
|
label: "KDE UI Inspect",
|
|
description:
|
|
"Ispeziona l'albero di accessibilità AT-SPI: elenca le applicazioni e i loro primi controlli (nomi e ruoli). Utile per capire cosa è presente a schermo e identificare elementi per l'automazione.",
|
|
parameters: Type.Object({
|
|
depth: Type.Optional(Type.Number({ description: "Quanti children mostrare per applicazione (default 3)" })),
|
|
}),
|
|
async execute(_id, params) {
|
|
const out = await inspectUI(params.depth ?? 3);
|
|
return { content: [{ type: "text", text: out }], details: {} };
|
|
},
|
|
});
|
|
}
|