/**
 * pi-qmem — indice locale SQLite/FTS5 (fallback testuale quando il gateway è giù).
 *
 * Perché: il gateway remoto (Qdrant + BGE-M3) non è sempre raggiungibile (VPN
 * giù, nodo offline). Questo modulo costruisce un indice **locale** dei record
 * di memoria a partire da due fonti:
 *
 *   1. le sessioni pi (~/.pi/agent/sessions/<cwd>/*.jsonl): ogni chiamata
 *      `qmem_store`/`qmem_correct`/`qmem_get`/`qmem_search` contiene l'ID
 *      fornito dal gateway e il testo integrale → storico ricostruibile offline;
 *   2. il gateway stesso, quando è raggiungibile (`enrich` via GET
 *      /v1/memories/{id}, `pull` via GET /v1/memories:export quando esisterà).
 *
 * La ricerca è **lessicale** (FTS5 + BM25), non neurale: nessun embedding,
 * nessun modello, nessuna dipendenza esterna. È il fallback dichiarato di
 * `qmem_search` quando il gateway non risponde.
 *
 * DB di default: ~/.local/share/pi-qmem/qmem.sqlite (override: env QMEM_SQLITE
 * oppure `localDbPath` in ~/.config/pi-qmem/config.json).
 *
 * Nessuna dipendenza: usa `node:sqlite` (Node ≥ 22.5). Se il modulo non è
 * disponibile, le funzioni degradano con un errore esplicito.
 */
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { createHash, randomUUID } from "node:crypto";
import { gatewayRequest, loadConfig, type MemoryConfig } from "./shared.ts";

// ---------------------------------------------------------------------------
// Percorsi e apertura
// ---------------------------------------------------------------------------
export const DEFAULT_DB_FILE = path.join(os.homedir(), ".local", "share", "pi-qmem", "qmem.sqlite");

export function localDbPath(cfg?: MemoryConfig & { localDbPath?: string }): string {
	return process.env.QMEM_SQLITE ?? cfg?.localDbPath ?? DEFAULT_DB_FILE;
}

export function sessionRoots(): string[] {
	const roots = process.env.QMEM_SESSIONS_DIR
		? [process.env.QMEM_SESSIONS_DIR]
		: [path.join(os.homedir(), ".pi", "agent", "sessions")];
	return roots.filter((r) => {
		try {
			return fs.statSync(r).isDirectory();
		} catch {
			return false;
		}
	});
}

type Db = any;

let sqliteModule: any | null | undefined;

/** Carica `node:sqlite` una volta sola, sopprimendo l'ExperimentalWarning. */
async function loadSqlite(): Promise<any | null> {
	if (sqliteModule !== undefined) return sqliteModule;
	const originalEmit = process.emitWarning;
	try {
		// il warning "SQLite is an experimental feature" sporcherebbe la TUI
		(process as any).emitWarning = (warning: any, ...rest: any[]) => {
			const msg = typeof warning === "string" ? warning : String(warning?.message ?? "");
			const type = (rest[0] as any)?.type ?? rest[0];
			if (type === "ExperimentalWarning" && /sqlite/i.test(msg)) return;
			return (originalEmit as any).call(process, warning, ...rest);
		};
		sqliteModule = await import("node:sqlite");
	} catch {
		sqliteModule = null;
	} finally {
		(process as any).emitWarning = originalEmit;
	}
	return sqliteModule;
}

export async function openLocalDb(dbFile?: string): Promise<Db> {
	const mod = await loadSqlite();
	if (!mod?.DatabaseSync) {
		throw new Error("node:sqlite non disponibile: serve Node >= 22.5 per l'indice locale");
	}
	const file = dbFile ?? localDbPath(loadConfig());
	if (file !== ":memory:") fs.mkdirSync(path.dirname(file), { recursive: true });
	const db = new mod.DatabaseSync(file);
	db.exec("PRAGMA journal_mode=WAL;");
	db.exec(`
CREATE TABLE IF NOT EXISTS records(
  rowid_ INTEGER PRIMARY KEY AUTOINCREMENT,
  memory_id TEXT UNIQUE NOT NULL,
  text TEXT, kind TEXT, project_id TEXT, scope TEXT, agent_id TEXT, confidence TEXT,
  importance REAL, created_at TEXT, supersedes_id TEXT, superseded_by TEXT,
  parent_id TEXT, topic TEXT, level TEXT, private INTEGER NOT NULL DEFAULT 0,
  text_hash TEXT, sources TEXT, synced_at TEXT, updated_at TEXT,
  /** 1 = record creato offline, non ancora presente sul gateway */
  pending INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS records_project ON records(project_id);
CREATE INDEX IF NOT EXISTS records_kind ON records(kind);
CREATE INDEX IF NOT EXISTS records_hash ON records(text_hash);
CREATE VIRTUAL TABLE IF NOT EXISTS records_fts USING fts5(
  text, kind, project_id, content='records', content_rowid='rowid_',
  tokenize="unicode61 remove_diacritics 2"
);
CREATE TRIGGER IF NOT EXISTS records_ai AFTER INSERT ON records BEGIN
  INSERT INTO records_fts(rowid, text, kind, project_id) VALUES (new.rowid_, new.text, new.kind, new.project_id);
END;
CREATE TRIGGER IF NOT EXISTS records_au AFTER UPDATE ON records BEGIN
  INSERT INTO records_fts(records_fts, rowid, text, kind, project_id) VALUES('delete', old.rowid_, old.text, old.kind, old.project_id);
  INSERT INTO records_fts(rowid, text, kind, project_id) VALUES (new.rowid_, new.text, new.kind, new.project_id);
END;
CREATE TRIGGER IF NOT EXISTS records_ad AFTER DELETE ON records BEGIN
  INSERT INTO records_fts(records_fts, rowid, text, kind, project_id) VALUES('delete', old.rowid_, old.text, old.kind, old.project_id);
END;
CREATE TABLE IF NOT EXISTS meta(k TEXT PRIMARY KEY, v TEXT);
CREATE TABLE IF NOT EXISTS pending(
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  local_id TEXT UNIQUE NOT NULL,
  payload TEXT NOT NULL,
  created_at TEXT NOT NULL,
  attempts INTEGER NOT NULL DEFAULT 0,
  last_attempt TEXT,
  last_error TEXT,
  status TEXT NOT NULL DEFAULT 'queued',   -- queued | synced | duplicate | failed
  remote_id TEXT,
  synced_at TEXT
);
CREATE INDEX IF NOT EXISTS pending_status ON pending(status);
`);
	// migrazione per DB creati prima dell'introduzione della coda
	const cols = db
		.prepare("PRAGMA table_info(records)")
		.all()
		.map((c: any) => c.name);
	if (!cols.includes("pending")) db.exec("ALTER TABLE records ADD COLUMN pending INTEGER NOT NULL DEFAULT 0");
	return db;
}

// ---------------------------------------------------------------------------
// Upsert
// ---------------------------------------------------------------------------
export interface LocalRecordInput {
	memory_id: string;
	text?: string | null;
	kind?: string | null;
	project_id?: string | null;
	scope?: string | null;
	agent_id?: string | null;
	confidence?: string | null;
	importance?: number | null;
	created_at?: string | null;
	supersedes_id?: string | null;
	superseded_by?: string | null;
	parent_id?: string | null;
	topic?: string | null;
	level?: string | null;
	private?: boolean | null;
	/** true = creato offline (in coda), false = presente sul gateway */
	pending?: boolean | null;
	/** Fonte dell'osservazione: store | correct | get | search | export | enrich | outbox | sync */
	source: string;
}

/** Sorgenti ordinate per affidabilità del testo (maggiore = più completo). */
const TEXT_PRIORITY: Record<string, number> = { export: 5, enrich: 5, sync: 5, get: 4, store: 3, correct: 3, outbox: 3, search: 2 };

function normalizeText(t: string): string {
	return t.replace(/\s+/g, " ").trim().toLowerCase();
}

export function textHashOf(text: string): string {
	return createHash("sha256").update(normalizeText(text)).digest("hex").slice(0, 32);
}

const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;

export function upsertRecords(db: Db, records: LocalRecordInput[]): number {
	if (!records.length) return 0;
	const find = db.prepare("SELECT * FROM records WHERE memory_id = ?");
	const insert = db.prepare(`INSERT INTO records
		(memory_id, text, kind, project_id, scope, agent_id, confidence, importance, created_at,
		 supersedes_id, superseded_by, parent_id, topic, level, private, text_hash, sources, synced_at, updated_at, pending)
		VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`);
	const update = db.prepare(`UPDATE records SET
		text=?, kind=?, project_id=?, scope=?, agent_id=?, confidence=?, importance=?, created_at=?,
		supersedes_id=?, superseded_by=?, parent_id=?, topic=?, level=?, private=?, text_hash=?, sources=?, synced_at=?, updated_at=?, pending=?
		WHERE rowid_=?`);
	const now = new Date().toISOString();
	let written = 0;
	for (const r of records) {
		if (!r.memory_id || !UUID_RE.test(r.memory_id)) continue;
		const existing = find.get(r.memory_id);
		const sources = new Set<string>((existing?.sources ?? "").split(",").filter(Boolean));
		sources.add(r.source);
		const prevText: string | null = existing?.text ?? null;
		const prevPri = Math.max(0, ...(existing?.sources ?? "").split(",").map((s: string) => TEXT_PRIORITY[s] ?? 0));
		const newPri = TEXT_PRIORITY[r.source] ?? 0;
		// il testo più affidabile vince; altrimenti si conserva quello esistente
		const text = prevText && newPri < prevPri ? prevText : (r.text ?? prevText);
		const supersededBy = r.superseded_by ?? existing?.superseded_by ?? null; // osservazione monotona
		const vals = [
			text,
			r.kind ?? null,
			r.project_id ?? null,
			r.scope ?? null,
			r.agent_id ?? null,
			r.confidence ?? null,
			typeof r.importance === "number" ? r.importance : null,
			r.created_at ?? null,
			r.supersedes_id ?? null,
			supersededBy,
			r.parent_id ?? null,
			r.topic ?? null,
			r.level ?? null,
			r.private ? 1 : 0,
			text ? textHashOf(text) : null,
			[...sources].sort().join(","),
			now,
			now,
			r.pending === true ? 1 : r.pending === false ? 0 : (existing?.pending ?? 0),
		];
		if (existing) {
			// I campi già noti non vengono mai azzerati da osservazioni più povere
			// (es. un risultato di ricerca che non riporta project_id).
			const keys = ["text", "kind", "project_id", "scope", "agent_id", "confidence", "importance", "created_at",
				"supersedes_id", "superseded_by", "parent_id", "topic", "level", "private", "text_hash", "sources", "synced_at", "updated_at", "pending"];
			const merged = vals.map((v, i) => {
				const key = keys[i];
				const cur = (existing as any)[key];
				const curEmpty = cur === null || cur === undefined || cur === "" || cur === 0;
				if (key === "private") return cur === 1 ? 1 : v; // monotono: una volta privato resta privato
				if (key === "pending") return v; // 0 = sincronizzato: prevale sullo stato locale
				if (key === "sources" || key === "synced_at" || key === "updated_at") return v;
				if (v === null || v === "" ) return curEmpty ? v : cur;  // non azzerare
				return v;
			});
			update.run(...merged, existing.rowid_);
		} else {
			insert.run(r.memory_id, ...vals);
		}
		written++;
	}
	return written;
}

// ---------------------------------------------------------------------------
// Import dalle sessioni pi
// ---------------------------------------------------------------------------
export interface ImportStats {
	files: number;
	lines: number;
	store: number;
	correct: number;
	get: number;
	searchHits: number;
	unparsed: number;
	records: number;
	written: number;
}

const ID_IN_LINE = /\(id:\s*([0-9a-f-]{36})([^)]*)\)/gi;

/** Estrae i record da un singolo file di sessione (JSONL). */
function parseSessionFile(file: string, stats: ImportStats, out: LocalRecordInput[]): void {
	let raw: string;
	try {
		raw = fs.readFileSync(file, "utf8");
	} catch {
		return;
	}
	stats.files++;
	const calls = new Map<string, { name: string; args: any; ts?: string }>();
	for (const line of raw.split("\n")) {
		if (!line.trim()) continue;
		stats.lines++;
		let entry: any;
		try {
			entry = JSON.parse(line);
		} catch {
			continue;
		}
		if (entry?.type !== "message") continue;
		const msg = entry.message ?? {};
		const content = msg.content;
		if (msg.role === "assistant" && Array.isArray(content)) {
			for (const part of content) {
				if (part?.type === "toolCall" && String(part.name ?? "").startsWith("qmem_")) {
					calls.set(part.id, { name: part.name, args: part.arguments ?? {}, ts: entry.timestamp });
				}
			}
			continue;
		}
		if (msg.role !== "toolResult") continue;
		const name = String(msg.toolName ?? "");
		if (!name.startsWith("qmem_")) continue;
		const call = calls.get(msg.toolCallId) ?? { name, args: {}, ts: entry.timestamp };
		const text = Array.isArray(content)
			? content.filter((p: any) => p?.type === "text").map((p: any) => p.text ?? "").join(" ")
			: String(content ?? "");
		const det = msg.details ?? {};
		const args = call.args ?? {};
		if (call.name === "qmem_store" && det.memory_id) {
			stats.store++;
			out.push({
				memory_id: det.memory_id,
				text: args.text ?? null,
				kind: args.kind ?? null,
				project_id: args.project_id ?? null,
				scope: args.scope ?? null,
				agent_id: args.agent_id ?? null,
				confidence: args.confidence ?? null,
				importance: typeof args.importance === "number" ? args.importance : null,
				created_at: det.created_at ?? call.ts ?? null,
				supersedes_id: args.supersedes_id ?? null,
				parent_id: args.parent_id ?? null,
				topic: args.topic ?? null,
				level: args.level ?? null,
				private: args.private ?? null,
				source: "store",
			});
		} else if (call.name === "qmem_correct" && det.new_id) {
			stats.correct++;
			out.push({
				memory_id: det.new_id,
				text: args.text ?? null,
				kind: args.kind ?? null,
				project_id: args.project_id ?? null,
				scope: args.scope ?? null,
				agent_id: args.agent_id ?? null,
				created_at: call.ts ?? null,
				supersedes_id: det.superseded_id ?? null,
				topic: args.topic ?? null,
				level: args.level ?? null,
				source: "correct",
			});
			if (det.superseded_id) {
				out.push({ memory_id: det.superseded_id, superseded_by: det.new_id, source: "correct" });
			}
		} else if (call.name === "qmem_get" && det.memory_id) {
			stats.get++;
			// formato: "memory_id: <uuid>\n[kind/scope ..] | project: X | agente: Y, creato: Z ...\n\n<testo>"
			const parts = text.split("\n");
			const bodyStart = parts.findIndex((l, i) => i > 0 && l.trim() === "");
			const body = bodyStart >= 0 ? parts.slice(bodyStart + 1).join("\n").trim() : null;
			const project = /project:\s*([^|\n]+)/.exec(text)?.[1]?.trim();
			const created = /creato:\s*([^,|\n]+)/.exec(text)?.[1]?.trim();
			const agent = /agente:\s*([^,|\n]+)/.exec(text)?.[1]?.trim();
			out.push({
				memory_id: det.memory_id,
				text: body && body !== "(nessun testo)" ? body : null,
				kind: det.kind ?? null,
				project_id: det.project_id ?? (project && project !== "?" ? project : null),
				scope: det.scope ?? null,
				agent_id: agent && agent !== "?" ? agent : null,
				created_at: created && created !== "?" ? created : null,
				superseded_by: det.superseded_by ?? null,
				topic: det.topic ?? null,
				level: det.level ?? null,
				source: "get",
			});
		} else if (call.name === "qmem_search") {
			// formato: "N. [kind/scope ...] TESTO\n   (id: <uuid>, agente: .., creato: ..)"
			const re = /^\s*\d+\.\s*\[([^\]/]+)\/([^\]\s]+)[^\]]*\]\s*([\s\S]*?)\n\s*\(id:\s*([0-9a-f-]{36})([^)]*)\)/gim;
			let m: RegExpExecArray | null;
			let found = 0;
			while ((m = re.exec(text))) {
				found++;
				const tail = m[5] ?? "";
				const sup = /supers[^\s]*\s+da\s+([0-9a-f-]{36})/i.exec(tail)?.[1] ?? null;
				const project = /project[=:]\s*([^,)\n]+)/i.exec(tail)?.[1]?.trim() ?? null;
				out.push({
					memory_id: m[4],
					text: m[3].trim(),
					kind: m[1].trim(),
					scope: m[2].trim(),
					project_id: project && project !== "?" ? project : null,
					created_at: /creato:\s*([^,)]+)/.exec(tail)?.[1]?.trim() ?? null,
					agent_id: /agente:\s*([^,)]+)/.exec(tail)?.[1]?.trim() ?? null,
					superseded_by: sup,
					source: "search",
				});
			}
			stats.searchHits += found;
			if (!found && /^\s*\d+\.\s*\[/.test(text)) stats.unparsed++;
			void ID_IN_LINE;
		}
	}
}

/** Import idempotente dalle sessioni pi (tutte le directory di progetto). */
export async function importFromSessions(opts?: { dbFile?: string; roots?: string[] }): Promise<ImportStats> {
	const db = await openLocalDb(opts?.dbFile);
	const stats: ImportStats = { files: 0, lines: 0, store: 0, correct: 0, get: 0, searchHits: 0, unparsed: 0, records: 0, written: 0 };
	const records: LocalRecordInput[] = [];
	for (const root of opts?.roots ?? sessionRoots()) {
		let dirs: string[] = [];
		try {
			dirs = fs.readdirSync(root).map((d) => path.join(root, d));
		} catch {
			continue;
		}
		for (const dir of dirs) {
			let files: string[] = [];
			try {
				files = fs
					.statSync(dir)
					.isDirectory()
					? fs.readdirSync(dir).filter((f) => f.endsWith(".jsonl")).map((f) => path.join(dir, f))
					: [dir];
			} catch {
				continue;
			}
			for (const file of files) parseSessionFile(file, stats, records);
		}
	}
	stats.records = new Set(records.map((r) => r.memory_id)).size;
	stats.written = upsertRecords(db, records);
	db.prepare("INSERT INTO meta(k,v) VALUES('last_import',?) ON CONFLICT(k) DO UPDATE SET v=excluded.v").run(new Date().toISOString());
	db.prepare("INSERT INTO meta(k,v) VALUES('last_import_stats',?) ON CONFLICT(k) DO UPDATE SET v=excluded.v").run(JSON.stringify(stats));
	db.close();
	return stats;
}

// ---------------------------------------------------------------------------
// Ricerca lessicale (FTS5 + BM25)
// ---------------------------------------------------------------------------
export interface LocalSearchParams {
	query: string;
	kind?: string;
	project_id?: string;
	scope?: string;
	level?: string;
	topic?: string;
	include_superseded?: boolean;
	include_private?: boolean;
	top_k?: number;
	/** true = match esatto della frase, senza espansione prefisso */
	exact?: boolean;
}

export interface LocalSearchHit {
	memory_id: string;
	text: string | null;
	kind: string | null;
	project_id: string | null;
	scope: string | null;
	agent_id: string | null;
	created_at: string | null;
	superseded_by: string | null;
	sources: string | null;
	/** 1 = creato offline, non ancora sul gateway */
	pending?: number;
	snippet: string;
	rank: number;
	/** "and" = tutti i termini presenti; "or" = match parziale (AND senza risultati) */
	match_mode?: "and" | "or";
}

/** Converte una query utente in una MATCH di FTS5 senza rischi di sintassi. */
export function toFtsMatch(query: string, exact = false, mode: "and" | "or" = "and"): string {
	const tokens = query
		.replace(/["'()*:^-]/g, " ")
		.split(/\s+/)
		.map((t) => t.trim())
		.filter((t) => t.length > 1);
	if (!tokens.length) return "";
	return tokens.map((t) => (exact ? `"${t}"` : `"${t}"*`)).join(mode === "and" ? " AND " : " OR ");
}

async function runSearch(db: Db, params: LocalSearchParams, match: string): Promise<LocalSearchHit[]> {
	const where: string[] = ["records_fts MATCH ?"];
	const args: any[] = [match];
	if (params.kind) {
		where.push("r.kind = ?");
		args.push(params.kind);
	}
	if (params.project_id) {
		where.push("r.project_id = ?");
		args.push(params.project_id);
	}
	if (params.scope) {
		where.push("r.scope = ?");
		args.push(params.scope);
	}
	if (params.level) {
		where.push("r.level = ?");
		args.push(params.level);
	}
	if (params.topic) {
		where.push("r.topic = ?");
		args.push(params.topic);
	}
	if (!params.include_superseded) where.push("r.superseded_by IS NULL");
	if (!params.include_private) where.push("r.private = 0");
	args.push(Math.min(Math.max(params.top_k ?? 5, 1), 50));
	const sql = `SELECT r.memory_id, r.text, r.kind, r.project_id, r.scope, r.agent_id, r.created_at,
		r.superseded_by, r.sources, r.pending, bm25(records_fts) AS rank,
		snippet(records_fts, 0, '«', '»', '…', 14) AS snippet
		FROM records_fts JOIN records r ON r.rowid_ = records_fts.rowid
		WHERE ${where.join(" AND ")} ORDER BY rank LIMIT ?`;
	return db.prepare(sql).all(...args) as LocalSearchHit[];
}

export async function localSearch(params: LocalSearchParams, opts?: { dbFile?: string }): Promise<LocalSearchHit[]> {
	const match = toFtsMatch(params.query, params.exact);
	if (!match) return [];
	const db = await openLocalDb(opts?.dbFile);
	try {
		let hits = await runSearch(db, params, match);
		for (const h of hits) h.match_mode = "and";
		// AND senza risultati → ripiega su OR (match parziale) per non lasciare l'agente a mani vuote
		if (!hits.length && !params.exact) {
			const orMatch = toFtsMatch(params.query, false, "or");
			if (orMatch && orMatch !== match) {
				hits = await runSearch(db, params, orMatch);
				for (const h of hits) h.match_mode = "or";
			}
		}
		return hits;
	} finally {
		db.close();
	}
}

/** Recupero locale di un singolo record (fallback di qmem_get). */
export async function localGet(memoryId: string, opts?: { dbFile?: string }): Promise<(LocalSearchHit & { queue_status?: string; remote_id?: string | null }) | null> {
	const db = await openLocalDb(opts?.dbFile);
	try {
		const row = db
			.prepare(
				`SELECT memory_id, text, kind, project_id, scope, agent_id, created_at, superseded_by, sources, pending, 0 AS rank,
				        COALESCE(substr(text,1,200),'') AS snippet
				 FROM records WHERE memory_id = ?`,
			)
			.get(memoryId) as LocalSearchHit | undefined;
		if (row) return row;
		// non è nell'indice: potrebbe essere un id locale della coda
		const q = db
			.prepare("SELECT local_id, payload, status, remote_id FROM pending WHERE local_id = ?")
			.get(memoryId) as any;
		if (!q) return null;
		const payload: QueuePayload = JSON.parse(q.payload);
		return {
			memory_id: q.local_id,
			text: payload.text,
			kind: payload.kind ?? "fact",
			project_id: payload.project_id,
			scope: payload.scope ?? "agent",
			agent_id: payload.agent_id ?? null,
			created_at: null,
			superseded_by: null,
			sources: "outbox",
			pending: q.status === "queued" ? 1 : 0,
			snippet: String(payload.text ?? "").slice(0, 200),
			rank: 0,
			queue_status: q.status,
			remote_id: q.remote_id ?? null,
		};
	} finally {
		db.close();
	}
}

// ---------------------------------------------------------------------------
// Arricchimento dal gateway (quando torna raggiungibile)
// ---------------------------------------------------------------------------
export interface EnrichStats {
	requested: number;
	ok: number;
	failed: number;
	updated: number;
	skipped: number;
	errors: string[];
}

/**
 * Arricchisce i record locali con i dati autorevoli del gateway
 * (GET /v1/memories/{id}): testo mancante, project_id, private, stato supersede.
 * Con `onlyIncomplete` (default) tocca solo i record con dati mancanti.
 */
export async function enrichFromGateway(
	cfg: MemoryConfig,
	opts?: { dbFile?: string; onlyIncomplete?: boolean; limit?: number; paceMs?: number },
): Promise<EnrichStats> {
	const db = await openLocalDb(opts?.dbFile);
	const stats: EnrichStats = { requested: 0, ok: 0, failed: 0, updated: 0, skipped: 0, errors: [] };
	try {
		const only = opts?.onlyIncomplete ?? true;
		const sql = only
			? `SELECT memory_id FROM records
			   WHERE text IS NULL OR project_id IS NULL OR synced_at IS NULL OR superseded_by IS NULL
			   ORDER BY (text IS NULL) DESC, synced_at IS NULL DESC LIMIT ?`
			: "SELECT memory_id FROM records LIMIT ?";
		const ids = db.prepare(sql).all(opts?.limit ?? 1000).map((r: any) => r.memory_id as string);
		const pace = opts?.paceMs ?? 600; // ~100 richieste/min (< rate limit 120/min)
		// ogni GET è limitata a 8s (o al timeout configurato se più basso): evita di
		// restare appesi decine di secondi per record quando il gateway è giù
		const perRequestMs = Math.min(cfg.timeoutMs ?? 30_000, 8000);
		for (const id of ids) {
			stats.requested++;
			let res: { ok: boolean; status: number; data: any };
			try {
				res = await gatewayRequest(cfg, "GET", `/v1/memories/${id}`, undefined, AbortSignal.timeout(perRequestMs));
			} catch (e) {
				// timeout/abort: gateway non raggiungibile → inutile insistere sui record successivi
				stats.failed++;
				stats.errors.push(`${id}: ${e instanceof Error ? e.message : String(e)} (timeout ${perRequestMs}ms)`);
				break;
			}
			const { ok, status, data } = res;
			if (!ok || !data?.memory_id) {
				stats.failed++;
				if (stats.errors.length < 3) stats.errors.push(`${id}: HTTP ${status}`);
				// 0 (rete), 401 (chiave) o 5xx (nodo/upstream giù): il gateway non è utilizzabile
				if (status === 0 || status === 401 || status >= 500) break;
				continue;
			}
			stats.ok++;
			upsertRecords(db, [
				{
					memory_id: data.memory_id,
					text: data.text ?? null,
					kind: data.kind ?? null,
					project_id: data.project_id ?? null,
					scope: data.scope ?? null,
					agent_id: data.agent_id ?? null,
					confidence: data.confidence ?? null,
					importance: typeof data.importance === "number" ? data.importance : null,
					created_at: data.created_at ?? null,
					supersedes_id: data.supersedes_id ?? null,
					superseded_by: data.superseded_by ?? null,
					parent_id: data.parent_id ?? null,
					topic: data.topic ?? null,
					level: data.level ?? null,
					private: data.private ?? null,
					source: "enrich",
				},
			]);
			stats.updated++;
			if (pace > 0) await new Promise((r) => setTimeout(r, pace));
		}
		db.prepare("INSERT INTO meta(k,v) VALUES('last_enrich',?) ON CONFLICT(k) DO UPDATE SET v=excluded.v").run(new Date().toISOString());
		return stats;
	} finally {
		db.close();
	}
}

/**
 * Pull incrementale dal gateway (richiede `GET /v1/memories:export` sul
 * gateway: endpoint previsto ma non ancora deployato). Se assente, ritorna
 * `supported: false` senza errore.
 */
export async function pullFromGatewayExport(
	cfg: MemoryConfig,
	opts?: { dbFile?: string; limit?: number; maxPages?: number },
): Promise<{ supported: boolean; pages: number; fetched: number; message?: string }> {
	const db = await openLocalDb(opts?.dbFile);
	try {
		let cursor = db.prepare("SELECT v FROM meta WHERE k='export_cursor'").get()?.v ?? "";
		const limit = opts?.limit ?? 500;
		const maxPages = opts?.maxPages ?? 20;
		let pages = 0;
		let fetched = 0;
		for (; pages < maxPages; pages++) {
			const route = `/v1/memories:export?limit=${limit}${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ""}`;
			const { ok, status, data } = await gatewayRequest(
				cfg,
				"GET",
				route,
				undefined,
				AbortSignal.timeout(Math.min(cfg.timeoutMs ?? 30_000, 10_000)),
			);
			if (status === 404 || status === 405 || status === 400) {
				return { supported: false, pages, fetched, message: `endpoint di export non disponibile sul gateway (HTTP ${status})` };
			}
			if (!ok) return { supported: false, pages, fetched, message: `export fallito: HTTP ${status}` };
			const items = data?.results ?? data?.records ?? [];
			if (Array.isArray(items) && items.length) {
				upsertRecords(db, items.map((r: any) => ({ ...r, source: "export" })));
				fetched += items.length;
			}
			cursor = data?.next_cursor ?? "";
			if (!cursor || !items.length) break;
			db.prepare("INSERT INTO meta(k,v) VALUES('export_cursor',?) ON CONFLICT(k) DO UPDATE SET v=excluded.v").run(String(cursor));
		}
		db.prepare("INSERT INTO meta(k,v) VALUES('last_export',?) ON CONFLICT(k) DO UPDATE SET v=excluded.v").run(new Date().toISOString());
		return { supported: true, pages, fetched };
	} finally {
		db.close();
	}
}

// ---------------------------------------------------------------------------
// Coda offline (outbox): store quando il gateway non è raggiungibile
// ---------------------------------------------------------------------------
// Il record viene scritto subito nell'indice locale (marcato `pending`) e in
// `pending` come payload JSON. Al ritorno della connessione il flush lo invia a
// POST /v1/memories con `Idempotency-Key` = local_id (il gateway deduplica i
// retry), poi il record locale **adotta l'ID remoto** (niente duplicati).
// 409 (duplicato noto) → status `duplicate` con l'id del match; 4xx di
// validazione → `failed` (non ritentato); 0/429/5xx → resta `queued` e il flush
// si ferma.
export interface QueuePayload {
	text: string;
	kind?: string;
	project_id: string;
	scope?: string;
	agent_id?: string;
	source?: string;
	confidence?: string;
	expires_at?: string;
	supersedes_id?: string;
	supersede_reason?: string;
	parent_id?: string;
	level?: string;
	topic?: string;
	links?: unknown;
	importance?: number;
	private?: boolean;
}

export interface QueueItem {
	local_id: string;
	payload: QueuePayload;
	created_at: string;
	attempts: number;
	last_attempt: string | null;
	last_error: string | null;
	status: "queued" | "synced" | "duplicate" | "failed";
	remote_id: string | null;
}

export interface QueueStats {
	queued: number;
	synced: number;
	duplicate: number;
	failed: number;
	oldestQueued: string | null;
	lastError: string | null;
}

export interface FlushStats {
	processed: number;
	synced: number;
	duplicates: number;
	failed: number;
	remaining: number;
	stopped?: string;
	remoteIds: Record<string, string>;
	errors: string[];
}

const nowIso = () => new Date().toISOString();
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));

function stripEmpty(obj: Record<string, unknown>): Record<string, unknown> {
	const out: Record<string, unknown> = {};
	for (const [k, v] of Object.entries(obj)) if (v !== undefined && v !== null && v !== "") out[k] = v;
	return out;
}

/** Accoda un record (crea anche il placeholder locale ricercabile). */
export async function queueStore(
	payload: QueuePayload,
	opts?: { dbFile?: string },
): Promise<{ local_id: string; queue_size: number }> {
	const db = await openLocalDb(opts?.dbFile);
	try {
		const localId = randomUUID();
		const at = nowIso();
		db.prepare("INSERT INTO pending(local_id,payload,created_at,status) VALUES(?,?,?,'queued')").run(
			localId,
			JSON.stringify(stripEmpty(payload as unknown as Record<string, unknown>)),
			at,
		);
		upsertRecords(db, [
			{
				memory_id: localId,
				text: payload.text,
				kind: payload.kind ?? "fact",
				project_id: payload.project_id,
				scope: payload.scope ?? "agent",
				agent_id: payload.agent_id ?? null,
				confidence: payload.confidence ?? null,
				importance: typeof payload.importance === "number" ? payload.importance : null,
				created_at: at,
				supersedes_id: payload.supersedes_id ?? null,
				parent_id: payload.parent_id ?? null,
				topic: payload.topic ?? null,
				level: payload.level ?? null,
				private: payload.private ?? null,
				pending: true,
				source: "outbox",
			},
		]);
		const n = db.prepare("SELECT COUNT(*) n FROM pending WHERE status='queued'").get()?.n ?? 0;
		return { local_id: localId, queue_size: n };
	} finally {
		db.close();
	}
}

export async function queueList(opts?: { dbFile?: string; status?: string; limit?: number }): Promise<QueueItem[]> {
	const db = await openLocalDb(opts?.dbFile);
	try {
		const where = opts?.status ? "WHERE status = ?" : "";
		const args: any[] = opts?.status ? [opts.status] : [];
		args.push(opts?.limit ?? 50);
		const rows = db
			.prepare(`SELECT local_id, payload, created_at, attempts, last_attempt, last_error, status, remote_id FROM pending ${where} ORDER BY id DESC LIMIT ?`)
			.all(...args) as any[];
		return rows.map((r) => ({ ...r, payload: JSON.parse(r.payload) }));
	} finally {
		db.close();
	}
}

export async function queueStats(opts?: { dbFile?: string }): Promise<QueueStats> {
	const db = await openLocalDb(opts?.dbFile);
	try {
		const count = (s: string) => db.prepare("SELECT COUNT(*) n FROM pending WHERE status=?").get(s)?.n ?? 0;
		const oldest = db.prepare("SELECT created_at FROM pending WHERE status='queued' ORDER BY id LIMIT 1").get()?.created_at ?? null;
		const lastError = db.prepare("SELECT last_error FROM pending WHERE last_error IS NOT NULL ORDER BY id DESC LIMIT 1").get()?.last_error ?? null;
		return { queued: count("queued"), synced: count("synced"), duplicate: count("duplicate"), failed: count("failed"), oldestQueued: oldest, lastError };
	} finally {
		db.close();
	}
}

/** Il record locale "diventa" quello remoto: adotta l'ID del gateway. */
function adoptRemoteId(db: Db, localId: string, remoteId: string, payload: QueuePayload, data: any): void {
	db.prepare("DELETE FROM records WHERE memory_id = ?").run(localId);
	upsertRecords(db, [
		{
			memory_id: remoteId,
			text: data?.text ?? payload.text,
			kind: data?.kind ?? payload.kind ?? "fact",
			project_id: data?.project_id ?? payload.project_id,
			scope: data?.scope ?? payload.scope ?? "agent",
			agent_id: data?.agent_id ?? payload.agent_id ?? null,
			confidence: data?.confidence ?? payload.confidence ?? null,
			importance: typeof data?.importance === "number" ? data.importance : (payload.importance ?? null),
			created_at: data?.created_at ?? null,
			supersedes_id: data?.supersedes_id ?? payload.supersedes_id ?? null,
			superseded_by: data?.superseded_by ?? null,
			parent_id: data?.parent_id ?? payload.parent_id ?? null,
			topic: data?.topic ?? payload.topic ?? null,
			level: data?.level ?? payload.level ?? null,
			private: data?.private ?? payload.private ?? null,
			pending: false,
			source: "sync",
		},
	]);
}

/** Upload dei record in coda verso il gateway (idempotente, FIFO). */
export async function flushQueue(
	cfg: MemoryConfig,
	opts?: { dbFile?: string; limit?: number; paceMs?: number },
): Promise<FlushStats> {
	const db = await openLocalDb(opts?.dbFile);
	const stats: FlushStats = { processed: 0, synced: 0, duplicates: 0, failed: 0, remaining: 0, remoteIds: {}, errors: [] };
	try {
		const items = db
			.prepare("SELECT local_id, payload FROM pending WHERE status='queued' ORDER BY id LIMIT ?")
			.all(opts?.limit ?? 100) as Array<{ local_id: string; payload: string }>;
		const pace = opts?.paceMs ?? 300; // < rate limit del gateway (120/min)
		const perRequestMs = Math.min(cfg.timeoutMs ?? 30_000, 12_000);
		for (const item of items) {
			stats.processed++;
			const payload: QueuePayload = JSON.parse(item.payload);
			// supersede verso un record creato offline: rimappa local_id → remote_id
			if (payload.supersedes_id) {
				const parent = db.prepare("SELECT status, remote_id FROM pending WHERE local_id = ?").get(payload.supersedes_id) as any;
				if (parent) {
					if ((parent.status === "synced" || parent.status === "duplicate") && parent.remote_id) {
						payload.supersedes_id = parent.remote_id;
					} else {
						const msg = `supersede di un record non ancora sincronizzato (${String(payload.supersedes_id).slice(0, 8)})`;
						db.prepare("UPDATE pending SET status='failed', attempts=attempts+1, last_attempt=?, last_error=? WHERE local_id=?").run(nowIso(), msg, item.local_id);
						stats.failed++;
						stats.errors.push(`${item.local_id.slice(0, 8)}: ${msg}`);
						continue;
					}
				}
			}
			const body = stripEmpty(payload as unknown as Record<string, unknown>);
			let res: { ok: boolean; status: number; data: any };
			try {
				res = await gatewayRequest(
					cfg,
					"POST",
					"/v1/memories",
					body,
					AbortSignal.timeout(perRequestMs),
					item.local_id, // Idempotency-Key: retry sicuri, nessun duplicato
				);
			} catch (e) {
				const msg = `${e instanceof Error ? e.message : String(e)} (timeout ${perRequestMs}ms)`;
				db.prepare("UPDATE pending SET attempts=attempts+1, last_attempt=?, last_error=? WHERE local_id=?").run(nowIso(), msg, item.local_id);
				stats.errors.push(`${item.local_id.slice(0, 8)}: ${msg}`);
				stats.stopped = msg;
				break;
			}
			if (res.ok && res.data?.memory_id) {
				db.prepare("UPDATE pending SET status='synced', remote_id=?, synced_at=?, attempts=attempts+1, last_attempt=?, last_error=NULL WHERE local_id=?").run(
					res.data.memory_id,
					nowIso(),
					nowIso(),
					item.local_id,
				);
				adoptRemoteId(db, item.local_id, res.data.memory_id, payload, res.data);
				stats.synced++;
				stats.remoteIds[item.local_id] = res.data.memory_id;
			} else if (res.status === 409) {
				const detail = res.data?.detail ?? res.data ?? {};
				const remoteId: string | null = detail?.matches?.[0]?.memory_id ?? null;
				db.prepare("UPDATE pending SET status='duplicate', remote_id=?, attempts=attempts+1, last_attempt=?, last_error=? WHERE local_id=?").run(
					remoteId,
					nowIso(),
					`409 ${detail?.reason ?? "duplicate_memory"}`,
					item.local_id,
				);
				if (remoteId) {
					// Il record è già sul gateway: NON sovrascrivere il testo locale
					// autorevole con il payload appena inviato — si elimina solo il
					// placeholder locale e, se il record remoto non è ancora in indice,
					// lo si aggiunge con il testo appena scritto.
					const known = db.prepare("SELECT memory_id FROM records WHERE memory_id = ?").get(remoteId);
					db.prepare("DELETE FROM records WHERE memory_id = ?").run(item.local_id);
					if (!known) {
						upsertRecords(db, [
							{
								memory_id: remoteId,
								text: payload.text,
								kind: payload.kind ?? "fact",
								project_id: payload.project_id,
								scope: payload.scope ?? "agent",
								agent_id: payload.agent_id ?? null,
								created_at: null,
								topic: payload.topic ?? null,
								level: payload.level ?? null,
								private: payload.private ?? null,
								pending: false,
								source: "sync",
							},
						]);
					}
				} else {
					db.prepare("UPDATE records SET pending=0 WHERE memory_id=?").run(item.local_id);
				}
				stats.duplicates++;
			} else if (res.status >= 400 && res.status < 500) {
				// errore permanente (422 validazione, 400, 401): non ritentare in automatico
				const msg = `${res.status}: ${JSON.stringify(res.data).slice(0, 300)}`;
				db.prepare("UPDATE pending SET status='failed', attempts=attempts+1, last_attempt=?, last_error=? WHERE local_id=?").run(nowIso(), msg, item.local_id);
				stats.failed++;
				stats.errors.push(`${item.local_id.slice(0, 8)}: HTTP ${res.status}`);
				if (res.status === 401) {
					stats.stopped = "chiave API non valida (401)";
					break;
				}
			} else {
				// 0/429/5xx: gateway non utilizzabile, resta in coda
				const msg = `HTTP ${res.status}`;
				db.prepare("UPDATE pending SET attempts=attempts+1, last_attempt=?, last_error=? WHERE local_id=?").run(nowIso(), msg, item.local_id);
				stats.errors.push(`${item.local_id.slice(0, 8)}: ${msg}`);
				stats.stopped = msg;
				break;
			}
			if (pace > 0) await sleep(pace);
		}
		stats.remaining = db.prepare("SELECT COUNT(*) n FROM pending WHERE status='queued'").get()?.n ?? 0;
		db.prepare("INSERT INTO meta(k,v) VALUES('last_flush',?) ON CONFLICT(k) DO UPDATE SET v=excluded.v").run(nowIso());
		return stats;
	} finally {
		db.close();
	}
}

/**
 * Store con fallback offline: prova il gateway, e se non è raggiungibile accoda
 * il record localmente (default `offlineQueue: true`).
 */
export async function submitOrQueue(
	cfg: MemoryConfig,
	payload: QueuePayload,
	opts?: { dbFile?: string; queue?: boolean },
): Promise<{ queued: boolean; remote_id?: string; local_id?: string; queue_size?: number; status: number; data?: any }> {
	const queueAllowed = opts?.queue ?? cfg.offlineQueue !== false;
	const body = stripEmpty(payload as unknown as Record<string, unknown>);
	const perRequestMs = Math.min(cfg.timeoutMs ?? 30_000, 15_000);
	const res = await gatewayRequest(cfg, "POST", "/v1/memories", body, AbortSignal.timeout(perRequestMs), randomUUID());
	if (res.ok && res.data?.memory_id) {
		// indicizza subito in locale (senza attendere enrich/export)
		try {
			const db = await openLocalDb(opts?.dbFile);
			upsertRecords(db, [
				{
					memory_id: res.data.memory_id,
					text: res.data.text ?? payload.text,
					kind: res.data.kind ?? payload.kind ?? "fact",
					project_id: res.data.project_id ?? payload.project_id,
					scope: res.data.scope ?? payload.scope ?? "agent",
					agent_id: res.data.agent_id ?? payload.agent_id ?? null,
					confidence: res.data.confidence ?? payload.confidence ?? null,
					importance: typeof res.data.importance === "number" ? res.data.importance : (payload.importance ?? null),
					created_at: res.data.created_at ?? null,
					supersedes_id: res.data.supersedes_id ?? payload.supersedes_id ?? null,
					parent_id: payload.parent_id ?? null,
					topic: payload.topic ?? null,
					level: payload.level ?? null,
					private: payload.private ?? null,
					pending: false,
					source: "store",
				},
			]);
			db.close();
		} catch {
			/* l'indice locale è best-effort sul percorso online */
		}
		maybeBackgroundFlush(cfg, opts?.dbFile);
		return { queued: false, remote_id: res.data.memory_id, status: res.status, data: res.data };
	}
	const down = res.status === 0 || res.status >= 500 || res.status === 429;
	if (down && queueAllowed) {
		const q = await queueStore(payload, { dbFile: opts?.dbFile });
		return { queued: true, local_id: q.local_id, queue_size: q.queue_size, status: res.status, data: res.data };
	}
	return { queued: false, status: res.status, data: res.data };
}

let flushInFlight: Promise<unknown> | null = null;

/** Flush in background (single-flight): usato dopo operazioni riuscite e su session_start. */
export function maybeBackgroundFlush(cfg: MemoryConfig, dbFile?: string): void {
	if (flushInFlight) return;
	flushInFlight = flushQueue(cfg, { dbFile, limit: 20, paceMs: 400 })
		.catch(() => undefined)
		.finally(() => {
			flushInFlight = null;
		});
}

export async function flushQueueIfPending(cfg: MemoryConfig, dbFile?: string): Promise<FlushStats | null> {
	const stats = await queueStats({ dbFile });
	if (!stats.queued) return null;
	return flushQueue(cfg, { dbFile, limit: 50, paceMs: 300 });
}

// ---------------------------------------------------------------------------
// Report / stato
// ---------------------------------------------------------------------------
export interface LocalDbReport {
	path: string;
	exists: boolean;
	sizeKb: number;
	total: number;
	withText: number;
	active: number;
	superseded: number;
	private: number;
	withProject: number;
	lastImport?: string;
	lastEnrich?: string;
	lastExport?: string;
	lastFlush?: string;
	queued: number;
	syncedQueue: number;
	duplicateQueue: number;
	failedQueue: number;
	pendingInIndex: number;
	oldestQueued?: string | null;
	queueLastError?: string | null;
	topProjects: Array<{ project_id: string | null; n: number }>;
	duplicates: Array<{ text_hash: string; n: number; ids: string[] }>;
}

export async function localDbReport(opts?: { dbFile?: string }): Promise<LocalDbReport> {
	const file = opts?.dbFile ?? localDbPath(loadConfig());
	const exists = fs.existsSync(file);
	const report: LocalDbReport = {
		path: file,
		exists,
		sizeKb: 0,
		total: 0,
		withText: 0,
		active: 0,
		superseded: 0,
		private: 0,
		withProject: 0,
		queued: 0,
		syncedQueue: 0,
		duplicateQueue: 0,
		failedQueue: 0,
		pendingInIndex: 0,
		topProjects: [],
		duplicates: [],
	};
	if (!exists) return report;
	for (const f of [file, `${file}-wal`, `${file}-shm`]) {
		try {
			report.sizeKb += Math.round(fs.statSync(f).size / 1024);
		} catch {
			/* assente */
		}
	}
	const db = await openLocalDb(file);
	try {
		const one = (sql: string) => db.prepare(sql).get()?.n ?? 0;
		report.total = one("SELECT COUNT(*) n FROM records");
		report.withText = one("SELECT COUNT(*) n FROM records WHERE text IS NOT NULL");
		report.active = one("SELECT COUNT(*) n FROM records WHERE superseded_by IS NULL");
		report.superseded = one("SELECT COUNT(*) n FROM records WHERE superseded_by IS NOT NULL");
		report.private = one("SELECT COUNT(*) n FROM records WHERE private = 1");
		report.withProject = one("SELECT COUNT(*) n FROM records WHERE project_id IS NOT NULL");
		report.lastImport = db.prepare("SELECT v FROM meta WHERE k='last_import'").get()?.v;
		report.lastEnrich = db.prepare("SELECT v FROM meta WHERE k='last_enrich'").get()?.v;
		report.lastExport = db.prepare("SELECT v FROM meta WHERE k='last_export'").get()?.v;
		report.lastFlush = db.prepare("SELECT v FROM meta WHERE k='last_flush'").get()?.v;
		report.queued = one("SELECT COUNT(*) n FROM pending WHERE status='queued'");
		report.syncedQueue = one("SELECT COUNT(*) n FROM pending WHERE status='synced'");
		report.duplicateQueue = one("SELECT COUNT(*) n FROM pending WHERE status='duplicate'");
		report.failedQueue = one("SELECT COUNT(*) n FROM pending WHERE status='failed'");
		report.pendingInIndex = one("SELECT COUNT(*) n FROM records WHERE pending = 1");
		report.oldestQueued = db.prepare("SELECT created_at FROM pending WHERE status='queued' ORDER BY id LIMIT 1").get()?.created_at ?? null;
		report.queueLastError = db.prepare("SELECT last_error FROM pending WHERE last_error IS NOT NULL ORDER BY id DESC LIMIT 1").get()?.last_error ?? null;
		report.topProjects = db.prepare("SELECT project_id, COUNT(*) n FROM records GROUP BY project_id ORDER BY n DESC LIMIT 8").all() as any;
		report.duplicates = db
			.prepare("SELECT text_hash, COUNT(*) n, GROUP_CONCAT(memory_id) ids FROM records WHERE text_hash IS NOT NULL GROUP BY text_hash HAVING n > 1 ORDER BY n DESC LIMIT 20")
			.all()
			.map((r: any) => ({ text_hash: r.text_hash, n: r.n, ids: String(r.ids).split(",") }));
		return report;
	} finally {
		db.close();
	}
}