feat(outbox): store offline con coda locale e sincronizzazione al ritorno della rete
Prima qmem_store falliva se il gateway non era raggiungibile: la conoscenza andava persa. Ora il record entra in una coda locale persistente e viene inviato automaticamente quando la connessione torna. Core (extensions/local-db.ts): - tabella `pending` (local_id, payload JSON, attempts, last_error, status, remote_id) + colonna `records.pending` (migrazione automatica dei DB esistenti) - queueStore(): accoda e crea subito il placeholder locale ricercabile (⏳) - flushQueue(): POST /v1/memories con Idempotency-Key = local_id (retry senza duplicati), FIFO, pacing sotto il rate limit, timeout 12s per richiesta - esiti: synced (il record locale adotta l'ID remoto, niente duplicati) · duplicate (409: registra l'ID del match e NON sovrascrive il testo locale autorevole) · failed (4xx di validazione, non ritentato) · 0/429/5xx: resta in coda e il flush si ferma - supersede offline: supersedes_id che punta a un local_id viene rimappato al remote_id al flush (se il genitore non è sincronizzato → failed esplicito) - submitOrQueue(): online → gateway + indicizzazione locale; offline → coda - maybeBackgroundFlush() (single-flight) e flushQueueIfPending() per session_start - stato/report: queued/synced/duplicate/failed, più vecchio, ultimo errore, last_flush, record pendenti in indice Estensione: - qmem_store: gateway giù → accoda e risponde con id locale, dimensione coda e spiegazione (details.queued/local_id/queue_size) - fallback offline di session_start: flush in background (non blocca l'avvio) - /qmem:local queue|flush; status con la coda; marker "⏳ in coda" nei risultati locali di qmem_search/qmem_get - regole e skill: un record in coda NON è ancora nella memoria condivisa CLI: store [--queue-only], queue, flush (+ status con la coda). Test: scripts/test-local.mjs ora copre anche outbox → 24 controlli (flush con 2 sync + 1 duplicato 409 + 1 fallito 422, Idempotency-Key, rimappatura del supersede, ricerca del record con l'ID remoto dopo il sync). Verifiche: 24/24 test superati; demo reale su DB temporaneo: store accodato, queue con local_id, flush con gateway giù → "fermato: HTTP 0" e voce che resta in coda con l'errore registrato.
This commit is contained in:
Matteo Benedetto
+440
-12
@@ -24,7 +24,7 @@
|
||||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { gatewayRequest, loadConfig, type MemoryConfig } from "./shared.ts";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -90,7 +90,9 @@ CREATE TABLE IF NOT EXISTS records(
|
||||
text TEXT, kind TEXT, project_id TEXT, scope TEXT, agent_id TEXT, confidence TEXT,
|
||||
importance REAL, created_at TEXT, supersedes_id TEXT, superseded_by TEXT,
|
||||
parent_id TEXT, topic TEXT, level TEXT, private INTEGER NOT NULL DEFAULT 0,
|
||||
text_hash TEXT, sources TEXT, synced_at TEXT, updated_at TEXT
|
||||
text_hash TEXT, sources TEXT, synced_at TEXT, updated_at TEXT,
|
||||
/** 1 = record creato offline, non ancora presente sul gateway */
|
||||
pending INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS records_project ON records(project_id);
|
||||
CREATE INDEX IF NOT EXISTS records_kind ON records(kind);
|
||||
@@ -110,7 +112,26 @@ CREATE TRIGGER IF NOT EXISTS records_ad AFTER DELETE ON records BEGIN
|
||||
INSERT INTO records_fts(records_fts, rowid, text, kind, project_id) VALUES('delete', old.rowid_, old.text, old.kind, old.project_id);
|
||||
END;
|
||||
CREATE TABLE IF NOT EXISTS meta(k TEXT PRIMARY KEY, v TEXT);
|
||||
CREATE TABLE IF NOT EXISTS pending(
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
local_id TEXT UNIQUE NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
last_attempt TEXT,
|
||||
last_error TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'queued', -- queued | synced | duplicate | failed
|
||||
remote_id TEXT,
|
||||
synced_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS pending_status ON pending(status);
|
||||
`);
|
||||
// migrazione per DB creati prima dell'introduzione della coda
|
||||
const cols = db
|
||||
.prepare("PRAGMA table_info(records)")
|
||||
.all()
|
||||
.map((c: any) => c.name);
|
||||
if (!cols.includes("pending")) db.exec("ALTER TABLE records ADD COLUMN pending INTEGER NOT NULL DEFAULT 0");
|
||||
return db;
|
||||
}
|
||||
|
||||
@@ -133,12 +154,14 @@ export interface LocalRecordInput {
|
||||
topic?: string | null;
|
||||
level?: string | null;
|
||||
private?: boolean | null;
|
||||
/** Fonte dell'osservazione: store | correct | get | search | export | enrich */
|
||||
/** true = creato offline (in coda), false = presente sul gateway */
|
||||
pending?: boolean | null;
|
||||
/** Fonte dell'osservazione: store | correct | get | search | export | enrich | outbox | sync */
|
||||
source: string;
|
||||
}
|
||||
|
||||
/** Sorgenti ordinate per affidabilità del testo (maggiore = più completo). */
|
||||
const TEXT_PRIORITY: Record<string, number> = { export: 5, enrich: 5, get: 4, store: 3, correct: 3, search: 2 };
|
||||
const TEXT_PRIORITY: Record<string, number> = { export: 5, enrich: 5, sync: 5, get: 4, store: 3, correct: 3, outbox: 3, search: 2 };
|
||||
|
||||
function normalizeText(t: string): string {
|
||||
return t.replace(/\s+/g, " ").trim().toLowerCase();
|
||||
@@ -155,11 +178,11 @@ export function upsertRecords(db: Db, records: LocalRecordInput[]): number {
|
||||
const find = db.prepare("SELECT * FROM records WHERE memory_id = ?");
|
||||
const insert = db.prepare(`INSERT INTO records
|
||||
(memory_id, text, kind, project_id, scope, agent_id, confidence, importance, created_at,
|
||||
supersedes_id, superseded_by, parent_id, topic, level, private, text_hash, sources, synced_at, updated_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`);
|
||||
supersedes_id, superseded_by, parent_id, topic, level, private, text_hash, sources, synced_at, updated_at, pending)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`);
|
||||
const update = db.prepare(`UPDATE records SET
|
||||
text=?, kind=?, project_id=?, scope=?, agent_id=?, confidence=?, importance=?, created_at=?,
|
||||
supersedes_id=?, superseded_by=?, parent_id=?, topic=?, level=?, private=?, text_hash=?, sources=?, synced_at=?, updated_at=?
|
||||
supersedes_id=?, superseded_by=?, parent_id=?, topic=?, level=?, private=?, text_hash=?, sources=?, synced_at=?, updated_at=?, pending=?
|
||||
WHERE rowid_=?`);
|
||||
const now = new Date().toISOString();
|
||||
let written = 0;
|
||||
@@ -193,17 +216,19 @@ export function upsertRecords(db: Db, records: LocalRecordInput[]): number {
|
||||
[...sources].sort().join(","),
|
||||
now,
|
||||
now,
|
||||
r.pending === true ? 1 : r.pending === false ? 0 : (existing?.pending ?? 0),
|
||||
];
|
||||
if (existing) {
|
||||
// I campi già noti non vengono mai azzerati da osservazioni più povere
|
||||
// (es. un risultato di ricerca che non riporta project_id).
|
||||
const keys = ["text", "kind", "project_id", "scope", "agent_id", "confidence", "importance", "created_at",
|
||||
"supersedes_id", "superseded_by", "parent_id", "topic", "level", "private", "text_hash", "sources", "synced_at", "updated_at"];
|
||||
"supersedes_id", "superseded_by", "parent_id", "topic", "level", "private", "text_hash", "sources", "synced_at", "updated_at", "pending"];
|
||||
const merged = vals.map((v, i) => {
|
||||
const key = keys[i];
|
||||
const cur = (existing as any)[key];
|
||||
const curEmpty = cur === null || cur === undefined || cur === "" || cur === 0;
|
||||
if (key === "private") return cur === 1 ? 1 : v; // monotono: una volta privato resta privato
|
||||
if (key === "pending") return v; // 0 = sincronizzato: prevale sullo stato locale
|
||||
if (key === "sources" || key === "synced_at" || key === "updated_at") return v;
|
||||
if (v === null || v === "" ) return curEmpty ? v : cur; // non azzerare
|
||||
return v;
|
||||
@@ -422,6 +447,8 @@ export interface LocalSearchHit {
|
||||
created_at: string | null;
|
||||
superseded_by: string | null;
|
||||
sources: string | null;
|
||||
/** 1 = creato offline, non ancora sul gateway */
|
||||
pending?: number;
|
||||
snippet: string;
|
||||
rank: number;
|
||||
/** "and" = tutti i termini presenti; "or" = match parziale (AND senza risultati) */
|
||||
@@ -466,7 +493,7 @@ async function runSearch(db: Db, params: LocalSearchParams, match: string): Prom
|
||||
if (!params.include_private) where.push("r.private = 0");
|
||||
args.push(Math.min(Math.max(params.top_k ?? 5, 1), 50));
|
||||
const sql = `SELECT r.memory_id, r.text, r.kind, r.project_id, r.scope, r.agent_id, r.created_at,
|
||||
r.superseded_by, r.sources, bm25(records_fts) AS rank,
|
||||
r.superseded_by, r.sources, r.pending, bm25(records_fts) AS rank,
|
||||
snippet(records_fts, 0, '«', '»', '…', 14) AS snippet
|
||||
FROM records_fts JOIN records r ON r.rowid_ = records_fts.rowid
|
||||
WHERE ${where.join(" AND ")} ORDER BY rank LIMIT ?`;
|
||||
@@ -495,17 +522,39 @@ export async function localSearch(params: LocalSearchParams, opts?: { dbFile?: s
|
||||
}
|
||||
|
||||
/** Recupero locale di un singolo record (fallback di qmem_get). */
|
||||
export async function localGet(memoryId: string, opts?: { dbFile?: string }): Promise<LocalSearchHit | null> {
|
||||
export async function localGet(memoryId: string, opts?: { dbFile?: string }): Promise<(LocalSearchHit & { queue_status?: string; remote_id?: string | null }) | null> {
|
||||
const db = await openLocalDb(opts?.dbFile);
|
||||
try {
|
||||
const row = db
|
||||
.prepare(
|
||||
`SELECT memory_id, text, kind, project_id, scope, agent_id, created_at, superseded_by, sources, 0 AS rank,
|
||||
`SELECT memory_id, text, kind, project_id, scope, agent_id, created_at, superseded_by, sources, pending, 0 AS rank,
|
||||
COALESCE(substr(text,1,200),'') AS snippet
|
||||
FROM records WHERE memory_id = ?`,
|
||||
)
|
||||
.get(memoryId) as LocalSearchHit | undefined;
|
||||
return row ?? null;
|
||||
if (row) return row;
|
||||
// non è nell'indice: potrebbe essere un id locale della coda
|
||||
const q = db
|
||||
.prepare("SELECT local_id, payload, status, remote_id FROM pending WHERE local_id = ?")
|
||||
.get(memoryId) as any;
|
||||
if (!q) return null;
|
||||
const payload: QueuePayload = JSON.parse(q.payload);
|
||||
return {
|
||||
memory_id: q.local_id,
|
||||
text: payload.text,
|
||||
kind: payload.kind ?? "fact",
|
||||
project_id: payload.project_id,
|
||||
scope: payload.scope ?? "agent",
|
||||
agent_id: payload.agent_id ?? null,
|
||||
created_at: null,
|
||||
superseded_by: null,
|
||||
sources: "outbox",
|
||||
pending: q.status === "queued" ? 1 : 0,
|
||||
snippet: String(payload.text ?? "").slice(0, 200),
|
||||
rank: 0,
|
||||
queue_status: q.status,
|
||||
remote_id: q.remote_id ?? null,
|
||||
};
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
@@ -641,6 +690,364 @@ export async function pullFromGatewayExport(
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Coda offline (outbox): store quando il gateway non è raggiungibile
|
||||
// ---------------------------------------------------------------------------
|
||||
// Il record viene scritto subito nell'indice locale (marcato `pending`) e in
|
||||
// `pending` come payload JSON. Al ritorno della connessione il flush lo invia a
|
||||
// POST /v1/memories con `Idempotency-Key` = local_id (il gateway deduplica i
|
||||
// retry), poi il record locale **adotta l'ID remoto** (niente duplicati).
|
||||
// 409 (duplicato noto) → status `duplicate` con l'id del match; 4xx di
|
||||
// validazione → `failed` (non ritentato); 0/429/5xx → resta `queued` e il flush
|
||||
// si ferma.
|
||||
export interface QueuePayload {
|
||||
text: string;
|
||||
kind?: string;
|
||||
project_id: string;
|
||||
scope?: string;
|
||||
agent_id?: string;
|
||||
source?: string;
|
||||
confidence?: string;
|
||||
expires_at?: string;
|
||||
supersedes_id?: string;
|
||||
supersede_reason?: string;
|
||||
parent_id?: string;
|
||||
level?: string;
|
||||
topic?: string;
|
||||
links?: unknown;
|
||||
importance?: number;
|
||||
private?: boolean;
|
||||
}
|
||||
|
||||
export interface QueueItem {
|
||||
local_id: string;
|
||||
payload: QueuePayload;
|
||||
created_at: string;
|
||||
attempts: number;
|
||||
last_attempt: string | null;
|
||||
last_error: string | null;
|
||||
status: "queued" | "synced" | "duplicate" | "failed";
|
||||
remote_id: string | null;
|
||||
}
|
||||
|
||||
export interface QueueStats {
|
||||
queued: number;
|
||||
synced: number;
|
||||
duplicate: number;
|
||||
failed: number;
|
||||
oldestQueued: string | null;
|
||||
lastError: string | null;
|
||||
}
|
||||
|
||||
export interface FlushStats {
|
||||
processed: number;
|
||||
synced: number;
|
||||
duplicates: number;
|
||||
failed: number;
|
||||
remaining: number;
|
||||
stopped?: string;
|
||||
remoteIds: Record<string, string>;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
const nowIso = () => new Date().toISOString();
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
function stripEmpty(obj: Record<string, unknown>): Record<string, unknown> {
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [k, v] of Object.entries(obj)) if (v !== undefined && v !== null && v !== "") out[k] = v;
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Accoda un record (crea anche il placeholder locale ricercabile). */
|
||||
export async function queueStore(
|
||||
payload: QueuePayload,
|
||||
opts?: { dbFile?: string },
|
||||
): Promise<{ local_id: string; queue_size: number }> {
|
||||
const db = await openLocalDb(opts?.dbFile);
|
||||
try {
|
||||
const localId = randomUUID();
|
||||
const at = nowIso();
|
||||
db.prepare("INSERT INTO pending(local_id,payload,created_at,status) VALUES(?,?,?,'queued')").run(
|
||||
localId,
|
||||
JSON.stringify(stripEmpty(payload as unknown as Record<string, unknown>)),
|
||||
at,
|
||||
);
|
||||
upsertRecords(db, [
|
||||
{
|
||||
memory_id: localId,
|
||||
text: payload.text,
|
||||
kind: payload.kind ?? "fact",
|
||||
project_id: payload.project_id,
|
||||
scope: payload.scope ?? "agent",
|
||||
agent_id: payload.agent_id ?? null,
|
||||
confidence: payload.confidence ?? null,
|
||||
importance: typeof payload.importance === "number" ? payload.importance : null,
|
||||
created_at: at,
|
||||
supersedes_id: payload.supersedes_id ?? null,
|
||||
parent_id: payload.parent_id ?? null,
|
||||
topic: payload.topic ?? null,
|
||||
level: payload.level ?? null,
|
||||
private: payload.private ?? null,
|
||||
pending: true,
|
||||
source: "outbox",
|
||||
},
|
||||
]);
|
||||
const n = db.prepare("SELECT COUNT(*) n FROM pending WHERE status='queued'").get()?.n ?? 0;
|
||||
return { local_id: localId, queue_size: n };
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function queueList(opts?: { dbFile?: string; status?: string; limit?: number }): Promise<QueueItem[]> {
|
||||
const db = await openLocalDb(opts?.dbFile);
|
||||
try {
|
||||
const where = opts?.status ? "WHERE status = ?" : "";
|
||||
const args: any[] = opts?.status ? [opts.status] : [];
|
||||
args.push(opts?.limit ?? 50);
|
||||
const rows = db
|
||||
.prepare(`SELECT local_id, payload, created_at, attempts, last_attempt, last_error, status, remote_id FROM pending ${where} ORDER BY id DESC LIMIT ?`)
|
||||
.all(...args) as any[];
|
||||
return rows.map((r) => ({ ...r, payload: JSON.parse(r.payload) }));
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function queueStats(opts?: { dbFile?: string }): Promise<QueueStats> {
|
||||
const db = await openLocalDb(opts?.dbFile);
|
||||
try {
|
||||
const count = (s: string) => db.prepare("SELECT COUNT(*) n FROM pending WHERE status=?").get(s)?.n ?? 0;
|
||||
const oldest = db.prepare("SELECT created_at FROM pending WHERE status='queued' ORDER BY id LIMIT 1").get()?.created_at ?? null;
|
||||
const lastError = db.prepare("SELECT last_error FROM pending WHERE last_error IS NOT NULL ORDER BY id DESC LIMIT 1").get()?.last_error ?? null;
|
||||
return { queued: count("queued"), synced: count("synced"), duplicate: count("duplicate"), failed: count("failed"), oldestQueued: oldest, lastError };
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** Il record locale "diventa" quello remoto: adotta l'ID del gateway. */
|
||||
function adoptRemoteId(db: Db, localId: string, remoteId: string, payload: QueuePayload, data: any): void {
|
||||
db.prepare("DELETE FROM records WHERE memory_id = ?").run(localId);
|
||||
upsertRecords(db, [
|
||||
{
|
||||
memory_id: remoteId,
|
||||
text: data?.text ?? payload.text,
|
||||
kind: data?.kind ?? payload.kind ?? "fact",
|
||||
project_id: data?.project_id ?? payload.project_id,
|
||||
scope: data?.scope ?? payload.scope ?? "agent",
|
||||
agent_id: data?.agent_id ?? payload.agent_id ?? null,
|
||||
confidence: data?.confidence ?? payload.confidence ?? null,
|
||||
importance: typeof data?.importance === "number" ? data.importance : (payload.importance ?? null),
|
||||
created_at: data?.created_at ?? null,
|
||||
supersedes_id: data?.supersedes_id ?? payload.supersedes_id ?? null,
|
||||
superseded_by: data?.superseded_by ?? null,
|
||||
parent_id: data?.parent_id ?? payload.parent_id ?? null,
|
||||
topic: data?.topic ?? payload.topic ?? null,
|
||||
level: data?.level ?? payload.level ?? null,
|
||||
private: data?.private ?? payload.private ?? null,
|
||||
pending: false,
|
||||
source: "sync",
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
/** Upload dei record in coda verso il gateway (idempotente, FIFO). */
|
||||
export async function flushQueue(
|
||||
cfg: MemoryConfig,
|
||||
opts?: { dbFile?: string; limit?: number; paceMs?: number },
|
||||
): Promise<FlushStats> {
|
||||
const db = await openLocalDb(opts?.dbFile);
|
||||
const stats: FlushStats = { processed: 0, synced: 0, duplicates: 0, failed: 0, remaining: 0, remoteIds: {}, errors: [] };
|
||||
try {
|
||||
const items = db
|
||||
.prepare("SELECT local_id, payload FROM pending WHERE status='queued' ORDER BY id LIMIT ?")
|
||||
.all(opts?.limit ?? 100) as Array<{ local_id: string; payload: string }>;
|
||||
const pace = opts?.paceMs ?? 300; // < rate limit del gateway (120/min)
|
||||
const perRequestMs = Math.min(cfg.timeoutMs ?? 30_000, 12_000);
|
||||
for (const item of items) {
|
||||
stats.processed++;
|
||||
const payload: QueuePayload = JSON.parse(item.payload);
|
||||
// supersede verso un record creato offline: rimappa local_id → remote_id
|
||||
if (payload.supersedes_id) {
|
||||
const parent = db.prepare("SELECT status, remote_id FROM pending WHERE local_id = ?").get(payload.supersedes_id) as any;
|
||||
if (parent) {
|
||||
if ((parent.status === "synced" || parent.status === "duplicate") && parent.remote_id) {
|
||||
payload.supersedes_id = parent.remote_id;
|
||||
} else {
|
||||
const msg = `supersede di un record non ancora sincronizzato (${String(payload.supersedes_id).slice(0, 8)})`;
|
||||
db.prepare("UPDATE pending SET status='failed', attempts=attempts+1, last_attempt=?, last_error=? WHERE local_id=?").run(nowIso(), msg, item.local_id);
|
||||
stats.failed++;
|
||||
stats.errors.push(`${item.local_id.slice(0, 8)}: ${msg}`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
const body = stripEmpty(payload as unknown as Record<string, unknown>);
|
||||
let res: { ok: boolean; status: number; data: any };
|
||||
try {
|
||||
res = await gatewayRequest(
|
||||
cfg,
|
||||
"POST",
|
||||
"/v1/memories",
|
||||
body,
|
||||
AbortSignal.timeout(perRequestMs),
|
||||
item.local_id, // Idempotency-Key: retry sicuri, nessun duplicato
|
||||
);
|
||||
} catch (e) {
|
||||
const msg = `${e instanceof Error ? e.message : String(e)} (timeout ${perRequestMs}ms)`;
|
||||
db.prepare("UPDATE pending SET attempts=attempts+1, last_attempt=?, last_error=? WHERE local_id=?").run(nowIso(), msg, item.local_id);
|
||||
stats.errors.push(`${item.local_id.slice(0, 8)}: ${msg}`);
|
||||
stats.stopped = msg;
|
||||
break;
|
||||
}
|
||||
if (res.ok && res.data?.memory_id) {
|
||||
db.prepare("UPDATE pending SET status='synced', remote_id=?, synced_at=?, attempts=attempts+1, last_attempt=?, last_error=NULL WHERE local_id=?").run(
|
||||
res.data.memory_id,
|
||||
nowIso(),
|
||||
nowIso(),
|
||||
item.local_id,
|
||||
);
|
||||
adoptRemoteId(db, item.local_id, res.data.memory_id, payload, res.data);
|
||||
stats.synced++;
|
||||
stats.remoteIds[item.local_id] = res.data.memory_id;
|
||||
} else if (res.status === 409) {
|
||||
const detail = res.data?.detail ?? res.data ?? {};
|
||||
const remoteId: string | null = detail?.matches?.[0]?.memory_id ?? null;
|
||||
db.prepare("UPDATE pending SET status='duplicate', remote_id=?, attempts=attempts+1, last_attempt=?, last_error=? WHERE local_id=?").run(
|
||||
remoteId,
|
||||
nowIso(),
|
||||
`409 ${detail?.reason ?? "duplicate_memory"}`,
|
||||
item.local_id,
|
||||
);
|
||||
if (remoteId) {
|
||||
// Il record è già sul gateway: NON sovrascrivere il testo locale
|
||||
// autorevole con il payload appena inviato — si elimina solo il
|
||||
// placeholder locale e, se il record remoto non è ancora in indice,
|
||||
// lo si aggiunge con il testo appena scritto.
|
||||
const known = db.prepare("SELECT memory_id FROM records WHERE memory_id = ?").get(remoteId);
|
||||
db.prepare("DELETE FROM records WHERE memory_id = ?").run(item.local_id);
|
||||
if (!known) {
|
||||
upsertRecords(db, [
|
||||
{
|
||||
memory_id: remoteId,
|
||||
text: payload.text,
|
||||
kind: payload.kind ?? "fact",
|
||||
project_id: payload.project_id,
|
||||
scope: payload.scope ?? "agent",
|
||||
agent_id: payload.agent_id ?? null,
|
||||
created_at: null,
|
||||
topic: payload.topic ?? null,
|
||||
level: payload.level ?? null,
|
||||
private: payload.private ?? null,
|
||||
pending: false,
|
||||
source: "sync",
|
||||
},
|
||||
]);
|
||||
}
|
||||
} else {
|
||||
db.prepare("UPDATE records SET pending=0 WHERE memory_id=?").run(item.local_id);
|
||||
}
|
||||
stats.duplicates++;
|
||||
} else if (res.status >= 400 && res.status < 500) {
|
||||
// errore permanente (422 validazione, 400, 401): non ritentare in automatico
|
||||
const msg = `${res.status}: ${JSON.stringify(res.data).slice(0, 300)}`;
|
||||
db.prepare("UPDATE pending SET status='failed', attempts=attempts+1, last_attempt=?, last_error=? WHERE local_id=?").run(nowIso(), msg, item.local_id);
|
||||
stats.failed++;
|
||||
stats.errors.push(`${item.local_id.slice(0, 8)}: HTTP ${res.status}`);
|
||||
if (res.status === 401) {
|
||||
stats.stopped = "chiave API non valida (401)";
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
// 0/429/5xx: gateway non utilizzabile, resta in coda
|
||||
const msg = `HTTP ${res.status}`;
|
||||
db.prepare("UPDATE pending SET attempts=attempts+1, last_attempt=?, last_error=? WHERE local_id=?").run(nowIso(), msg, item.local_id);
|
||||
stats.errors.push(`${item.local_id.slice(0, 8)}: ${msg}`);
|
||||
stats.stopped = msg;
|
||||
break;
|
||||
}
|
||||
if (pace > 0) await sleep(pace);
|
||||
}
|
||||
stats.remaining = db.prepare("SELECT COUNT(*) n FROM pending WHERE status='queued'").get()?.n ?? 0;
|
||||
db.prepare("INSERT INTO meta(k,v) VALUES('last_flush',?) ON CONFLICT(k) DO UPDATE SET v=excluded.v").run(nowIso());
|
||||
return stats;
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Store con fallback offline: prova il gateway, e se non è raggiungibile accoda
|
||||
* il record localmente (default `offlineQueue: true`).
|
||||
*/
|
||||
export async function submitOrQueue(
|
||||
cfg: MemoryConfig,
|
||||
payload: QueuePayload,
|
||||
opts?: { dbFile?: string; queue?: boolean },
|
||||
): Promise<{ queued: boolean; remote_id?: string; local_id?: string; queue_size?: number; status: number; data?: any }> {
|
||||
const queueAllowed = opts?.queue ?? cfg.offlineQueue !== false;
|
||||
const body = stripEmpty(payload as unknown as Record<string, unknown>);
|
||||
const perRequestMs = Math.min(cfg.timeoutMs ?? 30_000, 15_000);
|
||||
const res = await gatewayRequest(cfg, "POST", "/v1/memories", body, AbortSignal.timeout(perRequestMs), randomUUID());
|
||||
if (res.ok && res.data?.memory_id) {
|
||||
// indicizza subito in locale (senza attendere enrich/export)
|
||||
try {
|
||||
const db = await openLocalDb(opts?.dbFile);
|
||||
upsertRecords(db, [
|
||||
{
|
||||
memory_id: res.data.memory_id,
|
||||
text: res.data.text ?? payload.text,
|
||||
kind: res.data.kind ?? payload.kind ?? "fact",
|
||||
project_id: res.data.project_id ?? payload.project_id,
|
||||
scope: res.data.scope ?? payload.scope ?? "agent",
|
||||
agent_id: res.data.agent_id ?? payload.agent_id ?? null,
|
||||
confidence: res.data.confidence ?? payload.confidence ?? null,
|
||||
importance: typeof res.data.importance === "number" ? res.data.importance : (payload.importance ?? null),
|
||||
created_at: res.data.created_at ?? null,
|
||||
supersedes_id: res.data.supersedes_id ?? payload.supersedes_id ?? null,
|
||||
parent_id: payload.parent_id ?? null,
|
||||
topic: payload.topic ?? null,
|
||||
level: payload.level ?? null,
|
||||
private: payload.private ?? null,
|
||||
pending: false,
|
||||
source: "store",
|
||||
},
|
||||
]);
|
||||
db.close();
|
||||
} catch {
|
||||
/* l'indice locale è best-effort sul percorso online */
|
||||
}
|
||||
maybeBackgroundFlush(cfg, opts?.dbFile);
|
||||
return { queued: false, remote_id: res.data.memory_id, status: res.status, data: res.data };
|
||||
}
|
||||
const down = res.status === 0 || res.status >= 500 || res.status === 429;
|
||||
if (down && queueAllowed) {
|
||||
const q = await queueStore(payload, { dbFile: opts?.dbFile });
|
||||
return { queued: true, local_id: q.local_id, queue_size: q.queue_size, status: res.status, data: res.data };
|
||||
}
|
||||
return { queued: false, status: res.status, data: res.data };
|
||||
}
|
||||
|
||||
let flushInFlight: Promise<unknown> | null = null;
|
||||
|
||||
/** Flush in background (single-flight): usato dopo operazioni riuscite e su session_start. */
|
||||
export function maybeBackgroundFlush(cfg: MemoryConfig, dbFile?: string): void {
|
||||
if (flushInFlight) return;
|
||||
flushInFlight = flushQueue(cfg, { dbFile, limit: 20, paceMs: 400 })
|
||||
.catch(() => undefined)
|
||||
.finally(() => {
|
||||
flushInFlight = null;
|
||||
});
|
||||
}
|
||||
|
||||
export async function flushQueueIfPending(cfg: MemoryConfig, dbFile?: string): Promise<FlushStats | null> {
|
||||
const stats = await queueStats({ dbFile });
|
||||
if (!stats.queued) return null;
|
||||
return flushQueue(cfg, { dbFile, limit: 50, paceMs: 300 });
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Report / stato
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -657,6 +1064,14 @@ export interface LocalDbReport {
|
||||
lastImport?: string;
|
||||
lastEnrich?: string;
|
||||
lastExport?: string;
|
||||
lastFlush?: string;
|
||||
queued: number;
|
||||
syncedQueue: number;
|
||||
duplicateQueue: number;
|
||||
failedQueue: number;
|
||||
pendingInIndex: number;
|
||||
oldestQueued?: string | null;
|
||||
queueLastError?: string | null;
|
||||
topProjects: Array<{ project_id: string | null; n: number }>;
|
||||
duplicates: Array<{ text_hash: string; n: number; ids: string[] }>;
|
||||
}
|
||||
@@ -674,6 +1089,11 @@ export async function localDbReport(opts?: { dbFile?: string }): Promise<LocalDb
|
||||
superseded: 0,
|
||||
private: 0,
|
||||
withProject: 0,
|
||||
queued: 0,
|
||||
syncedQueue: 0,
|
||||
duplicateQueue: 0,
|
||||
failedQueue: 0,
|
||||
pendingInIndex: 0,
|
||||
topProjects: [],
|
||||
duplicates: [],
|
||||
};
|
||||
@@ -697,6 +1117,14 @@ export async function localDbReport(opts?: { dbFile?: string }): Promise<LocalDb
|
||||
report.lastImport = db.prepare("SELECT v FROM meta WHERE k='last_import'").get()?.v;
|
||||
report.lastEnrich = db.prepare("SELECT v FROM meta WHERE k='last_enrich'").get()?.v;
|
||||
report.lastExport = db.prepare("SELECT v FROM meta WHERE k='last_export'").get()?.v;
|
||||
report.lastFlush = db.prepare("SELECT v FROM meta WHERE k='last_flush'").get()?.v;
|
||||
report.queued = one("SELECT COUNT(*) n FROM pending WHERE status='queued'");
|
||||
report.syncedQueue = one("SELECT COUNT(*) n FROM pending WHERE status='synced'");
|
||||
report.duplicateQueue = one("SELECT COUNT(*) n FROM pending WHERE status='duplicate'");
|
||||
report.failedQueue = one("SELECT COUNT(*) n FROM pending WHERE status='failed'");
|
||||
report.pendingInIndex = one("SELECT COUNT(*) n FROM records WHERE pending = 1");
|
||||
report.oldestQueued = db.prepare("SELECT created_at FROM pending WHERE status='queued' ORDER BY id LIMIT 1").get()?.created_at ?? null;
|
||||
report.queueLastError = db.prepare("SELECT last_error FROM pending WHERE last_error IS NOT NULL ORDER BY id DESC LIMIT 1").get()?.last_error ?? null;
|
||||
report.topProjects = db.prepare("SELECT project_id, COUNT(*) n FROM records GROUP BY project_id ORDER BY n DESC LIMIT 8").all() as any;
|
||||
report.duplicates = db
|
||||
.prepare("SELECT text_hash, COUNT(*) n, GROUP_CONCAT(memory_id) ids FROM records WHERE text_hash IS NOT NULL GROUP BY text_hash HAVING n > 1 ORDER BY n DESC LIMIT 20")
|
||||
|
||||
Reference in New Issue
Block a user