#!/usr/bin/env python3 """Consolidamento assistito da cross-encoder (strategia D). Trova i veri duplicati nella collection 'memories': candidatos per cosine (bi-encoder) → cross-score a coppie col reranker (giudice "è lo stesso fatto?") → cluster di duplicati confermati → report (e, con --apply, rimozione dei duplicati perdenti via API gateway, con audit). Uso (su brain): python3 consolidate.py # report su stdout (+ ntfy se configurato) python3 consolidate.py --apply # applica le rimozioni suggerite python3 consolidate.py --limit 300 # limita il numero di record scansionati Env (da /opt/memory/.env se presente): QDRANT_URL, QDRANT_API_KEY, API_KEYS, RERANK_CHAIN, NTFY_CONSOLIDAMENTO (opzionale: URL completo del topic ntfy). """ from __future__ import annotations import argparse import json import math import os import sys import time from datetime import datetime import httpx DEFAULT_ENV_FILE = "/opt/memory/.env" def load_env(path: str) -> dict: env = {} if os.path.exists(path): for line in open(path): line = line.strip() if line and not line.startswith("#") and "=" in line: k, v = line.split("=", 1) env[k] = v return env def sigmoid(x: float) -> float: if x >= 0: z = math.exp(-x) return 1.0 / (1.0 + z) z = math.exp(x) return z / (1.0 + z) def parse_chain(raw: str) -> list[dict]: try: entries = json.loads(raw) if raw else [] except json.JSONDecodeError: return [] return [e for e in entries if isinstance(e, dict) and e.get("url")] def cross_score(http, chain: list[dict], query: str, docs: list[str], timeout_default: float) -> tuple[list[float], str] | None: payload = {"model": "bge-reranker-v2-m3", "query": query, "documents": docs, "top_n": len(docs)} for node in chain: headers = {"Content-Type": "application/json"} if node.get("key"): headers["Authorization"] = f"Bearer {node['key']}" try: r = http.post( f"{node['url'].rstrip('/')}/v1/rerank", json=payload, headers=headers, timeout=node.get("timeout_ms", 10000) / 1000.0, ) r.raise_for_status() scores = [0.0] * len(docs) for item in r.json().get("results", []): idx = int(item["index"]) if 0 <= idx < len(docs): scores[idx] = sigmoid(float(item.get("relevance_score", 0.0))) return scores, node.get("name", node["url"]) except Exception: continue return None def recency_of(created_at: str, half_life_days: float = 180.0) -> float: try: age = (time.time() - datetime.fromisoformat(str(created_at).replace("Z", "+00:00")).timestamp()) / 86400.0 except Exception: return 0.5 return pow(0.5, max(0.0, age) / half_life_days) import time # noqa: E402 (dopo i docstring per leggibilità dell'ordine di import) def priority(rec: dict, recency: float) -> float: """Chi resta nel cluster: confidence + importance + recency.""" conf = {"high": 1.0, "medium": 0.7, "low": 0.4}.get(rec.get("confidence"), 0.7) return conf * 0.5 + float(rec.get("importance", 0.5) or 0.5) * 0.3 + recency * 0.2 def main() -> int: ap = argparse.ArgumentParser(description="Consolidamento duplicati via cross-encoder") ap.add_argument("--env-file", default=DEFAULT_ENV_FILE) ap.add_argument("--limit", type=int, default=0, help="max record da scansionare (0 = tutti)") ap.add_argument("--cosine", type=float, default=0.70, help="soglia cosine per i candidati") ap.add_argument("--cross", type=float, default=0.88, help="soglia cross-encoder per duplicato confermato") ap.add_argument("--apply", action="store_true", help="rimuove i duplicati perdenti via API gateway") ap.add_argument("--gateway-url", default=os.environ.get("GATEWAY_URL", "http://127.0.0.1:8082")) ap.add_argument("--ntfy", default="", help="URL topic ntfy per il report (es. http://127.0.0.1:8091/qmem-consolidamento)") args = ap.parse_args() env = load_env(args.env_file) env.update({k: v for k, v in os.environ.items() if k in ("QDRANT_URL", "QDRANT_API_KEY", "RERANK_CHAIN", "GATEWAY_URL")}) qdrant_url = env.get("QDRANT_URL", "http://127.0.0.1:6333").rstrip("/") api_key = env.get("QDRANT_API_KEY", "") chain = parse_chain(env.get("RERANK_CHAIN", "")) if not chain: print("RERANK_CHAIN vuota: niente cross-scoring, esco", file=sys.stderr) return 2 import httpx headers = {"api-key": api_key} if api_key else {} with httpx.Client(timeout=60) as http: # 1) scroll record attivi (id, testo, metadata, vettore denso) records: dict[str, dict] = {} offset = None while True: body: dict = { "filter": {"must": [{"key": "superseded_by", "match": None}]}, "limit": 256, "with_payload": True, "with_vector": True, } if offset: body["offset"] = offset r = http.post(f"{qdrant_url}/collections/memories/points/scroll", json=body, headers=headers) r.raise_for_status() data = r.json() for p in data.get("points", []): vec = (p.get("vector") or {}).get("") if isinstance(p.get("vector"), dict) else p.get("vector") if not vec: continue records[p["id"]] = { "text": (p.get("payload") or {}).get("text", ""), "confidence": (p.get("payload") or {}).get("confidence", "medium"), "importance": (p.get("payload") or {}).get("importance", 0.5), "created_at": (p.get("payload") or {}).get("created_at", ""), "vector": vec, } offset = data.get("next_page_offset") if not offset: break if args.limit: records = dict(list(records.items())[: args.limit]) print(f"scansionati {len(records)} record attivi") # 2) candidati per cosine (il vettore del record stesso come query) pairs: set[tuple[str, str]] = set() for rid, rec in records.items(): r = http.post( f"{qdrant_url}/collections/memories/points/query", json={"query": rec["vector"], "limit": 4, "with_payload": False}, headers=headers, ) r.raise_for_status() for h in r.json().get("points", []): oid = h["id"] if oid == rid or oid not in records or h["score"] < args.cosine: continue pairs.add((min(rid, oid), max(rid, oid))) print(f"coppie candidate (cosine ≥ {args.cosine}): {len(pairs)}") # 3) cross-score a coppie confirmed: list[dict] = [] for a, b in sorted(pairs): rr = cross_score(http, chain, records[a]["text"][:800], [records[b]["text"][:800]], 10.0) if rr is None: print("catena rerank irraggiungibile: interrompo il cross-scoring", file=sys.stderr) return 3 cross = rr[0][0] if cross >= args.cross: keep, drop = (a, b) if priority(records[a], recency_of(records[a]["created_at"])) >= priority(records[b], recency_of(records[b]["created_at"])) else (b, a) confirmed.append({"keep": keep, "drop": drop, "cross": round(cross, 4)}) # 4) report print(f"duplicati confermati (cross ≥ {args.cross}): {len(confirmed)}") for c in confirmed: keep_txt = records[c["keep"]]["text"][:70].replace("\n", " ") drop_txt = records[c["drop"]]["text"][:60].replace("\n", " ") print(f" KEEP {c['keep']} DROP {c['drop']} cross={c['cross']} | drop: {drop_txt}") if args.apply and confirmed: gw_headers = {"Content-Type": "application/json", "X-API-Key": env.get("API_KEYS", "").split(",")[0]} removed = 0 for c in confirmed: try: r = http.delete(f"{args.gateway_url.rstrip('/')}/v1/memories/{c['drop']}", headers=gw_headers) if r.status_code == 200: removed += 1 else: print(f" delete {c['drop']}: HTTP {r.status_code}", file=sys.stderr) except Exception as exc: print(f" delete {c['drop']}: {exc}", file=sys.stderr) print(f"rimossi {removed}/{len(confirmed)} duplicati") if args.ntfy and confirmed: lines = [f"qmem consolidamento: {len(confirmed)} duplicati confermati"] lines += [f"• {c['cross']} — {records[c['drop']]['text'][:60]}" for c in confirmed[:5]] try: http.post(args.ntfy, data="\n".join(lines).encode(), headers={"Title": "qmem consolidamento"}) except Exception as exc: print(f"ntfy: {exc}", file=sys.stderr) return 0 if __name__ == "__main__": sys.exit(main())