pi-qmem/scripts/consolidate.py

#!/usr/bin/env python3
"""Consolidamento assistito da cross-encoder (strategia D).

Trova i veri duplicati nella collection 'memories': candidatos per cosine
(bi-encoder) → cross-score a coppie col reranker (giudice "è lo stesso fatto?")
→ cluster di duplicati confermati → report (e, con --apply, rimozione dei
duplicati perdenti via API gateway, con audit).

Uso (su brain):
  python3 consolidate.py                 # report su stdout (+ ntfy se configurato)
  python3 consolidate.py --apply         # applica le rimozioni suggerite
  python3 consolidate.py --limit 300     # limita il numero di record scansionati

Env (da /opt/memory/.env se presente): QDRANT_URL, QDRANT_API_KEY, API_KEYS,
RERANK_CHAIN, NTFY_CONSOLIDAMENTO (opzionale: URL completo del topic ntfy).
"""
from __future__ import annotations

import argparse
import json
import math
import os
import sys
import time
from datetime import datetime

import httpx

DEFAULT_ENV_FILE = "/opt/memory/.env"


def load_env(path: str) -> dict:
    env = {}
    if os.path.exists(path):
        for line in open(path):
            line = line.strip()
            if line and not line.startswith("#") and "=" in line:
                k, v = line.split("=", 1)
                env[k] = v
    return env


def sigmoid(x: float) -> float:
    if x >= 0:
        z = math.exp(-x)
        return 1.0 / (1.0 + z)
    z = math.exp(x)
    return z / (1.0 + z)


def parse_chain(raw: str) -> list[dict]:
    try:
        entries = json.loads(raw) if raw else []
    except json.JSONDecodeError:
        return []
    return [e for e in entries if isinstance(e, dict) and e.get("url")]


def cross_score(http, chain: list[dict], query: str, docs: list[str], timeout_default: float) -> tuple[list[float], str] | None:
    payload = {"model": "bge-reranker-v2-m3", "query": query, "documents": docs, "top_n": len(docs)}
    for node in chain:
        headers = {"Content-Type": "application/json"}
        if node.get("key"):
            headers["Authorization"] = f"Bearer {node['key']}"
        try:
            r = http.post(
                f"{node['url'].rstrip('/')}/v1/rerank",
                json=payload,
                headers=headers,
                timeout=node.get("timeout_ms", 10000) / 1000.0,
            )
            r.raise_for_status()
            scores = [0.0] * len(docs)
            for item in r.json().get("results", []):
                idx = int(item["index"])
                if 0 <= idx < len(docs):
                    scores[idx] = sigmoid(float(item.get("relevance_score", 0.0)))
            return scores, node.get("name", node["url"])
        except Exception:
            continue
    return None


def recency_of(created_at: str, half_life_days: float = 180.0) -> float:
    try:
        age = (time.time() - datetime.fromisoformat(str(created_at).replace("Z", "+00:00")).timestamp()) / 86400.0
    except Exception:
        return 0.5
    return pow(0.5, max(0.0, age) / half_life_days)


import time  # noqa: E402  (dopo i docstring per leggibilità dell'ordine di import)


def priority(rec: dict, recency: float) -> float:
    """Chi resta nel cluster: confidence + importance + recency."""
    conf = {"high": 1.0, "medium": 0.7, "low": 0.4}.get(rec.get("confidence"), 0.7)
    return conf * 0.5 + float(rec.get("importance", 0.5) or 0.5) * 0.3 + recency * 0.2


def main() -> int:
    ap = argparse.ArgumentParser(description="Consolidamento duplicati via cross-encoder")
    ap.add_argument("--env-file", default=DEFAULT_ENV_FILE)
    ap.add_argument("--limit", type=int, default=0, help="max record da scansionare (0 = tutti)")
    ap.add_argument("--cosine", type=float, default=0.70, help="soglia cosine per i candidati")
    ap.add_argument("--cross", type=float, default=0.88, help="soglia cross-encoder per duplicato confermato")
    ap.add_argument("--apply", action="store_true", help="rimuove i duplicati perdenti via API gateway")
    ap.add_argument("--gateway-url", default=os.environ.get("GATEWAY_URL", "http://127.0.0.1:8082"))
    ap.add_argument("--ntfy", default="", help="URL topic ntfy per il report (es. http://127.0.0.1:8091/qmem-consolidamento)")
    args = ap.parse_args()

    env = load_env(args.env_file)
    env.update({k: v for k, v in os.environ.items() if k in ("QDRANT_URL", "QDRANT_API_KEY", "RERANK_CHAIN", "GATEWAY_URL")})
    qdrant_url = env.get("QDRANT_URL", "http://127.0.0.1:6333").rstrip("/")
    api_key = env.get("QDRANT_API_KEY", "")
    chain = parse_chain(env.get("RERANK_CHAIN", ""))
    if not chain:
        print("RERANK_CHAIN vuota: niente cross-scoring, esco", file=sys.stderr)
        return 2

    import httpx

    headers = {"api-key": api_key} if api_key else {}
    with httpx.Client(timeout=60) as http:
        # 1) scroll record attivi (id, testo, metadata, vettore denso)
        records: dict[str, dict] = {}
        offset = None
        while True:
            body: dict = {
                "filter": {"must": [{"key": "superseded_by", "match": None}]},
                "limit": 256,
                "with_payload": True,
                "with_vector": True,
            }
            if offset:
                body["offset"] = offset
            r = http.post(f"{qdrant_url}/collections/memories/points/scroll", json=body, headers=headers)
            r.raise_for_status()
            data = r.json()
            for p in data.get("points", []):
                vec = (p.get("vector") or {}).get("") if isinstance(p.get("vector"), dict) else p.get("vector")
                if not vec:
                    continue
                records[p["id"]] = {
                    "text": (p.get("payload") or {}).get("text", ""),
                    "confidence": (p.get("payload") or {}).get("confidence", "medium"),
                    "importance": (p.get("payload") or {}).get("importance", 0.5),
                    "created_at": (p.get("payload") or {}).get("created_at", ""),
                    "vector": vec,
                }
            offset = data.get("next_page_offset")
            if not offset:
                break
        if args.limit:
            records = dict(list(records.items())[: args.limit])
        print(f"scansionati {len(records)} record attivi")

        # 2) candidati per cosine (il vettore del record stesso come query)
        pairs: set[tuple[str, str]] = set()
        for rid, rec in records.items():
            r = http.post(
                f"{qdrant_url}/collections/memories/points/query",
                json={"query": rec["vector"], "limit": 4, "with_payload": False},
                headers=headers,
            )
            r.raise_for_status()
            for h in r.json().get("points", []):
                oid = h["id"]
                if oid == rid or oid not in records or h["score"] < args.cosine:
                    continue
                pairs.add((min(rid, oid), max(rid, oid)))
        print(f"coppie candidate (cosine ≥ {args.cosine}): {len(pairs)}")

        # 3) cross-score a coppie
        confirmed: list[dict] = []
        for a, b in sorted(pairs):
            rr = cross_score(http, chain, records[a]["text"][:800], [records[b]["text"][:800]], 10.0)
            if rr is None:
                print("catena rerank irraggiungibile: interrompo il cross-scoring", file=sys.stderr)
                return 3
            cross = rr[0][0]
            if cross >= args.cross:
                keep, drop = (a, b) if priority(records[a], recency_of(records[a]["created_at"])) >= priority(records[b], recency_of(records[b]["created_at"])) else (b, a)
                confirmed.append({"keep": keep, "drop": drop, "cross": round(cross, 4)})

        # 4) report
        print(f"duplicati confermati (cross ≥ {args.cross}): {len(confirmed)}")
        for c in confirmed:
            keep_txt = records[c["keep"]]["text"][:70].replace("\n", " ")
            drop_txt = records[c["drop"]]["text"][:60].replace("\n", " ")
            print(f"  KEEP {c['keep']}  DROP {c['drop']}  cross={c['cross']}  | drop: {drop_txt}")

        if args.apply and confirmed:
            gw_headers = {"Content-Type": "application/json", "X-API-Key": env.get("API_KEYS", "").split(",")[0]}
            removed = 0
            for c in confirmed:
                try:
                    r = http.delete(f"{args.gateway_url.rstrip('/')}/v1/memories/{c['drop']}", headers=gw_headers)
                    if r.status_code == 200:
                        removed += 1
                    else:
                        print(f"  delete {c['drop']}: HTTP {r.status_code}", file=sys.stderr)
                except Exception as exc:
                    print(f"  delete {c['drop']}: {exc}", file=sys.stderr)
            print(f"rimossi {removed}/{len(confirmed)} duplicati")

        if args.ntfy and confirmed:
            lines = [f"qmem consolidamento: {len(confirmed)} duplicati confermati"]
            lines += [f"• {c['cross']} — {records[c['drop']]['text'][:60]}" for c in confirmed[:5]]
            try:
                http.post(args.ntfy, data="\n".join(lines).encode(), headers={"Title": "qmem consolidamento"})
            except Exception as exc:
                print(f"ntfy: {exc}", file=sys.stderr)
        return 0


if __name__ == "__main__":
    sys.exit(main())