feat(gateway): strategie rerank oltre la search (A-F)
- A: gate store con cross-encoder — guardrail.decide async, conferma/scarta quasi-duplicati (CROSS_DUP_CONFIRMED/WEAK/LOW_COSINE), suggerimento supersedes in WARN, degrada a cosine-only se il reranker è giù - B: verifica supersede — cross-score (nuovo,vecchio) sotto soglia → supersede_warning non bloccante + audit - C: score composito in search — rerank + importance (nuovo campo payload) + recency decay (180gg) + authority, pesi SCORE_W_* da env - E: multi-query — SearchIn.queries (max 3), pool unito con dedup, rerank unico; endpoint POST /v1/score come primitiva cross-encoder (F-lite) - extension search.ts: param queries + rerank_score/composite in output - D: scripts/consolidate.py — dedup periodico a coppie via cross-encoder con report ntfy e --apply via gateway - test: 69 pass (+11 strategie); guardrail_version similarity-v2
This commit is contained in:
enne2
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Consolidamento assistito da cross-encoder (strategia D).
|
||||
|
||||
Trova i veri duplicati nella collection 'memories': candidatos per cosine
|
||||
(bi-encoder) → cross-score a coppie col reranker (giudice "è lo stesso fatto?")
|
||||
→ cluster di duplicati confermati → report (e, con --apply, rimozione dei
|
||||
duplicati perdenti via API gateway, con audit).
|
||||
|
||||
Uso (su brain):
|
||||
python3 consolidate.py # report su stdout (+ ntfy se configurato)
|
||||
python3 consolidate.py --apply # applica le rimozioni suggerite
|
||||
python3 consolidate.py --limit 300 # limita il numero di record scansionati
|
||||
|
||||
Env (da /opt/memory/.env se presente): QDRANT_URL, QDRANT_API_KEY, API_KEYS,
|
||||
RERANK_CHAIN, NTFY_CONSOLIDAMENTO (opzionale: URL completo del topic ntfy).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
import httpx
|
||||
|
||||
DEFAULT_ENV_FILE = "/opt/memory/.env"
|
||||
|
||||
|
||||
def load_env(path: str) -> dict:
|
||||
env = {}
|
||||
if os.path.exists(path):
|
||||
for line in open(path):
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
k, v = line.split("=", 1)
|
||||
env[k] = v
|
||||
return env
|
||||
|
||||
|
||||
def sigmoid(x: float) -> float:
|
||||
if x >= 0:
|
||||
z = math.exp(-x)
|
||||
return 1.0 / (1.0 + z)
|
||||
z = math.exp(x)
|
||||
return z / (1.0 + z)
|
||||
|
||||
|
||||
def parse_chain(raw: str) -> list[dict]:
|
||||
try:
|
||||
entries = json.loads(raw) if raw else []
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
return [e for e in entries if isinstance(e, dict) and e.get("url")]
|
||||
|
||||
|
||||
def cross_score(http, chain: list[dict], query: str, docs: list[str], timeout_default: float) -> tuple[list[float], str] | None:
|
||||
payload = {"model": "bge-reranker-v2-m3", "query": query, "documents": docs, "top_n": len(docs)}
|
||||
for node in chain:
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if node.get("key"):
|
||||
headers["Authorization"] = f"Bearer {node['key']}"
|
||||
try:
|
||||
r = http.post(
|
||||
f"{node['url'].rstrip('/')}/v1/rerank",
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=node.get("timeout_ms", 10000) / 1000.0,
|
||||
)
|
||||
r.raise_for_status()
|
||||
scores = [0.0] * len(docs)
|
||||
for item in r.json().get("results", []):
|
||||
idx = int(item["index"])
|
||||
if 0 <= idx < len(docs):
|
||||
scores[idx] = sigmoid(float(item.get("relevance_score", 0.0)))
|
||||
return scores, node.get("name", node["url"])
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def recency_of(created_at: str, half_life_days: float = 180.0) -> float:
|
||||
try:
|
||||
age = (time.time() - datetime.fromisoformat(str(created_at).replace("Z", "+00:00")).timestamp()) / 86400.0
|
||||
except Exception:
|
||||
return 0.5
|
||||
return pow(0.5, max(0.0, age) / half_life_days)
|
||||
|
||||
|
||||
import time # noqa: E402 (dopo i docstring per leggibilità dell'ordine di import)
|
||||
|
||||
|
||||
def priority(rec: dict, recency: float) -> float:
|
||||
"""Chi resta nel cluster: confidence + importance + recency."""
|
||||
conf = {"high": 1.0, "medium": 0.7, "low": 0.4}.get(rec.get("confidence"), 0.7)
|
||||
return conf * 0.5 + float(rec.get("importance", 0.5) or 0.5) * 0.3 + recency * 0.2
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="Consolidamento duplicati via cross-encoder")
|
||||
ap.add_argument("--env-file", default=DEFAULT_ENV_FILE)
|
||||
ap.add_argument("--limit", type=int, default=0, help="max record da scansionare (0 = tutti)")
|
||||
ap.add_argument("--cosine", type=float, default=0.70, help="soglia cosine per i candidati")
|
||||
ap.add_argument("--cross", type=float, default=0.88, help="soglia cross-encoder per duplicato confermato")
|
||||
ap.add_argument("--apply", action="store_true", help="rimuove i duplicati perdenti via API gateway")
|
||||
ap.add_argument("--gateway-url", default=os.environ.get("GATEWAY_URL", "http://127.0.0.1:8082"))
|
||||
ap.add_argument("--ntfy", default="", help="URL topic ntfy per il report (es. http://127.0.0.1:8091/qmem-consolidamento)")
|
||||
args = ap.parse_args()
|
||||
|
||||
env = load_env(args.env_file)
|
||||
env.update({k: v for k, v in os.environ.items() if k in ("QDRANT_URL", "QDRANT_API_KEY", "RERANK_CHAIN", "GATEWAY_URL")})
|
||||
qdrant_url = env.get("QDRANT_URL", "http://127.0.0.1:6333").rstrip("/")
|
||||
api_key = env.get("QDRANT_API_KEY", "")
|
||||
chain = parse_chain(env.get("RERANK_CHAIN", ""))
|
||||
if not chain:
|
||||
print("RERANK_CHAIN vuota: niente cross-scoring, esco", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
import httpx
|
||||
|
||||
headers = {"api-key": api_key} if api_key else {}
|
||||
with httpx.Client(timeout=60) as http:
|
||||
# 1) scroll record attivi (id, testo, metadata, vettore denso)
|
||||
records: dict[str, dict] = {}
|
||||
offset = None
|
||||
while True:
|
||||
body: dict = {
|
||||
"filter": {"must": [{"key": "superseded_by", "match": None}]},
|
||||
"limit": 256,
|
||||
"with_payload": True,
|
||||
"with_vector": True,
|
||||
}
|
||||
if offset:
|
||||
body["offset"] = offset
|
||||
r = http.post(f"{qdrant_url}/collections/memories/points/scroll", json=body, headers=headers)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
for p in data.get("points", []):
|
||||
vec = (p.get("vector") or {}).get("") if isinstance(p.get("vector"), dict) else p.get("vector")
|
||||
if not vec:
|
||||
continue
|
||||
records[p["id"]] = {
|
||||
"text": (p.get("payload") or {}).get("text", ""),
|
||||
"confidence": (p.get("payload") or {}).get("confidence", "medium"),
|
||||
"importance": (p.get("payload") or {}).get("importance", 0.5),
|
||||
"created_at": (p.get("payload") or {}).get("created_at", ""),
|
||||
"vector": vec,
|
||||
}
|
||||
offset = data.get("next_page_offset")
|
||||
if not offset:
|
||||
break
|
||||
if args.limit:
|
||||
records = dict(list(records.items())[: args.limit])
|
||||
print(f"scansionati {len(records)} record attivi")
|
||||
|
||||
# 2) candidati per cosine (il vettore del record stesso come query)
|
||||
pairs: set[tuple[str, str]] = set()
|
||||
for rid, rec in records.items():
|
||||
r = http.post(
|
||||
f"{qdrant_url}/collections/memories/points/query",
|
||||
json={"query": rec["vector"], "limit": 4, "with_payload": False},
|
||||
headers=headers,
|
||||
)
|
||||
r.raise_for_status()
|
||||
for h in r.json().get("points", []):
|
||||
oid = h["id"]
|
||||
if oid == rid or oid not in records or h["score"] < args.cosine:
|
||||
continue
|
||||
pairs.add((min(rid, oid), max(rid, oid)))
|
||||
print(f"coppie candidate (cosine ≥ {args.cosine}): {len(pairs)}")
|
||||
|
||||
# 3) cross-score a coppie
|
||||
confirmed: list[dict] = []
|
||||
for a, b in sorted(pairs):
|
||||
rr = cross_score(http, chain, records[a]["text"][:800], [records[b]["text"][:800]], 10.0)
|
||||
if rr is None:
|
||||
print("catena rerank irraggiungibile: interrompo il cross-scoring", file=sys.stderr)
|
||||
return 3
|
||||
cross = rr[0][0]
|
||||
if cross >= args.cross:
|
||||
keep, drop = (a, b) if priority(records[a], recency_of(records[a]["created_at"])) >= priority(records[b], recency_of(records[b]["created_at"])) else (b, a)
|
||||
confirmed.append({"keep": keep, "drop": drop, "cross": round(cross, 4)})
|
||||
|
||||
# 4) report
|
||||
print(f"duplicati confermati (cross ≥ {args.cross}): {len(confirmed)}")
|
||||
for c in confirmed:
|
||||
keep_txt = records[c["keep"]]["text"][:70].replace("\n", " ")
|
||||
drop_txt = records[c["drop"]]["text"][:60].replace("\n", " ")
|
||||
print(f" KEEP {c['keep']} DROP {c['drop']} cross={c['cross']} | drop: {drop_txt}")
|
||||
|
||||
if args.apply and confirmed:
|
||||
gw_headers = {"Content-Type": "application/json", "X-API-Key": env.get("API_KEYS", "").split(",")[0]}
|
||||
removed = 0
|
||||
for c in confirmed:
|
||||
try:
|
||||
r = http.delete(f"{args.gateway_url.rstrip('/')}/v1/memories/{c['drop']}", headers=gw_headers)
|
||||
if r.status_code == 200:
|
||||
removed += 1
|
||||
else:
|
||||
print(f" delete {c['drop']}: HTTP {r.status_code}", file=sys.stderr)
|
||||
except Exception as exc:
|
||||
print(f" delete {c['drop']}: {exc}", file=sys.stderr)
|
||||
print(f"rimossi {removed}/{len(confirmed)} duplicati")
|
||||
|
||||
if args.ntfy and confirmed:
|
||||
lines = [f"qmem consolidamento: {len(confirmed)} duplicati confermati"]
|
||||
lines += [f"• {c['cross']} — {records[c['drop']]['text'][:60]}" for c in confirmed[:5]]
|
||||
try:
|
||||
http.post(args.ntfy, data="\n".join(lines).encode(), headers={"Title": "qmem consolidamento"})
|
||||
except Exception as exc:
|
||||
print(f"ntfy: {exc}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user