feat(gateway): stadio rerank con catena di fallback resiliente (frigate→brain)
- gateway/rerank.py: catena da RERANK_CHAIN (JSON, per-nodo key+timeout),
cooldown 60s sui nodi falliti, score sigmoide [0,1], degrada con grazia
all'ordine di fusione se tutti i nodi sono giù
- routes: /v1/memories:search applica il rerank post-fusione (fetch esteso a
RERANK_CANDIDATES), risposta con rerank{used,backend,took_ms}, flag
per-query rerank=false; /v1/version espone lo stato rerank
- store: search() accetta limit esteso; models: SearchIn.rerank
- metrics: qmem_rerank_calls_total + durata per backend
- test: 10 nuovi (fallback, cooldown, degradazione, integrazione) — 46 pass
This commit is contained in:
@@ -20,6 +20,11 @@ def record_search(hits: int) -> None:
|
||||
_metrics["search_hits"] += hits
|
||||
|
||||
|
||||
def record_rerank(backend: str, ok: bool, took_ms: int) -> None:
|
||||
_metrics["rerank_calls"][(backend, "ok" if ok else "fail")] += 1
|
||||
_metrics["rerank_duration_sum"][backend] += took_ms
|
||||
|
||||
|
||||
def snapshot(qdrant: Any, collection: str) -> dict:
|
||||
try:
|
||||
points = qdrant.get_collection(collection).points_count
|
||||
@@ -34,6 +39,8 @@ def snapshot(qdrant: Any, collection: str) -> dict:
|
||||
"errors": {f"{endpoint}:{status}": count for (endpoint, status), count in _metrics["errors"].items()},
|
||||
"search_queries": _metrics["search_queries"],
|
||||
"search_hits": _metrics["search_hits"],
|
||||
"rerank_calls": {f"{backend}:{outcome}": count for (backend, outcome), count in _metrics["rerank_calls"].items()},
|
||||
"rerank_avg_ms": {backend: round(total / _metrics["rerank_calls"][(backend, "ok")], 2) for backend, total in _metrics["rerank_duration_sum"].items() if _metrics["rerank_calls"][(backend, "ok")]},
|
||||
"points": points,
|
||||
}
|
||||
|
||||
@@ -50,6 +57,10 @@ def prometheus_lines(qdrant: Any, collection: str) -> list[str]:
|
||||
lines.append(f'qmem_errors_total{{endpoint="{endpoint}",status="{status}"}} {count}')
|
||||
lines.append(f"qmem_search_queries_total {_metrics['search_queries']}")
|
||||
lines.append(f"qmem_search_hits_total {_metrics['search_hits']}")
|
||||
for (backend, outcome), count in _metrics["rerank_calls"].items():
|
||||
lines.append(f'qmem_rerank_calls_total{{backend="{backend}",outcome="{outcome}"}} {count}')
|
||||
for backend, s in _metrics["rerank_duration_sum"].items():
|
||||
lines.append(f'qmem_rerank_duration_seconds_sum{{backend="{backend}"}} {s / 1000:.6f}')
|
||||
try:
|
||||
lines.append(f"qmem_points {qdrant.get_collection(collection).points_count}")
|
||||
except Exception: # noqa: BLE001
|
||||
|
||||
Reference in New Issue
Block a user