feat(gateway): stadio rerank con catena di fallback resiliente (frigate→brain)

- gateway/rerank.py: catena da RERANK_CHAIN (JSON, per-nodo key+timeout), cooldown 60s sui nodi falliti, score sigmoide [0,1], degrada con grazia all'ordine di fusione se tutti i nodi sono giù - routes: /v1/memories:search applica il rerank post-fusione (fetch esteso a RERANK_CANDIDATES), risposta con rerank{used,backend,took_ms}, flag per-query rerank=false; /v1/version espone lo stato rerank - store: search() accetta limit esteso; models: SearchIn.rerank - metrics: qmem_rerank_calls_total + durata per backend - test: 10 nuovi (fallback, cooldown, degradazione, integrazione) — 46 pass
2026-09-08 12:10:27 +02:00
parent c21ef5e92a
commit 20766de540
8 changed files with 447 additions and 11 deletions
@@ -0,0 +1,205 @@
+"""Test dello stadio rerank: catena di fallback, cooldown, degrada con grazia."""
+from __future__ import annotations
+
+import asyncio
+import json
+
+import httpx
+import pytest
+
+import rerank
+from test_api import auth_headers, make_record
+
+CHAIN = json.dumps(
+    [
+        {"name": "primario", "url": "http://primario:9002", "key": "k1", "timeout_ms": 500},
+        {"name": "fallback", "url": "http://fallback:9003", "key": "k2", "timeout_ms": 5000},
+    ]
+)
+
+
+@pytest.fixture(autouse=True)
+def _reset_chain(monkeypatch):
+    rerank.reset_chain_cache()
+    rerank._http = None
+    yield
+    rerank.reset_chain_cache()
+    rerank._http = None
+
+
+def _use_chain(monkeypatch, raw=CHAIN):
+    # rerank.py importa i valori di config con `from config import`: si patchano
+    # gli attributi del modulo rerank, non config.
+    monkeypatch.setattr(rerank, "RERANK_CHAIN", raw)
+    rerank.reset_chain_cache()
+
+
+def _mock_client(handler) -> list[str]:
+    """Client con transport mockato; ritorna la lista in cui registrare le chiamate."""
+    calls: list[str] = []
+
+    def tracking_handler(request):
+        calls.append(f"{request.url.host}{request.url.path}")
+        return handler(request)
+
+    rerank._http = httpx.AsyncClient(transport=httpx.MockTransport(tracking_handler))
+    return calls
+
+
+# ---------------------------------------------------------------------------
+# parse della catena
+# ---------------------------------------------------------------------------
+
+
+def test_parse_chain_valida(monkeypatch):
+    _use_chain(monkeypatch)
+    nodes = rerank._get_chain()
+    assert [n.name for n in nodes] == ["primario", "fallback"]
+    assert nodes[0].key == "k1"
+    assert nodes[0].timeout_ms == 500
+    assert nodes[1].timeout_ms == 5000
+
+
+def test_parse_chain_json_invalido(monkeypatch):
+    _use_chain(monkeypatch, raw="non-json")
+    assert rerank._get_chain() == []
+    assert not rerank.enabled()
+
+
+def test_parse_chain_scarta_url_senza_schema(monkeypatch):
+    _use_chain(monkeypatch, raw=json.dumps([{"name": "x", "url": "primario:9002"}]))
+    assert rerank._get_chain() == []
+
+
+# ---------------------------------------------------------------------------
+# fallback e cooldown
+# ---------------------------------------------------------------------------
+
+
+def test_fallback_primario_500(monkeypatch):
+    _use_chain(monkeypatch)
+
+    def handler(request):
+        if request.url.host == "primario":
+            return httpx.Response(500)
+        return httpx.Response(200, json={"results": [{"index": 1, "relevance_score": 2.0}, {"index": 0, "relevance_score": -1.0}]})
+
+    _mock_client(handler)
+    scores, backend, _took = asyncio.run(rerank.rerank("q", ["docA", "docB"]))
+    assert backend == "fallback"
+    # gli score tornano allineati alla posizione originaria dei documenti
+    assert scores == pytest.approx([-1.0, 2.0])
+
+
+def test_cooldown_salta_nodo_fallito(monkeypatch):
+    _use_chain(monkeypatch)
+
+    def handler(request):
+        if request.url.host == "primario":
+            return httpx.Response(500)
+        return httpx.Response(200, json={"results": [{"index": 0, "relevance_score": 1.0}]})
+
+    _mock_client(handler)
+    asyncio.run(rerank.rerank("q", ["a", "b"]))
+    scores, backend, _ = asyncio.run(rerank.rerank("q", ["a", "b"]))
+    assert backend == "fallback"  # il primario è in cooldown e non viene richiamato
+
+
+def test_cooldown_non_blocca_per_sempre(monkeypatch):
+    _use_chain(monkeypatch)
+    calls: list[str] = []
+
+    def handler(request):
+        calls.append(request.url.host)
+        if request.url.host == "primario":
+            return httpx.Response(500)
+        return httpx.Response(200, json={"results": [{"index": 0, "relevance_score": 1.0}]})
+
+    _mock_client(handler)
+    asyncio.run(rerank.rerank("q", ["a"]))
+    # svuota il cooldown: il primario torna eleggibile
+    rerank._down_until.clear()
+    asyncio.run(rerank.rerank("q", ["a"]))
+    assert calls == ["primario", "fallback", "primario", "fallback"]
+
+
+def test_tutti_nodi_falliti_restifica_none(monkeypatch):
+    _use_chain(monkeypatch)
+    _mock_client(lambda request: httpx.Response(500))
+    assert asyncio.run(rerank.rerank("q", ["a", "b"])) is None
+
+
+def test_enabled_richiede_catena(monkeypatch):
+    _use_chain(monkeypatch, raw="")
+    assert not rerank.enabled()
+    _use_chain(monkeypatch)
+    monkeypatch.setattr(rerank, "RERANK_ENABLED", True)
+    assert rerank.enabled()
+    monkeypatch.setattr(rerank, "RERANK_ENABLED", False)
+    assert not rerank.enabled()
+
+
+def test_normalize_score():
+    assert rerank.normalize_score(0.0) == pytest.approx(0.5)
+    assert rerank.normalize_score(10.0) > 0.99
+    assert rerank.normalize_score(-10.0) < 0.01
+
+
+# ---------------------------------------------------------------------------
+# integrazione endpoint search
+# ---------------------------------------------------------------------------
+
+
+def test_search_rerank_riordina(client, monkeypatch):
+    for text in ["alpha", "beta", "gamma"]:
+        r = client.post("/v1/memories", json=make_record(text=text), headers=auth_headers())
+        assert r.status_code == 200
+
+    async def fake_rerank(query, docs):
+        # inverte: beta (index 1) primo, poi gamma/alpha
+        return [0.1, 0.9, 0.5], "finto", 12
+
+    monkeypatch.setattr(rerank, "enabled", lambda: True)
+    monkeypatch.setattr(rerank, "rerank", fake_rerank)
+
+    resp = client.post("/v1/memories:search", json={"query": "q", "top_k": 3}, headers=auth_headers())
+    assert resp.status_code == 200
+    data = resp.json()
+    assert data["rerank"]["used"] is True
+    assert data["rerank"]["backend"] == "finto"
+    texts = [r["text"] for r in data["results"]]
+    assert texts == ["beta", "gamma", "alpha"]
+    assert data["results"][0]["rerank_score"] == pytest.approx(rerank.normalize_score(0.9), abs=0.01)
+
+
+def test_search_rerank_disattivato_per_query(client, monkeypatch):
+    for text in ["alpha", "beta"]:
+        client.post("/v1/memories", json=make_record(text=text), headers=auth_headers())
+
+    async def fail_rerank(query, docs):
+        raise AssertionError("rerank non deve essere chiamato con rerank=false")
+
+    monkeypatch.setattr(rerank, "enabled", lambda: True)
+    monkeypatch.setattr(rerank, "rerank", fail_rerank)
+
+    resp = client.post("/v1/memories:search", json={"query": "q", "top_k": 2, "rerank": False}, headers=auth_headers())
+    assert resp.status_code == 200
+    assert resp.json()["rerank"]["enabled"] is False
+
+
+def test_search_rerank_fallito_degrada_con_grazia(client, monkeypatch):
+    for text in ["alpha", "beta"]:
+        client.post("/v1/memories", json=make_record(text=text), headers=auth_headers())
+
+    async def fail_rerank(query, docs):
+        return None
+
+    monkeypatch.setattr(rerank, "enabled", lambda: True)
+    monkeypatch.setattr(rerank, "rerank", fail_rerank)
+
+    resp = client.post("/v1/memories:search", json={"query": "q", "top_k": 2}, headers=auth_headers())
+    assert resp.status_code == 200
+    data = resp.json()
+    assert data["rerank"]["used"] is False
+    assert "non raggiungibili" in data["rerank"]["reason"]
+    assert len(data["results"]) == 2  # ordine di fusione preservato