fix(gateway): troncamento documenti rerank (RERANK_MAX_DOC_CHARS=800) — evita 500 batch-size e costi oltre il ctx

gateway/config.py

@@ -39,6 +39,7 @@ SPARSE_VECTOR_NAME = "bm25"
 RERANK_ENABLED = os.environ.get("RERANK_ENABLED", "false").lower() == "true"
 RERANK_MODEL = os.environ.get("RERANK_MODEL", "bge-reranker-v2-m3")
 RERANK_CANDIDATES = int(os.environ.get("RERANK_CANDIDATES", "16"))
+RERANK_MAX_DOC_CHARS = int(os.environ.get("RERANK_MAX_DOC_CHARS", "800"))
 RERANK_TIMEOUT_MS = int(os.environ.get("RERANK_TIMEOUT_MS", "10000"))
 RERANK_RETRY_COOLDOWN_S = int(os.environ.get("RERANK_RETRY_COOLDOWN_S", "60"))
 RERANK_CHAIN = os.environ.get("RERANK_CHAIN", "")

gateway/rerank.py

@@ -21,6 +21,7 @@ import metrics
 from config import (
     RERANK_CHAIN,
     RERANK_ENABLED,
+    RERANK_MAX_DOC_CHARS,
     RERANK_MODEL,
     RERANK_RETRY_COOLDOWN_S,
     RERANK_TIMEOUT_MS,
@@ -121,6 +122,9 @@ async def rerank(query: str, docs: list[str]) -> Optional[tuple[list[float], str
     chain = _get_chain()
     if not chain or not docs:
         return None
+    # Troncamento dei documenti: limita il costo di inferenza (i cross-encoder
+    # scala con la lunghezza della coppia query+doc) e evita input oltre il ctx.
+    docs = [d[:RERANK_MAX_DOC_CHARS] for d in docs]
     live, all_down = live_nodes()
     if not live:
         # tutti in cooldown: ritenta comunque il primo (meglio di un fallimento immediato)

gateway/tests/test_rerank.py

@@ -145,6 +145,20 @@ def test_normalize_score():
     assert rerank.normalize_score(-10.0) < 0.01
 
 
+def test_troncamento_documenti(monkeypatch):
+    _use_chain(monkeypatch)
+    seen: dict = {}
+
+    def handler(request):
+        seen["docs"] = json.loads(request.content)["documents"]
+        return httpx.Response(200, json={"results": [{"index": 0, "relevance_score": 1.0}]})
+
+    _mock_client(handler)
+    asyncio.run(rerank.rerank("q", ["x" * 5000, "corto"]))
+    assert len(seen["docs"][0]) == 800  # default RERANK_MAX_DOC_CHARS
+    assert seen["docs"][1] == "corto"
+
+
 # ---------------------------------------------------------------------------
 # integrazione endpoint search
 # ---------------------------------------------------------------------------