feat(gateway): catena di fallback per gli embedding + retry transiente su Qdrant

- embed.py: EMBED_CHAIN (JSON per-nodo {name,url,api,key,timeout_ms}, api
  llamacpp|ollama), cooldown 60s sui nodi falliti, validazione dimensione
  EMBED_DIM, compatibilità legacy quando la catena è vuota
- state.py: ResilientQdrant — proxy che ritenta i metodi del client Qdrant
  su httpx.TransportError (store/search/transienti), errori applicativi
  esenti; contatore qdrant_retries in metriche
- metrics: qmem_embed_calls_total + durata per backend
- /v1/version espone embed_nodes; versione 2.10.0
- test: 11 nuovi (chain, cooldown, dim mismatch, legacy, retry transiente) — 58 pass
This commit is contained in:
enne2
2026-09-08 12:19:16 +02:00
parent 78a93ee771
commit fcd6b1670e
6 changed files with 418 additions and 25 deletions
+11 -1
View File
@@ -13,6 +13,13 @@ EMBED_URL = os.environ.get("EMBED_URL", os.environ.get("OLLAMA_URL", "http://127
EMBED_MODEL = os.environ.get("EMBED_MODEL", "bge-m3")
EMBED_API_KEY = os.environ.get("EMBED_API_KEY", "")
EMBED_DIM = int(os.environ.get("EMBED_DIM", "1024"))
# Catena di fallback per gli embedding (JSON, formato RERANK_CHAIN + campo "api").
# Vuota → comportamento legacy: endpoint singolo da EMBED_API/EMBED_URL/EMBED_API_KEY.
EMBED_CHAIN = os.environ.get("EMBED_CHAIN", "")
EMBED_TIMEOUT_MS = int(os.environ.get("EMBED_TIMEOUT_MS", "30000"))
EMBED_RETRY_COOLDOWN_S = int(os.environ.get("EMBED_RETRY_COOLDOWN_S", "60"))
# Retry transiente per le chiamate Qdrant (store/search inclusi)
QDRANT_RETRIES = int(os.environ.get("QDRANT_RETRIES", "3"))
COLLECTION = os.environ.get("COLLECTION", "memories")
API_KEYS: set[str] = {k.strip() for k in os.environ.get("API_KEYS", "").split(",") if k.strip()}
RATE_LIMIT_PER_MIN = int(os.environ.get("RATE_LIMIT_PER_MIN", "120"))
@@ -23,7 +30,7 @@ GUARDRAIL_BLOCK_THRESHOLD = float(os.environ.get("GUARDRAIL_BLOCK_THRESHOLD", "0
GUARDRAIL_WARN_THRESHOLD = float(os.environ.get("GUARDRAIL_WARN_THRESHOLD", "0.70"))
GUARDRAIL_VERSION = "similarity-v1"
GIT_COMMIT = os.environ.get("GIT_COMMIT", "unknown").strip()
GATEWAY_VERSION = os.environ.get("GATEWAY_VERSION", "2.9.0").strip()
GATEWAY_VERSION = os.environ.get("GATEWAY_VERSION", "2.10.0").strip()
VM_PUSH_URL = os.environ.get("VM_PUSH_URL", "http://host.docker.internal:8428/api/v1/import/prometheus")
VM_PUSH_INTERVAL = int(os.environ.get("VM_PUSH_INTERVAL", "30"))
@@ -52,4 +59,7 @@ _metrics: dict[str, Any] = {
"search_hits": 0,
"rerank_calls": Counter(),
"rerank_duration_sum": Counter(),
"embed_calls": Counter(),
"embed_duration_sum": Counter(),
"qdrant_retries": 0,
}
+155 -21
View File
@@ -1,16 +1,38 @@
"""Embedding denso (Ollama/llama.cpp) e sparse BM25."""
"""Embedding denso (Ollama/llama.cpp) e sparse BM25, con catena di fallback resiliente.
Catena da EMBED_CHAIN (JSON, stesso formato di RERANK_CHAIN + campo "api"):
il primo nodo raggiungibile vince, i nodi falliti entrano in cooldown. Se
EMBED_CHAIN è vuota si usa il comportamento legacy (endpoint singolo da
EMBED_API/EMBED_URL/EMBED_API_KEY). L'ultimo errore viene rilanciato al client
come gli endpoint precedenti: nessuna degradazione silenziosa della scrittura.
"""
from __future__ import annotations
import json
import time
from dataclasses import dataclass
from typing import Any, Optional
import httpx
from qdrant_client.http import models as qm
from config import EMBED_API, EMBED_API_KEY, EMBED_MODEL, EMBED_URL, SPARSE_VECTOR_NAME, log
import metrics
from config import (
EMBED_API,
EMBED_API_KEY,
EMBED_CHAIN,
EMBED_DIM,
EMBED_MODEL,
EMBED_RETRY_COOLDOWN_S,
EMBED_TIMEOUT_MS,
EMBED_URL,
SPARSE_VECTOR_NAME,
log,
)
try:
from fastembed import SparseTextEmbedding
_sparse_model: Optional[SparseTextEmbedding] = None
_sparse_model: Optional[Any] = None
SPARSE_AVAILABLE = True
except Exception: # noqa: BLE001
_sparse_model = None
@@ -20,6 +42,66 @@ except Exception: # noqa: BLE001
_http: Optional[httpx.AsyncClient] = None
@dataclass(frozen=True)
class EmbedNode:
"""Un endpoint embedding nella catena di fallback."""
name: str
url: str
api: str # "llamacpp" (/v1/embeddings) | "ollama" (/api/embed)
key: str
timeout_ms: int
def parse_chain(raw: str, default_api: str, default_url: str, default_key: str) -> list[EmbedNode]:
"""Parsa EMBED_CHAIN (JSON); vuota o invalida → endpoint legacy singolo."""
nodes: list[EmbedNode] = []
if raw:
try:
entries = json.loads(raw)
for entry in entries if isinstance(entries, list) else []:
if not isinstance(entry, dict) or not entry.get("url"):
continue
url = str(entry["url"]).rstrip("/")
api = str(entry.get("api") or "llamacpp")
if api not in ("llamacpp", "ollama") or not url.startswith(("http://", "https://")):
continue
nodes.append(
EmbedNode(
name=str(entry.get("name") or url),
url=url,
api=api,
key=str(entry.get("key") or ""),
timeout_ms=int(entry.get("timeout_ms", EMBED_TIMEOUT_MS)),
)
)
except (json.JSONDecodeError, TypeError, ValueError):
log.error("EMBED_CHAIN non è JSON valido: uso l'endpoint legacy")
if not nodes and default_url:
# Compatibilità legacy: endpoint singolo dagli env EMBED_*
nodes = [EmbedNode(name="embed", url=default_url.rstrip("/"), api=default_api, key=default_key, timeout_ms=EMBED_TIMEOUT_MS)]
return nodes
_chain: Optional[list[EmbedNode]] = None
_down_until: dict[str, float] = {} # url → monotonic deadline del cooldown
_http: Optional[httpx.AsyncClient] = None
def _get_chain() -> list[EmbedNode]:
global _chain
if _chain is None:
_chain = parse_chain(EMBED_CHAIN, EMBED_API, EMBED_URL, EMBED_API_KEY)
return _chain
def reset_chain_cache() -> None:
"""Forza il re-parse della catena (usato dai test)."""
global _chain
_chain = None
_down_until.clear()
def get_http() -> httpx.AsyncClient:
global _http
if _http is None:
@@ -34,17 +116,58 @@ async def close_http() -> None:
_http = None
def chain_nodes() -> list[EmbedNode]:
return _get_chain()
async def embed(text: str) -> list[float]:
if EMBED_API == "llamacpp":
"""Embedding con catena di fallback: ritorna il vettore o rilancia dopo l'ultimo fallimento."""
chain = _get_chain()
if not chain:
raise RuntimeError("nessun endpoint embedding configurato")
now = time.monotonic()
live = [n for n in chain if _down_until.get(n.url, 0) <= now]
if not live:
# tutti in cooldown: ritenta comunque il primo (meglio di un fallimento immediato)
live = [chain[0]]
payload = {"model": EMBED_MODEL, "input": text}
started = time.monotonic()
last_exc: Optional[Exception] = None
for node in live:
headers = {"Content-Type": "application/json"}
if EMBED_API_KEY:
headers["Authorization"] = f"Bearer {EMBED_API_KEY}"
response = await get_http().post(f"{EMBED_URL}/v1/embeddings", json={"model": EMBED_MODEL, "input": text}, headers=headers)
response.raise_for_status()
return response.json()["data"][0]["embedding"]
response = await get_http().post(f"{EMBED_URL}/api/embed", json={"model": EMBED_MODEL, "input": text})
response.raise_for_status()
return response.json()["embeddings"][0]
if node.key:
headers["Authorization"] = f"Bearer {node.key}"
path = "/v1/embeddings" if node.api == "llamacpp" else "/api/embed"
try:
t0 = time.monotonic()
response = await get_http().post(
f"{node.url}{path}",
json=payload,
headers=headers,
timeout=httpx.Timeout(node.timeout_ms / 1000.0),
)
response.raise_for_status()
data = response.json()
vector = data["data"][0]["embedding"] if node.api == "llamacpp" else data["embeddings"][0]
if len(vector) != EMBED_DIM:
raise ValueError(f"dimensione vettore {len(vector)} != EMBED_DIM {EMBED_DIM}")
took = int((time.monotonic() - started) * 1000)
metrics.record_embed(node.name, True, took)
return vector
except (httpx.HTTPError, ValueError, KeyError, IndexError, TypeError) as exc:
took = int((time.monotonic() - t0) * 1000)
_down_until[node.url] = time.monotonic() + EMBED_RETRY_COOLDOWN_S
last_exc = exc
metrics.record_embed(node.name, False, took)
log.warning(
"embed: nodo '%s' fallito dopo %dms (%s: %s) → cooldown %ds",
node.name,
took,
exc.__class__.__name__,
exc,
EMBED_RETRY_COOLDOWN_S,
)
raise RuntimeError(f"tutti i nodi embedding falliti ({len(live)} tentativi)") from last_exc
def get_sparse_model():
@@ -68,21 +191,32 @@ def backfill_sparse(qdrant: Any, collection: str) -> None:
offset: Any = None
updated = 0
while True:
points, next_offset = qdrant.scroll(collection_name=collection, limit=100, with_payload=["text"], with_vectors=True, offset=offset)
points, next_offset = qdrant.scroll(
collection_name=collection,
limit=100,
with_payload=["text"],
with_vectors=True,
offset=offset,
)
batch: list[qm.PointStruct] = []
for point in points:
vectors = point.vector or {}
if SPARSE_VECTOR_NAME in vectors:
for p in points:
vecs = p.vector or {}
if SPARSE_VECTOR_NAME in vecs:
continue
text = (point.payload or {}).get("text", "")
sparse = sparse_encode(text) if text else None
if sparse is not None:
batch.append(qm.PointStruct(id=point.id, vector={SPARSE_VECTOR_NAME: sparse}))
text = (p.payload or {}).get("text", "")
if not text:
continue
sparse = sparse_encode(text)
if sparse is None:
continue
batch.append(qm.PointStruct(id=p.id, vector={SPARSE_VECTOR_NAME: sparse}))
if batch:
# update_vectors: aggiorna SOLO il vettore sparso, preservando payload e vettore denso
# (upsert parziale sostituirebbe l'intero punto — incidente 2026-08-16)
qdrant.update_vectors(collection_name=collection, points=batch)
updated += len(batch)
if not next_offset:
break
offset = next_offset
if updated:
log.info("backfill sparse: %d record aggiornati", updated)
log.info("backfill sparse: %d record aggiornati", updated)
+13
View File
@@ -25,6 +25,11 @@ def record_rerank(backend: str, ok: bool, took_ms: int) -> None:
_metrics["rerank_duration_sum"][backend] += took_ms
def record_embed(backend: str, ok: bool, took_ms: int) -> None:
_metrics["embed_calls"][(backend, "ok" if ok else "fail")] += 1
_metrics["embed_duration_sum"][backend] += took_ms
def snapshot(qdrant: Any, collection: str) -> dict:
try:
points = qdrant.get_collection(collection).points_count
@@ -41,6 +46,9 @@ def snapshot(qdrant: Any, collection: str) -> dict:
"search_hits": _metrics["search_hits"],
"rerank_calls": {f"{backend}:{outcome}": count for (backend, outcome), count in _metrics["rerank_calls"].items()},
"rerank_avg_ms": {backend: round(total / _metrics["rerank_calls"][(backend, "ok")], 2) for backend, total in _metrics["rerank_duration_sum"].items() if _metrics["rerank_calls"][(backend, "ok")]},
"embed_calls": {f"{backend}:{outcome}": count for (backend, outcome), count in _metrics["embed_calls"].items()},
"embed_avg_ms": {backend: round(total / _metrics["embed_calls"][(backend, "ok")], 2) for backend, total in _metrics["embed_duration_sum"].items() if _metrics["embed_calls"][(backend, "ok")]},
"qdrant_retries": _metrics["qdrant_retries"],
"points": points,
}
@@ -61,6 +69,11 @@ def prometheus_lines(qdrant: Any, collection: str) -> list[str]:
lines.append(f'qmem_rerank_calls_total{{backend="{backend}",outcome="{outcome}"}} {count}')
for backend, s in _metrics["rerank_duration_sum"].items():
lines.append(f'qmem_rerank_duration_seconds_sum{{backend="{backend}"}} {s / 1000:.6f}')
for (backend, outcome), count in _metrics["embed_calls"].items():
lines.append(f'qmem_embed_calls_total{{backend="{backend}",outcome="{outcome}"}} {count}')
for backend, s in _metrics["embed_duration_sum"].items():
lines.append(f'qmem_embed_duration_seconds_sum{{backend="{backend}"}} {s / 1000:.6f}')
lines.append(f"qmem_qdrant_retries_total {_metrics['qdrant_retries']}")
try:
lines.append(f"qmem_points {qdrant.get_collection(collection).points_count}")
except Exception: # noqa: BLE001
+2 -1
View File
@@ -11,6 +11,7 @@ from fastapi import APIRouter, Depends, Header, HTTPException, Request
from qdrant_client.http import models as qm
import config
import embed
import guardrail
import metrics
import rerank
@@ -247,7 +248,7 @@ async def status(request: Request) -> dict:
@router.get("/v1/version")
async def version() -> dict:
return {"version": GATEWAY_VERSION, "git_commit": __import__("config").GIT_COMMIT, "guardrail_version": GUARDRAIL_VERSION, "guardrail_enabled": GUARDRAIL_ENABLED, "guardrail_block_threshold": GUARDRAIL_BLOCK_THRESHOLD, "guardrail_warn_threshold": GUARDRAIL_WARN_THRESHOLD, "embedding_model": EMBED_MODEL, "collection": COLLECTION, "rerank_enabled": rerank.enabled(), "rerank_model": config.RERANK_MODEL, "rerank_nodes": [n.name for n in rerank._get_chain()]}
return {"version": GATEWAY_VERSION, "git_commit": __import__("config").GIT_COMMIT, "guardrail_version": GUARDRAIL_VERSION, "guardrail_enabled": GUARDRAIL_ENABLED, "guardrail_block_threshold": GUARDRAIL_BLOCK_THRESHOLD, "guardrail_warn_threshold": GUARDRAIL_WARN_THRESHOLD, "embedding_model": EMBED_MODEL, "collection": COLLECTION, "embed_nodes": [n.name for n in embed.chain_nodes()], "rerank_enabled": rerank.enabled(), "rerank_model": config.RERANK_MODEL, "rerank_nodes": [n.name for n in rerank._get_chain()]}
@router.get("/v1/metrics")
+48 -2
View File
@@ -7,11 +7,57 @@ import json
import time
from typing import Any
import httpx
from qdrant_client import QdrantClient
from config import QDRANT_API_KEY, QDRANT_URL
from config import QDRANT_API_KEY, QDRANT_URL, QDRANT_RETRIES, log, _metrics
qdrant = QdrantClient(url=QDRANT_URL, api_key=QDRANT_API_KEY)
class ResilientQdrant:
"""Proxy del client Qdrant che ritenta i metodi su errori di transport
(connessione/timeout transienti, es. riavvio del container Qdrant).
Gli errori applicativi (404, validazione) non vengono ritentati."""
def __init__(self, client: Any, attempts: int = QDRANT_RETRIES, backoff_s: float = 0.4):
self._client = client
self._attempts = max(1, attempts)
self._backoff = backoff_s
@staticmethod
def _transient(exc: Exception) -> bool:
if isinstance(exc, httpx.TransportError):
return True
return type(exc).__name__ in ("ConnectionError", "TimeoutError")
def __getattr__(self, name: str) -> Any:
attr = getattr(self._client, name)
if not callable(attr):
return attr
def wrapped(*args: Any, **kwargs: Any) -> Any:
for attempt in range(self._attempts):
try:
return attr(*args, **kwargs)
except Exception as exc: # noqa: BLE001
if attempt == self._attempts - 1 or not self._transient(exc):
raise
delay = self._backoff * (2**attempt)
_metrics["qdrant_retries"] += 1
log.warning(
"qdrant.%s: errore transiente (%s: %s) → retry %d/%d tra %.1fs",
name,
exc.__class__.__name__,
exc,
attempt + 2,
self._attempts,
delay,
)
time.sleep(delay)
return wrapped
qdrant = ResilientQdrant(QdrantClient(url=QDRANT_URL, api_key=QDRANT_API_KEY))
request_id: contextvars.ContextVar[str] = contextvars.ContextVar("request_id", default="-")
ratelimit: dict[str, list[float]] = {}
status_ratelimit: dict[str, list[float]] = {}
+189
View File
@@ -0,0 +1,189 @@
"""Test della catena di fallback per gli embedding e del wrapper Qdrant resilient."""
from __future__ import annotations
import asyncio
import json
import httpx
import pytest
import embed as embed_mod
import state
from test_api import auth_headers, make_record
CHAIN = json.dumps(
[
{"name": "primario", "url": "http://primario:9001", "api": "llamacpp", "key": "k1", "timeout_ms": 500},
{"name": "fallback", "url": "http://fallback:9002", "api": "ollama", "key": "k2", "timeout_ms": 5000},
]
)
VEC_1024 = [0.01] * 1024
@pytest.fixture(autouse=True)
def _reset_chain(monkeypatch):
embed_mod.reset_chain_cache()
embed_mod._http = None
yield
embed_mod.reset_chain_cache()
embed_mod._http = None
def _mock_client(handler) -> list[str]:
calls: list[str] = []
def tracking_handler(request):
calls.append(f"{request.url.host}{request.url.path}")
return handler(request)
embed_mod._http = httpx.AsyncClient(transport=httpx.MockTransport(tracking_handler))
return calls
# ---------------------------------------------------------------------------
# parse della catena
# ---------------------------------------------------------------------------
def test_parse_chain_valida(monkeypatch):
monkeypatch.setattr(embed_mod, "EMBED_CHAIN", CHAIN)
embed_mod.reset_chain_cache()
nodes = embed_mod.chain_nodes()
assert [n.name for n in nodes] == ["primario", "fallback"]
assert [n.api for n in nodes] == ["llamacpp", "ollama"]
def test_parse_legacy_quando_catena_vuota(monkeypatch):
monkeypatch.setattr(embed_mod, "EMBED_CHAIN", "")
monkeypatch.setattr(embed_mod, "EMBED_API", "ollama")
monkeypatch.setattr(embed_mod, "EMBED_URL", "http://legacy:11434")
monkeypatch.setattr(embed_mod, "EMBED_API_KEY", "lk")
embed_mod.reset_chain_cache()
nodes = embed_mod.chain_nodes()
assert len(nodes) == 1
assert nodes[0].name == "embed"
assert nodes[0].api == "ollama"
assert nodes[0].url == "http://legacy:11434"
assert nodes[0].key == "lk"
def test_parse_chain_json_invalido_cade_su_legacy(monkeypatch):
monkeypatch.setattr(embed_mod, "EMBED_CHAIN", "non-json")
monkeypatch.setattr(embed_mod, "EMBED_URL", "http://legacy:11434")
embed_mod.reset_chain_cache()
assert [n.url for n in embed_mod.chain_nodes()] == ["http://legacy:11434"]
# ---------------------------------------------------------------------------
# fallback e cooldown
# ---------------------------------------------------------------------------
def test_fallback_primario_llamacpp_fallito(monkeypatch):
monkeypatch.setattr(embed_mod, "EMBED_CHAIN", CHAIN)
embed_mod.reset_chain_cache()
def handler(request):
if request.url.host == "primario":
return httpx.Response(500)
# nodo ollama-style: risposta con campo "embeddings"
return httpx.Response(200, json={"embeddings": [VEC_1024]})
embed_mod._http = httpx.AsyncClient(transport=httpx.MockTransport(handler))
vector = asyncio.run(embed_mod.embed("test"))
assert vector == VEC_1024
def test_cooldown_salta_primario(monkeypatch):
monkeypatch.setattr(embed_mod, "EMBED_CHAIN", CHAIN)
embed_mod.reset_chain_cache()
calls: list[str] = []
def tracking(request):
calls.append(request.url.host)
if request.url.host == "primario":
return httpx.Response(500)
return httpx.Response(200, json={"embeddings": [VEC_1024]})
embed_mod._http = httpx.AsyncClient(transport=httpx.MockTransport(tracking))
asyncio.run(embed_mod.embed("test"))
asyncio.run(embed_mod.embed("test"))
# il primario fallito entra in cooldown: la seconda chiamata lo salta
assert calls == ["primario", "fallback", "fallback"]
def test_dimensione_errata_salta_nodo(monkeypatch):
monkeypatch.setattr(embed_mod, "EMBED_CHAIN", CHAIN)
embed_mod.reset_chain_cache()
def handler(request):
if request.url.host == "primario":
return httpx.Response(200, json={"data": [{"embedding": [0.0] * 512}]}) # dim sbagliata
return httpx.Response(200, json={"embeddings": [VEC_1024]})
embed_mod._http = httpx.AsyncClient(transport=httpx.MockTransport(handler))
vector = asyncio.run(embed_mod.embed("test"))
assert len(vector) == 1024
def test_tutti_nodi_falliti_rilancia(monkeypatch):
monkeypatch.setattr(embed_mod, "EMBED_CHAIN", CHAIN)
embed_mod.reset_chain_cache()
embed_mod._http = httpx.AsyncClient(transport=httpx.MockTransport(lambda r: httpx.Response(500)))
with pytest.raises(RuntimeError, match="tutti i nodi embedding falliti"):
asyncio.run(embed_mod.embed("test"))
def test_empty_text_comunque_chiamata(monkeypatch):
# il modello pydantic valida già la query; qui verifichiamo il passthrough
monkeypatch.setattr(embed_mod, "EMBED_CHAIN", CHAIN)
embed_mod.reset_chain_cache()
def handler(request):
body = json.loads(request.content)
return httpx.Response(200, json={"embeddings": [VEC_1024] if body["input"] else []})
embed_mod._http = httpx.AsyncClient(transport=httpx.MockTransport(handler))
assert asyncio.run(embed_mod.embed("ok")) == VEC_1024
# ---------------------------------------------------------------------------
# ResilientQdrant (retry transiente)
# ---------------------------------------------------------------------------
class FlakyQdrant:
def __init__(self, failures: int, exc: Exception):
self.calls = 0
self.failures = failures
self.exc = exc
def upsert(self, **kw):
self.calls += 1
if self.calls <= self.failures:
raise self.exc
return "ok"
def test_retry_su_errore_transiente(monkeypatch):
fake = FlakyQdrant(2, httpx.ConnectError("conn"))
client = state.ResilientQdrant(fake, attempts=3, backoff_s=0.01)
assert client.upsert(x=1) == "ok"
assert fake.calls == 3
def test_niente_retry_su_errore_applicativo():
fake = FlakyQdrant(2, ValueError("404 logico"))
client = state.ResilientQdrant(fake, attempts=3, backoff_s=0.01)
with pytest.raises(ValueError):
client.upsert(x=1)
assert fake.calls == 1
def test_retry_esaurito_rilancia():
fake = FlakyQdrant(99, httpx.ReadTimeout("t"))
client = state.ResilientQdrant(fake, attempts=2, backoff_s=0.01)
with pytest.raises(httpx.ReadTimeout):
client.upsert(x=1)
assert fake.calls == 2