enne2
fcd6b1670e
- embed.py: EMBED_CHAIN (JSON per-nodo {name,url,api,key,timeout_ms}, api
llamacpp|ollama), cooldown 60s sui nodi falliti, validazione dimensione
EMBED_DIM, compatibilità legacy quando la catena è vuota
- state.py: ResilientQdrant — proxy che ritenta i metodi del client Qdrant
su httpx.TransportError (store/search/transienti), errori applicativi
esenti; contatore qdrant_retries in metriche
- metrics: qmem_embed_calls_total + durata per backend
- /v1/version espone embed_nodes; versione 2.10.0
- test: 11 nuovi (chain, cooldown, dim mismatch, legacy, retry transiente) — 58 pass
95 lines
4.7 KiB
Python
95 lines
4.7 KiB
Python
"""Metriche in-memory e push Prometheus/VictoriaMetrics."""
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from typing import Any
|
|
|
|
from config import VM_PUSH_INTERVAL, VM_PUSH_URL, _metrics, log
|
|
|
|
|
|
def record_request(endpoint: str, duration: float, status_code: int) -> None:
|
|
_metrics["requests"][endpoint] += 1
|
|
_metrics["duration_sum"][endpoint] += duration
|
|
_metrics["duration_count"][endpoint] += 1
|
|
if status_code >= 400:
|
|
_metrics["errors"][(endpoint, status_code)] += 1
|
|
|
|
|
|
def record_search(hits: int) -> None:
|
|
_metrics["search_queries"] += 1
|
|
_metrics["search_hits"] += hits
|
|
|
|
|
|
def record_rerank(backend: str, ok: bool, took_ms: int) -> None:
|
|
_metrics["rerank_calls"][(backend, "ok" if ok else "fail")] += 1
|
|
_metrics["rerank_duration_sum"][backend] += took_ms
|
|
|
|
|
|
def record_embed(backend: str, ok: bool, took_ms: int) -> None:
|
|
_metrics["embed_calls"][(backend, "ok" if ok else "fail")] += 1
|
|
_metrics["embed_duration_sum"][backend] += took_ms
|
|
|
|
|
|
def snapshot(qdrant: Any, collection: str) -> dict:
|
|
try:
|
|
points = qdrant.get_collection(collection).points_count
|
|
except Exception: # noqa: BLE001
|
|
points = None
|
|
return {
|
|
"requests": dict(_metrics["requests"]),
|
|
"avg_duration_ms": {
|
|
endpoint: round(_metrics["duration_sum"][endpoint] / _metrics["duration_count"][endpoint] * 1000, 2)
|
|
for endpoint in _metrics["duration_count"]
|
|
},
|
|
"errors": {f"{endpoint}:{status}": count for (endpoint, status), count in _metrics["errors"].items()},
|
|
"search_queries": _metrics["search_queries"],
|
|
"search_hits": _metrics["search_hits"],
|
|
"rerank_calls": {f"{backend}:{outcome}": count for (backend, outcome), count in _metrics["rerank_calls"].items()},
|
|
"rerank_avg_ms": {backend: round(total / _metrics["rerank_calls"][(backend, "ok")], 2) for backend, total in _metrics["rerank_duration_sum"].items() if _metrics["rerank_calls"][(backend, "ok")]},
|
|
"embed_calls": {f"{backend}:{outcome}": count for (backend, outcome), count in _metrics["embed_calls"].items()},
|
|
"embed_avg_ms": {backend: round(total / _metrics["embed_calls"][(backend, "ok")], 2) for backend, total in _metrics["embed_duration_sum"].items() if _metrics["embed_calls"][(backend, "ok")]},
|
|
"qdrant_retries": _metrics["qdrant_retries"],
|
|
"points": points,
|
|
}
|
|
|
|
|
|
def prometheus_lines(qdrant: Any, collection: str) -> list[str]:
|
|
lines: list[str] = []
|
|
for endpoint, count in _metrics["requests"].items():
|
|
lines.append(f'qmem_requests_total{{endpoint="{endpoint}"}} {count}')
|
|
for endpoint, total in _metrics["duration_sum"].items():
|
|
count = _metrics["duration_count"][endpoint]
|
|
lines.append(f'qmem_request_duration_seconds_sum{{endpoint="{endpoint}"}} {total:.6f}')
|
|
lines.append(f'qmem_request_duration_seconds_count{{endpoint="{endpoint}"}} {count}')
|
|
for (endpoint, status), count in _metrics["errors"].items():
|
|
lines.append(f'qmem_errors_total{{endpoint="{endpoint}",status="{status}"}} {count}')
|
|
lines.append(f"qmem_search_queries_total {_metrics['search_queries']}")
|
|
lines.append(f"qmem_search_hits_total {_metrics['search_hits']}")
|
|
for (backend, outcome), count in _metrics["rerank_calls"].items():
|
|
lines.append(f'qmem_rerank_calls_total{{backend="{backend}",outcome="{outcome}"}} {count}')
|
|
for backend, s in _metrics["rerank_duration_sum"].items():
|
|
lines.append(f'qmem_rerank_duration_seconds_sum{{backend="{backend}"}} {s / 1000:.6f}')
|
|
for (backend, outcome), count in _metrics["embed_calls"].items():
|
|
lines.append(f'qmem_embed_calls_total{{backend="{backend}",outcome="{outcome}"}} {count}')
|
|
for backend, s in _metrics["embed_duration_sum"].items():
|
|
lines.append(f'qmem_embed_duration_seconds_sum{{backend="{backend}"}} {s / 1000:.6f}')
|
|
lines.append(f"qmem_qdrant_retries_total {_metrics['qdrant_retries']}")
|
|
try:
|
|
lines.append(f"qmem_points {qdrant.get_collection(collection).points_count}")
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
return lines
|
|
|
|
|
|
async def push_loop(qdrant: Any, collection: str, get_http) -> None:
|
|
while True:
|
|
try:
|
|
now_ms = int(time.time() * 1000)
|
|
body = "\n".join(f"{line} {now_ms}" for line in prometheus_lines(qdrant, collection)) + "\n"
|
|
response = await get_http().post(VM_PUSH_URL, content=body, headers={"Content-Type": "text/plain"})
|
|
if response.status_code >= 300:
|
|
log.warning("metrics push: HTTP %s", response.status_code)
|
|
except Exception as exc: # noqa: BLE001
|
|
log.warning("metrics push error: %s", exc)
|
|
await __import__("asyncio").sleep(VM_PUSH_INTERVAL)
|