252 lines
10 KiB
Python
252 lines
10 KiB
Python
"""Test strategie rerank oltre la search: gate store (A), supersede verify (B),
|
|
score composito (C), multi-query (E), primitiva /v1/score."""
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from conftest import auth_headers, make_record
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def enable_guardrail(monkeypatch):
|
|
import config
|
|
|
|
monkeypatch.setattr(config, "GUARDRAIL_ENABLED", True)
|
|
|
|
|
|
def _patch_rerank(monkeypatch, scores, backend="finto", took=10):
|
|
"""Abilita il rerank anche nel guardrail (che importa i valori da config)."""
|
|
monkeypatch.setattr("rerank.enabled", lambda: True)
|
|
monkeypatch.setattr("guardrail.GUARDRAIL_RERANK", True)
|
|
recorded: dict = {}
|
|
|
|
async def fake_rerank(query, docs):
|
|
recorded["query"] = query
|
|
recorded["docs"] = list(docs)
|
|
return scores, backend, took
|
|
|
|
monkeypatch.setattr("rerank.rerank", fake_rerank)
|
|
return recorded
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# A: gate store con cross-encoder
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_cosine_alto_cross_basso_downgrade_a_warn(client, monkeypatch):
|
|
"""Cosine 0.9 (zona BLOCK) ma cross-score basso → WARN CROSS_DUP_WEAK, salva."""
|
|
_patch_rerank(monkeypatch, scores=[-6.0]) # sigmoid ≈ 0.0024
|
|
client.fake_qdrant.query_score = 0.9
|
|
client.post("/v1/memories", json=make_record(text="primo record"), headers=auth_headers())
|
|
r2 = client.post("/v1/memories", json=make_record(text="secondo simile"), headers=auth_headers())
|
|
assert r2.status_code == 200
|
|
saved = client.fake_qdrant.points[r2.json()["memory_id"]].payload
|
|
assert saved["guardrail"]["decision"] == "WARN"
|
|
assert saved["guardrail"]["reason"] == "CROSS_DUP_WEAK"
|
|
|
|
|
|
def test_cosine_warn_cross_alto_upgrade_a_block(client, monkeypatch):
|
|
"""Cosine in zona WARN (0.75) ma cross-score altissimo → BLOCK parafrasato catturato."""
|
|
import rerank as rr_mod
|
|
monkeypatch.setattr("guardrail.GUARDRAIL_RERANK", True)
|
|
monkeypatch.setattr("rerank.enabled", lambda: True)
|
|
|
|
async def fake_rerank(query, docs):
|
|
return [3.0], "finto", 5 # sigmoid ≈ 0.953 ≥ 0.88
|
|
|
|
monkeypatch.setattr("rerank.rerank", fake_rerank)
|
|
client.fake_qdrant.query_score = 0.75
|
|
client.post("/v1/memories", json=make_record(text="primo record"), headers=auth_headers())
|
|
r2 = client.post("/v1/memories", json=make_record(text="stesso fatto riformulato"), headers=auth_headers())
|
|
assert r2.status_code == 409
|
|
assert r2.json()["detail"]["reason"] == "CROSS_DUP_CONFIRMED"
|
|
|
|
|
|
def test_cosine_basso_cross_alto_block_low_cosine(client, monkeypatch):
|
|
"""Cosine sotto soglia WARN (0.5) ma cross altissimo → CROSS_DUP_LOW_COSINE."""
|
|
monkeypatch.setattr("guardrail.GUARDRAIL_RERANK", True)
|
|
monkeypatch.setattr("rerank.enabled", lambda: True)
|
|
|
|
async def fake_rerank(query, docs):
|
|
return [4.0], "finto", 5 # sigmoid ≈ 0.982
|
|
|
|
monkeypatch.setattr("rerank.rerank", fake_rerank)
|
|
client.fake_qdrant.query_score = 0.5
|
|
client.post("/v1/memories", json=make_record(text="primo record"), headers=auth_headers())
|
|
r2 = client.post("/v1/memories", json=make_record(text="secondo riformulato"), headers=auth_headers())
|
|
assert r2.status_code == 409
|
|
assert r2.json()["detail"]["reason"] == "CROSS_DUP_LOW_COSINE"
|
|
|
|
|
|
def test_warn_con_suggerimento_supersedes(client, monkeypatch):
|
|
"""Cosine in zona WARN, cross ≥ soglia suggest → WARN con suggestion.supersedes_id."""
|
|
monkeypatch.setattr("guardrail.GUARDRAIL_RERANK", True)
|
|
monkeypatch.setattr("rerank.enabled", lambda: True)
|
|
|
|
async def fake_rerank(query, docs):
|
|
return [1.8], "finto", 5 # sigmoid ≈ 0.858: ≥ 0.85 (suggest), < 0.90 (block)
|
|
|
|
monkeypatch.setattr("rerank.rerank", fake_rerank)
|
|
client.fake_qdrant.query_score = 0.75
|
|
r1 = client.post("/v1/memories", json=make_record(text="primo record"), headers=auth_headers())
|
|
old_id = r1.json()["memory_id"]
|
|
r2 = client.post("/v1/memories", json=make_record(text="secondo simile"), headers=auth_headers())
|
|
assert r2.status_code == 200
|
|
saved = client.fake_qdrant.points[r2.json()["memory_id"]].payload
|
|
assert saved["guardrail"]["reason"] == "MODERATE_SIMILARITY"
|
|
assert saved["guardrail"]["suggestion"]["supersedes_id"] == old_id
|
|
|
|
|
|
def test_rerank_giu_degrada_a_solo_cosine(client, monkeypatch):
|
|
"""Reranker irraggiungibile → decisione legacy per cosine (BLOCK a 0.9)."""
|
|
monkeypatch.setattr("guardrail.GUARDRAIL_RERANK", True)
|
|
monkeypatch.setattr("rerank.enabled", lambda: True)
|
|
|
|
async def fail_rerank(query, docs):
|
|
return None
|
|
|
|
monkeypatch.setattr("rerank.rerank", fail_rerank)
|
|
client.fake_qdrant.query_score = 0.9
|
|
client.post("/v1/memories", json=make_record(text="primo record"), headers=auth_headers())
|
|
r2 = client.post("/v1/memories", json=make_record(text="secondo simile"), headers=auth_headers())
|
|
assert r2.status_code == 409
|
|
assert r2.json()["detail"]["reason"] == "KNOWN_SOLUTION"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# B: verifica supersede
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_supersede_cross_basso_warning(client, monkeypatch):
|
|
monkeypatch.setattr("config.GUARDRAIL_SUPERSEDE_CHECK", True)
|
|
monkeypatch.setattr("rerank.enabled", lambda: True)
|
|
|
|
async def fake_rerank(query, docs):
|
|
return [-8.0], "finto", 5 # sigmoid ≈ 0.0003 < 0.50
|
|
|
|
monkeypatch.setattr("rerank.rerank", fake_rerank)
|
|
r1 = client.post("/v1/memories", json=make_record(text="record originale"), headers=auth_headers())
|
|
old_id = r1.json()["memory_id"]
|
|
r2 = client.post(
|
|
"/v1/memories",
|
|
json=make_record(text="contenuto del tutto diverso", supersedes_id=old_id, supersede_reason="fix"),
|
|
headers=auth_headers(),
|
|
)
|
|
assert r2.status_code == 200
|
|
data = r2.json()
|
|
assert data["supersede_warning"]["cross_score"] < 0.1
|
|
assert "lineage" in data["supersede_warning"]["message"]
|
|
|
|
|
|
def test_supersede_cross_alto_nessun_warning(client, monkeypatch):
|
|
monkeypatch.setattr("config.GUARDRAIL_SUPERSEDE_CHECK", True)
|
|
monkeypatch.setattr("rerank.enabled", lambda: True)
|
|
|
|
async def fake_rerank(query, docs):
|
|
return [3.0], "finto", 5 # sigmoid ≈ 0.95 ≥ 0.50
|
|
|
|
monkeypatch.setattr("rerank.rerank", fake_rerank)
|
|
r1 = client.post("/v1/memories", json=make_record(text="record originale"), headers=auth_headers())
|
|
old_id = r1.json()["memory_id"]
|
|
r2 = client.post(
|
|
"/v1/memories",
|
|
json=make_record(text="record originale corretto", supersedes_id=old_id, supersede_reason="correzione"),
|
|
headers=auth_headers(),
|
|
)
|
|
assert r2.status_code == 200
|
|
assert "supersede_warning" not in r2.json()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# C: score composito
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_composite_score_in_risultati(client, monkeypatch):
|
|
client.fake_qdrant.query_score = 0.5 # il guardrail cosine non blocca il seeding
|
|
client.post("/v1/memories", json=make_record(text="record alpha", importance=1.0, confidence="high"), headers=auth_headers())
|
|
client.post("/v1/memories", json=make_record(text="record beta", importance=0.0, confidence="low"), headers=auth_headers())
|
|
|
|
async def fake_rerank(query, docs):
|
|
return [2.0, 2.0], "finto", 5 # rerank in parità → il composito decide
|
|
|
|
monkeypatch.setattr("rerank.enabled", lambda: True)
|
|
monkeypatch.setattr("rerank.rerank", fake_rerank)
|
|
|
|
resp = client.post("/v1/memories:search", json={"query": "record", "top_k": 2}, headers=auth_headers())
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
results = data["results"]
|
|
assert all("composite_score" in r for r in results)
|
|
assert results[0]["text"] == "record alpha"
|
|
assert results[0]["composite_score"] > results[1]["composite_score"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# E: multi-query
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_multi_query_pool_unito(client, monkeypatch):
|
|
client.fake_qdrant.query_score = 0.5 # il guardrail cosine non blocca il seeding
|
|
for text in ["alpha", "beta", "gamma"]:
|
|
client.post("/v1/memories", json=make_record(text=text), headers=auth_headers())
|
|
|
|
embed_calls: list[str] = []
|
|
|
|
async def fake_embed(text):
|
|
embed_calls.append(text)
|
|
return [0.0] * 1024
|
|
|
|
monkeypatch.setattr("state.embed", fake_embed)
|
|
monkeypatch.setattr("state.sparse_encode", lambda text: None)
|
|
|
|
async def fake_rerank(query, docs):
|
|
assert query == "alpha" # il rerank usa la query principale
|
|
return [0.9, 0.5, 0.1], "finto", 5
|
|
|
|
monkeypatch.setattr("rerank.enabled", lambda: True)
|
|
monkeypatch.setattr("rerank.rerank", fake_rerank)
|
|
|
|
resp = client.post(
|
|
"/v1/memories:search",
|
|
json={"query": "alpha", "queries": ["gamma", "alpha "], "top_k": 3},
|
|
headers=auth_headers(),
|
|
)
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert data["rerank"]["queries_used"] == 2 # "alpha " normalizzata e deduplicata
|
|
assert len(embed_calls) == 2
|
|
assert set(r["text"] for r in data["results"]) == {"alpha", "beta", "gamma"}
|
|
assert data["results"][0]["text"] == "alpha"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Primitiva /v1/score
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_score_endpoint_ok(client, monkeypatch):
|
|
async def fake_rerank(query, docs):
|
|
assert query == "q"
|
|
return [2.0, -3.0], "finto", 7
|
|
|
|
monkeypatch.setattr("rerank.enabled", lambda: True)
|
|
monkeypatch.setattr("rerank.rerank", fake_rerank)
|
|
resp = client.post("/v1/score", json={"query": "q", "documents": ["a", "b"]}, headers=auth_headers())
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert data["scores"][0] == pytest.approx(0.88, abs=0.01)
|
|
assert data["scores"][1] < 0.1
|
|
assert data["backend"] == "finto"
|
|
|
|
|
|
def test_score_endpoint_503_se_catena_giu(client, monkeypatch):
|
|
async def fail_rerank(query, docs):
|
|
return None
|
|
|
|
monkeypatch.setattr("rerank.enabled", lambda: True)
|
|
monkeypatch.setattr("rerank.rerank", fail_rerank)
|
|
resp = client.post("/v1/score", json={"query": "q", "documents": ["a"]}, headers=auth_headers())
|
|
assert resp.status_code == 503 |