tzt
f04a6c4c43
- routes：live 分流 T1 响应后按 sample_rate 采样旁路——premium 条目以评审提示词 给本地答案打 PASS/FAIL，经 PromotionTable.observe 写入 T-X5 晋升表 （打通晋升质量信号的自动来源，夜间 labeler 之外的实时通路） - 零客户端延迟：asyncio.create_task 旁路 + wait_for 超时 + Semaphore(2) 并发上限； 上游故障/解析失败记 ERROR，不算 FAIL（评审器不可用不惩罚本地模型）； 仅非流式响应采样（流式无同步答案文本），默认关闭（sense.shadow_review.enabled） - 模块级计数字典 sampled/pass/fail/error + reset_shadow_review_stats() 测试钩子 - 新增 tests/test_shadow_review.py 5 项（默认关/条目选取/PASS-FAIL-ERROR 解析/ 晋升表喂入/关闭态零副作用），全假上游不依赖真实模型
129 lines
4.7 KiB
Python
129 lines
4.7 KiB
Python
"""shadow 旁路评审单元测试（T-X11，采纳 cortiq shadow bypass 设计）。
|
|
|
|
全部走 monkeypatch 假上游，不依赖真实模型/API key。
|
|
"""
|
|
import asyncio
|
|
import json
|
|
|
|
import pytest
|
|
|
|
import gateway.proxy.routes as R
|
|
from gateway.proxy.routes import (_grade_answer, _pick_review_entry,
|
|
_shadow_review_settings,
|
|
reset_shadow_review_stats)
|
|
from gateway.sense.promotion import PromotionTable, promotion_label
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset():
|
|
reset_shadow_review_stats()
|
|
yield
|
|
reset_shadow_review_stats()
|
|
|
|
|
|
def _fake_upstream(responses):
|
|
"""构造假 upstream_stream：按调用顺序吐预定 verdict 文本（SSE 形态）。"""
|
|
calls = {"n": 0}
|
|
|
|
def _stream(body, entry, sink, chain):
|
|
async def gen():
|
|
idx = min(calls["n"], len(responses) - 1)
|
|
calls["n"] += 1
|
|
verdict = responses[idx]
|
|
chunk = {"choices": [{"delta": {"content": verdict}}]}
|
|
yield (f"data: {json.dumps(chunk)}\n\n").encode("utf-8")
|
|
yield b"data: [DONE]\n\n"
|
|
return gen()
|
|
|
|
return _stream, calls
|
|
|
|
|
|
def _entry(tier="premium", model="cloud-x"):
|
|
return {"id": "e1", "enabled": True, "backend": "openai",
|
|
"base_url": "http://127.0.0.1:9/v1", "model": model, "tier": tier}
|
|
|
|
|
|
def test_settings_default_disabled_and_clamped():
|
|
"""默认关闭；采样率/超时非法值被夹取。"""
|
|
s = _shadow_review_settings({})
|
|
assert s["enabled"] is False and s["sample_rate"] == 0.1
|
|
assert _shadow_review_settings({"sense": {"shadow_review": {
|
|
"enabled": True, "sample_rate": 5, "timeout_s": -3}}})["sample_rate"] == 1.0
|
|
assert _shadow_review_settings({"sense": {"shadow_review": {
|
|
"enabled": True, "timeout_s": -3}}})["timeout_s"] == 1.0
|
|
|
|
|
|
def test_pick_review_entry_prefers_tier():
|
|
"""评审条目优先指定档位，缺失时回退任意启用条目。"""
|
|
pool_entries = [_entry("budget", "b1"), _entry("premium", "p1")]
|
|
class _P:
|
|
@staticmethod
|
|
def usable_entries():
|
|
return list(pool_entries)
|
|
assert _pick_review_entry(_P, "premium")["model"] == "p1"
|
|
assert _pick_review_entry(_P, "local")["model"] in ("b1", "p1")
|
|
|
|
|
|
def test_grade_answer_parses_pass_fail_and_error(monkeypatch):
|
|
"""PASS/FAIL 解析；上游异常回 ERROR 而非抛出。"""
|
|
fake, calls = _fake_upstream(["PASS", "FAIL"])
|
|
monkeypatch.setattr(R, "upstream_stream", fake)
|
|
e = _entry()
|
|
assert asyncio.run(_grade_answer(e, "问", "答", "cloud-x", 5)) == "PASS"
|
|
assert asyncio.run(_grade_answer(e, "问", "答", "cloud-x", 5)) == "FAIL"
|
|
|
|
def _boom(*a, **k):
|
|
async def gen():
|
|
raise RuntimeError("上游炸了")
|
|
yield b"" # pragma: no cover
|
|
return gen()
|
|
monkeypatch.setattr(R, "upstream_stream", _boom)
|
|
assert asyncio.run(_grade_answer(e, "问", "答", "cloud-x", 5)) == "ERROR"
|
|
assert calls["n"] == 2
|
|
|
|
|
|
def test_shadow_review_task_feeds_promotion(tmp_path, monkeypatch):
|
|
"""PASS/FAIL 写入晋升表（observe 计数），ERROR 只计错误不进通过率。"""
|
|
from gateway.sense.store import SenseStore
|
|
sstore = SenseStore.init_db(str(tmp_path / "sense.sqlite3"))
|
|
promo = PromotionTable(sstore, now=lambda: 1_000_000.0)
|
|
label = promotion_label("proxy", "")
|
|
|
|
fake, _ = _fake_upstream(["PASS", "PASS", "FAIL", "垃圾输出"])
|
|
monkeypatch.setattr(R, "upstream_stream", fake)
|
|
entry = _entry()
|
|
sd = {"sense": {"shadow_review": {"enabled": True, "sample_rate": 1.0}}}
|
|
|
|
async def _run():
|
|
for _ in range(4):
|
|
await R._shadow_review_task("问题", "本地答案", _FixedPool(entry),
|
|
sd, promo)
|
|
|
|
asyncio.run(_run())
|
|
assert R._shadow_review_stats == {"sampled": 4, "pass": 2, "fail": 1,
|
|
"error": 1, "skipped_busy": 0}
|
|
row = sstore.get_promotion(label)
|
|
assert row["n_total"] == 3 and row["n_ok"] == 2 # ERROR 不进通过率
|
|
|
|
|
|
class _FixedPool:
|
|
def __init__(self, entries):
|
|
self._e = entries if isinstance(entries, list) else [entries]
|
|
|
|
def usable_entries(self):
|
|
return list(self._e)
|
|
|
|
|
|
def test_maybe_shadow_review_disabled_is_noop(monkeypatch):
|
|
"""默认关闭：不建任务、不计采样。"""
|
|
class _SP:
|
|
@staticmethod
|
|
def to_dict():
|
|
return {"sense": {"shadow_review": {"enabled": False}}}
|
|
|
|
payload = json.dumps({"choices": [{"message": {"content": "本地答案"}}]}).encode()
|
|
resp = type("JR", (), {"body": payload})()
|
|
before = dict(R._shadow_review_stats)
|
|
R._maybe_shadow_review({}, resp, _FixedPool(_entry()), _SP, scfg=None)
|
|
assert R._shadow_review_stats == before
|