feat(proxy): T-X11 采纳 cortiq shadow 旁路评审——live T1 本地应答异步云端评审喂晋升表
- routes:live 分流 T1 响应后按 sample_rate 采样旁路——premium 条目以评审提示词 给本地答案打 PASS/FAIL,经 PromotionTable.observe 写入 T-X5 晋升表 (打通晋升质量信号的自动来源,夜间 labeler 之外的实时通路) - 零客户端延迟:asyncio.create_task 旁路 + wait_for 超时 + Semaphore(2) 并发上限; 上游故障/解析失败记 ERROR,不算 FAIL(评审器不可用不惩罚本地模型); 仅非流式响应采样(流式无同步答案文本),默认关闭(sense.shadow_review.enabled) - 模块级计数字典 sampled/pass/fail/error + reset_shadow_review_stats() 测试钩子 - 新增 tests/test_shadow_review.py 5 项(默认关/条目选取/PASS-FAIL-ERROR 解析/ 晋升表喂入/关闭态零副作用),全假上游不依赖真实模型
This commit is contained in:
@@ -0,0 +1,128 @@
|
||||
"""shadow 旁路评审单元测试(T-X11,采纳 cortiq shadow bypass 设计)。
|
||||
|
||||
全部走 monkeypatch 假上游,不依赖真实模型/API key。
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
import gateway.proxy.routes as R
|
||||
from gateway.proxy.routes import (_grade_answer, _pick_review_entry,
|
||||
_shadow_review_settings,
|
||||
reset_shadow_review_stats)
|
||||
from gateway.sense.promotion import PromotionTable, promotion_label
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset():
|
||||
reset_shadow_review_stats()
|
||||
yield
|
||||
reset_shadow_review_stats()
|
||||
|
||||
|
||||
def _fake_upstream(responses):
|
||||
"""构造假 upstream_stream:按调用顺序吐预定 verdict 文本(SSE 形态)。"""
|
||||
calls = {"n": 0}
|
||||
|
||||
def _stream(body, entry, sink, chain):
|
||||
async def gen():
|
||||
idx = min(calls["n"], len(responses) - 1)
|
||||
calls["n"] += 1
|
||||
verdict = responses[idx]
|
||||
chunk = {"choices": [{"delta": {"content": verdict}}]}
|
||||
yield (f"data: {json.dumps(chunk)}\n\n").encode("utf-8")
|
||||
yield b"data: [DONE]\n\n"
|
||||
return gen()
|
||||
|
||||
return _stream, calls
|
||||
|
||||
|
||||
def _entry(tier="premium", model="cloud-x"):
|
||||
return {"id": "e1", "enabled": True, "backend": "openai",
|
||||
"base_url": "http://127.0.0.1:9/v1", "model": model, "tier": tier}
|
||||
|
||||
|
||||
def test_settings_default_disabled_and_clamped():
|
||||
"""默认关闭;采样率/超时非法值被夹取。"""
|
||||
s = _shadow_review_settings({})
|
||||
assert s["enabled"] is False and s["sample_rate"] == 0.1
|
||||
assert _shadow_review_settings({"sense": {"shadow_review": {
|
||||
"enabled": True, "sample_rate": 5, "timeout_s": -3}}})["sample_rate"] == 1.0
|
||||
assert _shadow_review_settings({"sense": {"shadow_review": {
|
||||
"enabled": True, "timeout_s": -3}}})["timeout_s"] == 1.0
|
||||
|
||||
|
||||
def test_pick_review_entry_prefers_tier():
|
||||
"""评审条目优先指定档位,缺失时回退任意启用条目。"""
|
||||
pool_entries = [_entry("budget", "b1"), _entry("premium", "p1")]
|
||||
class _P:
|
||||
@staticmethod
|
||||
def usable_entries():
|
||||
return list(pool_entries)
|
||||
assert _pick_review_entry(_P, "premium")["model"] == "p1"
|
||||
assert _pick_review_entry(_P, "local")["model"] in ("b1", "p1")
|
||||
|
||||
|
||||
def test_grade_answer_parses_pass_fail_and_error(monkeypatch):
|
||||
"""PASS/FAIL 解析;上游异常回 ERROR 而非抛出。"""
|
||||
fake, calls = _fake_upstream(["PASS", "FAIL"])
|
||||
monkeypatch.setattr(R, "upstream_stream", fake)
|
||||
e = _entry()
|
||||
assert asyncio.run(_grade_answer(e, "问", "答", "cloud-x", 5)) == "PASS"
|
||||
assert asyncio.run(_grade_answer(e, "问", "答", "cloud-x", 5)) == "FAIL"
|
||||
|
||||
def _boom(*a, **k):
|
||||
async def gen():
|
||||
raise RuntimeError("上游炸了")
|
||||
yield b"" # pragma: no cover
|
||||
return gen()
|
||||
monkeypatch.setattr(R, "upstream_stream", _boom)
|
||||
assert asyncio.run(_grade_answer(e, "问", "答", "cloud-x", 5)) == "ERROR"
|
||||
assert calls["n"] == 2
|
||||
|
||||
|
||||
def test_shadow_review_task_feeds_promotion(tmp_path, monkeypatch):
|
||||
"""PASS/FAIL 写入晋升表(observe 计数),ERROR 只计错误不进通过率。"""
|
||||
from gateway.sense.store import SenseStore
|
||||
sstore = SenseStore.init_db(str(tmp_path / "sense.sqlite3"))
|
||||
promo = PromotionTable(sstore, now=lambda: 1_000_000.0)
|
||||
label = promotion_label("proxy", "")
|
||||
|
||||
fake, _ = _fake_upstream(["PASS", "PASS", "FAIL", "垃圾输出"])
|
||||
monkeypatch.setattr(R, "upstream_stream", fake)
|
||||
entry = _entry()
|
||||
sd = {"sense": {"shadow_review": {"enabled": True, "sample_rate": 1.0}}}
|
||||
|
||||
async def _run():
|
||||
for _ in range(4):
|
||||
await R._shadow_review_task("问题", "本地答案", _FixedPool(entry),
|
||||
sd, promo)
|
||||
|
||||
asyncio.run(_run())
|
||||
assert R._shadow_review_stats == {"sampled": 4, "pass": 2, "fail": 1,
|
||||
"error": 1, "skipped_busy": 0}
|
||||
row = sstore.get_promotion(label)
|
||||
assert row["n_total"] == 3 and row["n_ok"] == 2 # ERROR 不进通过率
|
||||
|
||||
|
||||
class _FixedPool:
|
||||
def __init__(self, entries):
|
||||
self._e = entries if isinstance(entries, list) else [entries]
|
||||
|
||||
def usable_entries(self):
|
||||
return list(self._e)
|
||||
|
||||
|
||||
def test_maybe_shadow_review_disabled_is_noop(monkeypatch):
|
||||
"""默认关闭:不建任务、不计采样。"""
|
||||
class _SP:
|
||||
@staticmethod
|
||||
def to_dict():
|
||||
return {"sense": {"shadow_review": {"enabled": False}}}
|
||||
|
||||
payload = json.dumps({"choices": [{"message": {"content": "本地答案"}}]}).encode()
|
||||
resp = type("JR", (), {"body": payload})()
|
||||
before = dict(R._shadow_review_stats)
|
||||
R._maybe_shadow_review({}, resp, _FixedPool(_entry()), _SP, scfg=None)
|
||||
assert R._shadow_review_stats == before
|
||||
Reference in New Issue
Block a user