feat(proxy): T-X11 采纳 cortiq shadow 旁路评审——live T1 本地应答异步云端评审喂晋升表

- routes:live 分流 T1 响应后按 sample_rate 采样旁路——premium 条目以评审提示词
  给本地答案打 PASS/FAIL,经 PromotionTable.observe 写入 T-X5 晋升表
  (打通晋升质量信号的自动来源,夜间 labeler 之外的实时通路)
- 零客户端延迟:asyncio.create_task 旁路 + wait_for 超时 + Semaphore(2) 并发上限;
  上游故障/解析失败记 ERROR,不算 FAIL(评审器不可用不惩罚本地模型);
  仅非流式响应采样(流式无同步答案文本),默认关闭(sense.shadow_review.enabled)
- 模块级计数字典 sampled/pass/fail/error + reset_shadow_review_stats() 测试钩子
- 新增 tests/test_shadow_review.py 5 项(默认关/条目选取/PASS-FAIL-ERROR 解析/
  晋升表喂入/关闭态零副作用),全假上游不依赖真实模型
This commit is contained in:
tzt
2026-09-19 10:03:38 +08:00
parent e5470a3715
commit f04a6c4c43
2 changed files with 284 additions and 0 deletions
+128
View File
@@ -0,0 +1,128 @@
"""shadow 旁路评审单元测试(T-X11,采纳 cortiq shadow bypass 设计)。
全部走 monkeypatch 假上游,不依赖真实模型/API key。
"""
import asyncio
import json
import pytest
import gateway.proxy.routes as R
from gateway.proxy.routes import (_grade_answer, _pick_review_entry,
_shadow_review_settings,
reset_shadow_review_stats)
from gateway.sense.promotion import PromotionTable, promotion_label
@pytest.fixture(autouse=True)
def _reset():
reset_shadow_review_stats()
yield
reset_shadow_review_stats()
def _fake_upstream(responses):
"""构造假 upstream_stream:按调用顺序吐预定 verdict 文本(SSE 形态)。"""
calls = {"n": 0}
def _stream(body, entry, sink, chain):
async def gen():
idx = min(calls["n"], len(responses) - 1)
calls["n"] += 1
verdict = responses[idx]
chunk = {"choices": [{"delta": {"content": verdict}}]}
yield (f"data: {json.dumps(chunk)}\n\n").encode("utf-8")
yield b"data: [DONE]\n\n"
return gen()
return _stream, calls
def _entry(tier="premium", model="cloud-x"):
return {"id": "e1", "enabled": True, "backend": "openai",
"base_url": "http://127.0.0.1:9/v1", "model": model, "tier": tier}
def test_settings_default_disabled_and_clamped():
"""默认关闭;采样率/超时非法值被夹取。"""
s = _shadow_review_settings({})
assert s["enabled"] is False and s["sample_rate"] == 0.1
assert _shadow_review_settings({"sense": {"shadow_review": {
"enabled": True, "sample_rate": 5, "timeout_s": -3}}})["sample_rate"] == 1.0
assert _shadow_review_settings({"sense": {"shadow_review": {
"enabled": True, "timeout_s": -3}}})["timeout_s"] == 1.0
def test_pick_review_entry_prefers_tier():
"""评审条目优先指定档位,缺失时回退任意启用条目。"""
pool_entries = [_entry("budget", "b1"), _entry("premium", "p1")]
class _P:
@staticmethod
def usable_entries():
return list(pool_entries)
assert _pick_review_entry(_P, "premium")["model"] == "p1"
assert _pick_review_entry(_P, "local")["model"] in ("b1", "p1")
def test_grade_answer_parses_pass_fail_and_error(monkeypatch):
"""PASS/FAIL 解析;上游异常回 ERROR 而非抛出。"""
fake, calls = _fake_upstream(["PASS", "FAIL"])
monkeypatch.setattr(R, "upstream_stream", fake)
e = _entry()
assert asyncio.run(_grade_answer(e, "", "", "cloud-x", 5)) == "PASS"
assert asyncio.run(_grade_answer(e, "", "", "cloud-x", 5)) == "FAIL"
def _boom(*a, **k):
async def gen():
raise RuntimeError("上游炸了")
yield b"" # pragma: no cover
return gen()
monkeypatch.setattr(R, "upstream_stream", _boom)
assert asyncio.run(_grade_answer(e, "", "", "cloud-x", 5)) == "ERROR"
assert calls["n"] == 2
def test_shadow_review_task_feeds_promotion(tmp_path, monkeypatch):
"""PASS/FAIL 写入晋升表(observe 计数),ERROR 只计错误不进通过率。"""
from gateway.sense.store import SenseStore
sstore = SenseStore.init_db(str(tmp_path / "sense.sqlite3"))
promo = PromotionTable(sstore, now=lambda: 1_000_000.0)
label = promotion_label("proxy", "")
fake, _ = _fake_upstream(["PASS", "PASS", "FAIL", "垃圾输出"])
monkeypatch.setattr(R, "upstream_stream", fake)
entry = _entry()
sd = {"sense": {"shadow_review": {"enabled": True, "sample_rate": 1.0}}}
async def _run():
for _ in range(4):
await R._shadow_review_task("问题", "本地答案", _FixedPool(entry),
sd, promo)
asyncio.run(_run())
assert R._shadow_review_stats == {"sampled": 4, "pass": 2, "fail": 1,
"error": 1, "skipped_busy": 0}
row = sstore.get_promotion(label)
assert row["n_total"] == 3 and row["n_ok"] == 2 # ERROR 不进通过率
class _FixedPool:
def __init__(self, entries):
self._e = entries if isinstance(entries, list) else [entries]
def usable_entries(self):
return list(self._e)
def test_maybe_shadow_review_disabled_is_noop(monkeypatch):
"""默认关闭:不建任务、不计采样。"""
class _SP:
@staticmethod
def to_dict():
return {"sense": {"shadow_review": {"enabled": False}}}
payload = json.dumps({"choices": [{"message": {"content": "本地答案"}}]}).encode()
resp = type("JR", (), {"body": payload})()
before = dict(R._shadow_review_stats)
R._maybe_shadow_review({}, resp, _FixedPool(_entry()), _SP, scfg=None)
assert R._shadow_review_stats == before