feat(sense): T-X5 云端评判晋升表×conformal 双闸门——本地档自动接管(采纳 cortiq promotion)

- gateway/sense/promotion.py:PromotionTable 状态机(candidate/promoted/demoted)
  * 晋升门:n_total>=n_min 且 通过率>=promote_lb 且 soak 浸泡期满足
  * 退化:promoted 期间通过率<demote_lb 自动降级;降级后可凭数据恢复
  * 一票否决:tier=T3 观察不进通过率统计,已晋升者立即降级
  * 持久化 sense_promotion 表(CREATE IF NOT EXISTS 幂等)
- grader:live 模式对已晋升标签的 T2 决策(无 T3 信号/非 fallback)升级 T1,
  hard_gates.promotion_applied 如实标注;T1/T3/fallback 路径不受影响
- 回填链路:T1 审计抽样带 promo:<label> 标签 -> 人工 verdict approve=ok
  经 /review/{id} 提交时回填晋升表(失败不影响审核主流程)
- 双闸门语义:conformal 阈值保证单条决策风险率,晋升表保证标签级接管节奏;
  collect 数据不足(n_min 未满)不晋升,全部迁移可经 T-X4 留痕审计

pytest 474 passed(T-X4 后 466 + 8)
This commit is contained in:
tzt
2026-09-18 23:01:52 +08:00
parent 7e01dc9b16
commit 1aaa05cacc
7 changed files with 334 additions and 5 deletions
+16
View File
@@ -451,6 +451,22 @@ try:
raise HTTPException(status_code=400, detail=str(e)) raise HTTPException(status_code=400, detail=str(e))
if not ok: if not ok:
raise HTTPException(status_code=404, detail=f"审核记录不存在或已审核: {review_id}") raise HTTPException(status_code=404, detail=f"审核记录不存在或已审核: {review_id}")
# 晋升表回填(T-X5):带 promo 标签的 sense 审计样本,人工 verdict
# approve=ok -> 晋升表累计;失败不影响审核主流程
try:
row = get_review().get(review_id)
promo = next((t[len("promo:"):] for t in (row.get("tags") or [])
if isinstance(t, str) and t.startswith("promo:")), None)
if promo:
from gateway.sense.config import build_sense_config
from gateway.sense.promotion import PromotionTable
from gateway.sense.store import SenseStore
scfg = build_sense_config(settings_store().to_dict())
if scfg.enabled:
PromotionTable(SenseStore.init_db(scfg.db_path)).observe(
promo, ok=(verdict == "approve"))
except Exception:
pass
return {"ok": True, "review_id": review_id, "verdict": verdict} return {"ok": True, "review_id": review_id, "verdict": verdict}
# ---------------- 模型池(多价位异构模型) ---------------- # ---------------- 模型池(多价位异构模型) ----------------
+8 -3
View File
@@ -93,11 +93,13 @@ def build_proxy_router(cfg: ProxyConfig, pool, settings_provider=None) -> APIRou
from gateway.sense.config import build_sense_config from gateway.sense.config import build_sense_config
from gateway.sense.grader import Grader from gateway.sense.grader import Grader
from gateway.sense.observer import get_observer from gateway.sense.observer import get_observer
from gateway.sense.promotion import PromotionTable
from gateway.sense.store import SenseStore from gateway.sense.store import SenseStore
scfg = build_sense_config(settings_provider().to_dict()) scfg = build_sense_config(settings_provider().to_dict())
if scfg.enabled and scfg.mode == "live": if scfg.enabled and scfg.mode == "live":
sstore = SenseStore.init_db(scfg.db_path) sstore = SenseStore.init_db(scfg.db_path)
grader = Grader(scfg, sstore, get_observer(sstore)) grader = Grader(scfg, sstore, get_observer(sstore),
promotion=PromotionTable(sstore))
qtext = "\n".join(str(m.get("content") or "") qtext = "\n".join(str(m.get("content") or "")
for m in (body.get("messages") or [])) for m in (body.get("messages") or []))
d = await grader.decide(qtext or str(body.get("model") or ""), d = await grader.decide(qtext or str(body.get("model") or ""),
@@ -114,14 +116,17 @@ def build_proxy_router(cfg: ProxyConfig, pool, settings_provider=None) -> APIRou
resp = await _run_chat(body, dict(request.headers), ctx, cfg, ledger, pool) resp = await _run_chat(body, dict(request.headers), ctx, cfg, ledger, pool)
if tier_used: if tier_used:
resp.headers["x-campus-tier"] = tier_used resp.headers["x-campus-tier"] = tier_used
# T1 审计抽样(§9.4):按 review.sample_rate 入队人工核 # T1 审计抽样(§9.4):按 review.sample_rate 入队人工核
# promo 标签供人工 verdict 回填晋升表(T-X5
try: try:
import random as _random import random as _random
from gateway.sense.promotion import promotion_label
rate = float(load_config().get("review", {}).get("sample_rate", 0.1)) rate = float(load_config().get("review", {}).get("sample_rate", 0.1))
if _random.random() < rate: if _random.random() < rate:
get_review().enqueue( get_review().enqueue(
request_id + "-sense", str(body.get("model") or "proxy"), request_id + "-sense", str(body.get("model") or "proxy"),
"(sense T1 审计抽样)", tags=["sense_t1"], "(sense T1 审计抽样)",
tags=["sense_t1", "promo:" + promotion_label("proxy", "")],
reason="sense_audit") reason="sense_audit")
except Exception: except Exception:
pass pass
+17 -2
View File
@@ -21,6 +21,7 @@ from gateway.sense.decision_cache import DecisionCache
from gateway.sense.embedder import embed from gateway.sense.embedder import embed
from gateway.sense.errors import EmbedderDown from gateway.sense.errors import EmbedderDown
from gateway.sense.features import gate from gateway.sense.features import gate
from gateway.sense.promotion import promotion_label
from gateway.sense.store import json_dumps from gateway.sense.store import json_dumps
@@ -99,10 +100,11 @@ def _rejected_tiers(fallback: bool, probs: Dict[str, float],
class Grader: class Grader:
"""决策器(持有 active 工件缓存;工件/阈值切换后调 invalidate)。""" """决策器(持有 active 工件缓存;工件/阈值切换后调 invalidate)。"""
def __init__(self, cfg: SenseConfig, store, observer=None): def __init__(self, cfg: SenseConfig, store, observer=None, promotion=None):
self.cfg = cfg self.cfg = cfg
self.store = store self.store = store
self.observer = observer self.observer = observer
self._promotion = promotion # T-X5:晋升表(可 None
self._head: Optional[LinearHead] = None self._head: Optional[LinearHead] = None
self._head_loaded = False self._head_loaded = False
self._thresholds: Optional[Dict[str, Any]] = None self._thresholds: Optional[Dict[str, Any]] = None
@@ -194,6 +196,18 @@ class Grader:
else: else:
tier = "T2" tier = "T2"
# ---- 晋升表应用(T-X5conformal 之外的统计闸门;仅 live----
# 仅对 T2 决策且无任何 T3 信号时,允许已晋升标签升级 T1;
# T1/T3 决策与 fallback 路径不受晋升表影响。
promotion_applied = False
if (self._promotion is not None and self.cfg.mode == "live"
and not fallback and tier == "T2"
and not feats.repo_signals
and probs.get("t3", 0.0) < float(th.get("t3", 0.9))):
if self._promotion.is_promoted(promotion_label(consumer, domain)):
tier = "T1"
promotion_applied = True
# ---- mode 裁剪(D-G7---- # ---- mode 裁剪(D-G7----
if self.cfg.mode == "live": if self.cfg.mode == "live":
executed = tier # live:决策即执行 executed = tier # live:决策即执行
@@ -208,7 +222,8 @@ class Grader:
"intent_blocked": feats.intent_blocked, "intent_blocked": feats.intent_blocked,
"repo_signals": feats.repo_signals, "repo_signals": feats.repo_signals,
"over_length": feats.over_length, "over_length": feats.over_length,
"t1_hard_ok": feats.t1_hard_ok}, "t1_hard_ok": feats.t1_hard_ok,
"promotion_applied": promotion_applied},
thresholds_version=th_version, head_version=head_version, thresholds_version=th_version, head_version=head_version,
mode=self.cfg.mode, fallback=fallback, mode=self.cfg.mode, fallback=fallback,
executed_tier=executed) executed_tier=executed)
+81
View File
@@ -0,0 +1,81 @@
"""云端评判式晋升表(T-X5,采纳 cortiq promotion.rs,与 conformal 双闸门)。
定位:conformal 阈值给出「分对率 >= 1-α」的统计保证(单条查询粒度),
本表在其之上给出「某类任务(label)可由本地档接管」的**节奏自动化**:
- 观察:人工核验 verdictapprove=ok)等质量信号按 label 累计通过率;
- 晋升:n_total >= n_min 且 通过率 >= promote_lb 且 soak 浸泡期满足
-> state=promoted(此后 grader 对该 label 的 T2 决策可升级 T1);
- 退化:promoted 期间通过率 < demote_lb -> demoted(自动回退分级路径);
- 一票否决:tier=T3(高复杂度/仓库级)观察不进通过率统计,且若已晋升
立即降级——与 cortiq "HIGH tier escalation is served by the cloud" 同构。
灰度纪律(沿用 D-G7):collect 数据不足(n_min 未满)不得晋升;
全部状态迁移可由 T-X4 的决策留痕审计。
"""
from __future__ import annotations
import time
from typing import Any, Dict, Optional
def promotion_label(consumer: str, domain: str = "") -> str:
"""晋升粒度:消费方×领域(domain 缺省归 general;调用方可传更细粒度)。"""
return f"{consumer or 'default'}:{domain or 'general'}"
class PromotionTable:
"""标签晋升状态机(candidate / promoted / demoted;持久化于 sense_promotion)。"""
def __init__(self, store, now=None, n_min: int = 20, promote_lb: float = 0.95,
soak_days: float = 3.0, demote_lb: float = 0.85,
high_tier_veto: bool = True):
self.store = store
self._now = now or (lambda: time.time())
self.n_min = max(1, int(n_min))
self.promote_lb = float(promote_lb)
self.demote_lb = float(demote_lb)
self.soak_s = float(soak_days) * 86400.0
self.high_tier_veto = bool(high_tier_veto)
def observe(self, label: str, ok: bool, tier: str = "",
ts: Optional[float] = None) -> Dict[str, Any]:
"""记录一次质量信号并推进状态机;返回迁移后的行。"""
ts = float(ts) if ts is not None else float(self._now())
row = self.store.get_promotion(label) or {
"label": label, "n_total": 0, "n_ok": 0, "state": "candidate",
"first_ts": int(ts), "promoted_ts": 0, "last_ts": 0}
# 一票否决:T3 观察不进通过率统计;已晋升者立即降级
if self.high_tier_veto and str(tier).upper() == "T3":
row["last_ts"] = int(ts)
if row["state"] == "promoted":
row["state"] = "demoted"
row["promoted_ts"] = 0
self.store.upsert_promotion(row)
return row
row["n_total"] = int(row["n_total"]) + 1
row["n_ok"] = int(row["n_ok"]) + (1 if ok else 0)
row["last_ts"] = int(ts)
rate = row["n_ok"] / row["n_total"] if row["n_total"] else 0.0
soaked = (ts - int(row["first_ts"])) >= self.soak_s
if row["state"] == "promoted":
if row["n_total"] >= self.n_min and rate < self.demote_lb:
row["state"] = "demoted" # 退化自动降级
row["promoted_ts"] = 0
else:
# candidate / demoted 共用同一晋升门(降级后可凭数据恢复)
if row["n_total"] >= self.n_min and rate >= self.promote_lb and soaked:
row["state"] = "promoted"
row["promoted_ts"] = int(ts)
self.store.upsert_promotion(row)
return row
def is_promoted(self, label: str) -> bool:
row = self.store.get_promotion(label)
return bool(row) and row["state"] == "promoted"
def snapshot(self, label: str) -> Optional[Dict[str, Any]]:
return self.store.get_promotion(label)
+32
View File
@@ -41,6 +41,14 @@ CREATE TABLE IF NOT EXISTS sense_decisions(
candidate_scores TEXT NOT NULL DEFAULT '{}', candidate_scores TEXT NOT NULL DEFAULT '{}',
rejected TEXT NOT NULL DEFAULT '[]'); rejected TEXT NOT NULL DEFAULT '[]');
CREATE INDEX IF NOT EXISTS idx_dec_ts ON sense_decisions(ts); CREATE INDEX IF NOT EXISTS idx_dec_ts ON sense_decisions(ts);
CREATE TABLE IF NOT EXISTS sense_promotion(
label TEXT PRIMARY KEY, n_total INTEGER NOT NULL DEFAULT 0,
n_ok INTEGER NOT NULL DEFAULT 0,
state TEXT NOT NULL DEFAULT 'candidate',
first_ts INTEGER NOT NULL DEFAULT 0,
promoted_ts INTEGER NOT NULL DEFAULT 0,
last_ts INTEGER NOT NULL DEFAULT 0);
""" """
@@ -178,6 +186,30 @@ class SenseStore:
out.append(d) out.append(d)
return out return out
# ---------- 晋升表(T-X5:本地档自动接管的双闸门之一) ----------
def upsert_promotion(self, row: Dict[str, Any]) -> None:
with self._lock, self._connect() as conn:
conn.execute(
"""INSERT OR REPLACE INTO sense_promotion
(label, n_total, n_ok, state, first_ts, promoted_ts, last_ts)
VALUES (?,?,?,?,?,?,?)""",
(row["label"], int(row.get("n_total", 0)),
int(row.get("n_ok", 0)), str(row.get("state", "candidate")),
int(row.get("first_ts", 0)), int(row.get("promoted_ts", 0)),
int(row.get("last_ts", 0))))
def get_promotion(self, label: str) -> Optional[Dict[str, Any]]:
with self._lock, self._connect() as conn:
row = conn.execute(
"SELECT * FROM sense_promotion WHERE label = ?", (label,)).fetchone()
return dict(row) if row else None
def list_promotions(self) -> List[Dict[str, Any]]:
with self._lock, self._connect() as conn:
rows = conn.execute(
"SELECT * FROM sense_promotion ORDER BY last_ts DESC").fetchall()
return [dict(r) for r in rows]
# ---------- 工件 ---------- # ---------- 工件 ----------
def register_artifact(self, version: str, kind: str, path: str, def register_artifact(self, version: str, kind: str, path: str,
metrics: Dict[str, Any], active: bool = False, metrics: Dict[str, Any], active: bool = False,
+178
View File
@@ -0,0 +1,178 @@
"""晋升表测试(T-X5):candidate→promoted→demoted→恢复状态机 + 一票否决 +
grader 晋升应用(T2→T1)与 T3/闸门互斥。"""
import asyncio
import pytest
from gateway.sense.config import build_sense_config
from gateway.sense.grader import Grader
from gateway.sense.promotion import PromotionTable, promotion_label
from gateway.sense.store import SenseStore
class _Clock:
def __init__(self, t=1789874000.0):
self.t = t
def __call__(self):
return self.t
def _table(tmp_path, **kw):
store = SenseStore.init_db(tmp_path / "s.sqlite3")
clock = _Clock()
table = PromotionTable(store, now=clock, **kw)
return store, clock, table
def test_promotion_requires_min_and_soak(tmp_path):
"""n_min 未满不晋升; soak 期未满不晋升;两者满足才 promoted。"""
store, clock, table = _table(tmp_path, n_min=5, promote_lb=0.9,
soak_days=3.0)
label = "proxy:general"
for _ in range(5):
table.observe(label, ok=True, ts=clock.t) # 通过率 1.0 但浸泡期未满
assert not table.is_promoted(label)
clock.t += 4 * 86400.0 # 浸泡期满足
table.observe(label, ok=True, ts=clock.t)
assert table.is_promoted(label)
def test_promotion_demotes_on_degradation(tmp_path):
"""promoted 后通过率跌破 demote_lb -> 自动降级。"""
store, clock, table = _table(tmp_path, n_min=5, promote_lb=0.9,
demote_lb=0.85, soak_days=0.0)
label = "proxy:general"
for _ in range(6):
table.observe(label, ok=True, ts=clock.t)
assert table.is_promoted(label)
for _ in range(6):
clock.t += 60
table.observe(label, ok=False, ts=clock.t) # 通过率跌至 6/12 = 0.5
assert not table.is_promoted(label)
assert table.snapshot(label)["state"] == "demoted"
def test_promotion_recovery_after_demotion(tmp_path):
"""demoted 后凭数据恢复:再次满足晋升门 -> promoted。"""
store, clock, table = _table(tmp_path, n_min=4, promote_lb=0.9,
demote_lb=0.85, soak_days=0.0)
label = "proxy:general"
for _ in range(4):
table.observe(label, ok=True, ts=clock.t)
assert table.is_promoted(label)
for _ in range(4):
clock.t += 60
table.observe(label, ok=False, ts=clock.t) # 跌破
assert table.snapshot(label)["state"] == "demoted"
for _ in range(20):
clock.t += 60
table.observe(label, ok=True, ts=clock.t) # 20/24 ≈ 0.83... 继续
# 24 条中 20 ok = 0.833 < 0.9,再补 ok 至 >0.9
for _ in range(6):
clock.t += 60
table.observe(label, ok=True, ts=clock.t)
# 30 条 26 ok ≈ 0.867,仍 <0.9;再补
for _ in range(6):
clock.t += 60
table.observe(label, ok=True, ts=clock.t)
# 36 条 32 ok ≈ 0.889;最后一条后 >= 0.9
table.observe(label, ok=True, ts=clock.t + 60) # 37 条 33 ok ≈ 0.892
for _ in range(4):
table.observe(label, ok=True, ts=clock.t + 120)
# 41 条 37 ok ≈ 0.902 >= 0.9
assert table.snapshot(label)["state"] == "promoted"
def test_high_tier_veto(tmp_path):
"""T3 观察:不计入通过率;已晋升立即降级。"""
store, clock, table = _table(tmp_path, n_min=3, promote_lb=0.9, soak_days=0.0)
label = "proxy:general"
for _ in range(3):
table.observe(label, ok=True, ts=clock.t)
assert table.is_promoted(label)
before = table.snapshot(label)["n_total"]
table.observe(label, ok=True, tier="T3", ts=clock.t + 10)
snap = table.snapshot(label)
assert snap["n_total"] == before # T3 不进统计
assert snap["state"] == "demoted" # 一票否决
def test_promotion_label_format():
assert promotion_label("proxy", "") == "proxy:general"
assert promotion_label("proxy", "code") == "proxy:code"
# ---------------- grader 晋升应用 ----------------
class _FakeHeadMid:
version = "test-head"
def predict(self, vec):
return {"t1": 0.50, "t2": 0.47, "t3": 0.03} # 低于阈值 -> default T2
class _NoopObserver:
def log(self, obs):
pass
def _make(tmp_path, table):
cfg = build_sense_config({"sense": {
"enabled": True, "mode": "live",
"db_path": str(tmp_path / "s.sqlite3"),
"models_dir": str(tmp_path / "models")}})
store = SenseStore.init_db(cfg.db_path)
g = Grader(cfg, store, _NoopObserver(), promotion=table)
g._load_head = lambda: _FakeHeadMid()
async def fake_embed(text, c):
return [1, 2, 3]
import gateway.sense.grader as gr
orig = gr.embed
gr.embed = fake_embed
return g, orig
def test_grader_applies_promotion_t2_to_t1(tmp_path):
"""已晋升标签:T2 决策升级 T1 且 hard_gates 标注 promotion_applied。"""
store = SenseStore.init_db(tmp_path / "s.sqlite3")
clock = _Clock()
table = PromotionTable(store, now=clock, n_min=2, promote_lb=0.9, soak_days=0.0)
label = promotion_label("proxy", "general")
table.observe(label, ok=True, ts=clock.t)
table.observe(label, ok=True, ts=clock.t)
g, orig = _make(tmp_path, table)
import gateway.sense.grader as gr
try:
d = asyncio.run(g.decide("什么是递归", "proxy"))
finally:
gr.embed = orig
assert d.tier == "T1"
assert d.hard_gates["promotion_applied"] is True
def test_grader_skips_promotion_when_not_promoted(tmp_path):
"""未晋升:同样的 T2 决策保持 T2。"""
store = SenseStore.init_db(tmp_path / "s2.sqlite3")
table = PromotionTable(store, now=_Clock(), n_min=100, soak_days=0.0)
g, orig = _make(tmp_path, table)
import gateway.sense.grader as gr
try:
d = asyncio.run(g.decide("什么是递归", "proxy"))
finally:
gr.embed = orig
assert d.tier == "T2"
assert d.hard_gates["promotion_applied"] is False
def test_grader_promotion_not_applied_without_table(tmp_path):
"""未接晋升表(向后兼容):行为与既有决策一致。"""
g, orig = _make(tmp_path, None)
import gateway.sense.grader as gr
try:
d = asyncio.run(g.decide("什么是递归", "proxy"))
finally:
gr.embed = orig
assert d.tier == "T2"
+2
View File
@@ -170,3 +170,5 @@ P0 完成后的能力:干净的后端抽象 + 可量化的评测 + 可追溯
| T-X3 | 路由决策缓存(外部采纳 cortiq 决策哈希缓存):DecisionCachesha256+LRU+TTL60s4096 条)前置 grader live 模式,embed+线性头去重;观察/审计不跳过、invalidate 同步清空、collect/shadow 不缓存 | ✅ 完成 | T-X3 | | T-X3 | 路由决策缓存(外部采纳 cortiq 决策哈希缓存):DecisionCachesha256+LRU+TTL60s4096 条)前置 grader live 模式,embed+线性头去重;观察/审计不跳过、invalidate 同步清空、collect/shadow 不缓存 | ✅ 完成 | T-X3 |
| T-X6 | 模型池能力位过滤(外部采纳 cortiq capabilities):条目 capabilities{vision,tools,context_window}(缺省全兼容)+ filter_by_capabilities 硬过滤 + 代理请求需求推断(图片→vision/tools/上下文)与重定向 | ✅ 完成 | T-X6 | | T-X6 | 模型池能力位过滤(外部采纳 cortiq capabilities):条目 capabilities{vision,tools,context_window}(缺省全兼容)+ filter_by_capabilities 硬过滤 + 代理请求需求推断(图片→vision/tools/上下文)与重定向 | ✅ 完成 | T-X6 |
| T-X4 | 可解释决策留痕(外部采纳 ai-model-router 决策三元):sense_decisions 表(reasons/candidate_scores/rejected+ grader 全模式落库 + /proxy/admin/sense-decisions 查询;附带修复 settings_provider 未注入导致 sense live 分流失效的潜伏 bug | ✅ 完成 | T-X4 | | T-X4 | 可解释决策留痕(外部采纳 ai-model-router 决策三元):sense_decisions 表(reasons/candidate_scores/rejected+ grader 全模式落库 + /proxy/admin/sense-decisions 查询;附带修复 settings_provider 未注入导致 sense live 分流失效的潜伏 bug | ✅ 完成 | T-X4 |
| T-X5 | 云端评判晋升表×conformal 双闸门(外部采纳 cortiq promotion):PromotionTable 状态机(n_min/通过率/soak 晋升、退化降级、T3 一票否决)+ grader live 模式 T2→T1 应用 + 人工核验 verdict 回填(promo 标签) | ✅ 完成 | T-X5 |
| T-X7~X9 | P2 立项不实施:多协议透传(/v1/messages)/ 工具输出清洗沙箱(受限 DSL)/ 网关内置 A/B 实验框架 | 📋 立项 | 待排期 |