tzt
324c419c35
- labeler.py：derive_true_tier 纯函数（规则0 人工优先 / plan_multi->T3 / 失败->下一档(T3保持) / 成功->executed / 空结果跳过）+ derive_true_tiers 批量回填 + 180d 留存清理 - calibrate.py：compute_thresholds（τ1/τ3 分组扫描——并列分数整组判定， 精度 >= 1-α 的最大覆盖阈值；<min_labels -> ok=False 沿用 last-good）； save_thresholds 工件落盘+登记；load_active 回退链 active->同kind扫描->last-good->保守值 - store：all_observations/list_artifacts 支撑方法 - 测试 +9（四规则/批量清理/覆盖达标/标签不足/工件往返与回退），全量 395 passed
113 lines
4.8 KiB
Python
113 lines
4.8 KiB
Python
"""标签+校准测试（T-G3）：四条推导规则 / T3 保持 / 180d 清理 / conformal 覆盖 / last-good。"""
|
|
import json
|
|
import time
|
|
|
|
from gateway.sense.calibrate import (
|
|
compute_thresholds,
|
|
load_active,
|
|
save_thresholds,
|
|
)
|
|
from gateway.sense.labeler import derive_true_tier, derive_true_tiers
|
|
from gateway.sense.store import SenseStore
|
|
|
|
|
|
def _obs(rid, executed, outcome, probs=None, features=None, override="", true_tier=""):
|
|
return {"request_id": rid, "consumer": "pipeline", "decided_tier": executed,
|
|
"executed_tier": executed, "probs": probs or "{}", "outcome": outcome,
|
|
"features": features or "{}", "human_override": override,
|
|
"true_tier": true_tier}
|
|
|
|
|
|
# ---------------- derive_true_tier 纯函数 ----------------
|
|
|
|
def test_rule1_success_t1():
|
|
assert derive_true_tier(_obs("r", "T1", "ok")) == "T1"
|
|
assert derive_true_tier(_obs("r", "T1", "verified")) == "T1"
|
|
|
|
|
|
def test_rule2_failure_next_tier_and_t3_stays():
|
|
assert derive_true_tier(_obs("r", "T1", "failed")) == "T2"
|
|
assert derive_true_tier(_obs("r", "T2", "user_retry")) == "T3"
|
|
assert derive_true_tier(_obs("r", "T3", "timeout")) == "T3" # T3 保持 T3
|
|
|
|
|
|
def test_rule3_plan_multi_signals_t3():
|
|
row = _obs("r", "T2", "ok", features=json.dumps({"plan_multi": True}))
|
|
assert derive_true_tier(row) == "T3"
|
|
|
|
|
|
def test_rule4_human_override_wins():
|
|
row = _obs("r", "T1", "failed", override="T3")
|
|
assert derive_true_tier(row) == "T3"
|
|
|
|
|
|
def test_empty_outcome_skipped():
|
|
assert derive_true_tier(_obs("r", "T1", "")) is None
|
|
|
|
|
|
# ---------------- derive_true_tiers 批量 + 清理 ----------------
|
|
|
|
def test_batch_derive_and_purge(tmp_path):
|
|
store = SenseStore.init_db(tmp_path / "s.sqlite3")
|
|
now = time.time()
|
|
for rid, outcome in [("a", "ok"), ("b", "failed"), ("c", "")]:
|
|
store.insert_observation(_obs(rid, "T2", outcome))
|
|
# 一条 200 天前的旧数据（先推导后清理）
|
|
store.insert_observation({**_obs("old", "T1", "ok"), "ts": now - 200 * 86400})
|
|
count = derive_true_tiers(store, now=now, retention_days=180)
|
|
assert count == 3 # a→T2, b→T3, old→T1
|
|
rows = {r["request_id"]: r for r in store.all_observations()}
|
|
assert rows["a"]["true_tier"] == "T2"
|
|
assert rows["b"]["true_tier"] == "T3"
|
|
assert "old" not in rows # 180d 清理生效（推导后删除）
|
|
|
|
|
|
# ---------------- compute_thresholds ----------------
|
|
|
|
def test_conformal_threshold_coverage():
|
|
"""合成分布：T1 高分样本真为 T1 -> τ1 满足精度 >= 1-α；τ3 同理（需真 T3 样本）。"""
|
|
rows = []
|
|
for i in range(40):
|
|
rows.append({"probs": json.dumps({"t1": 0.95 - i * 0.001, "t3": 0.01}),
|
|
"true_tier": "T1"})
|
|
for i in range(10):
|
|
rows.append({"probs": json.dumps({"t1": 0.3, "t3": 0.3}),
|
|
"true_tier": "T2"})
|
|
for i in range(5):
|
|
rows.append({"probs": json.dumps({"t3": 0.9, "t1": 0.02}),
|
|
"true_tier": "T3"})
|
|
th = compute_thresholds(rows, alpha=0.05, min_labels=10)
|
|
assert th["ok"] is True and th["n"] == 55
|
|
kept = [r for r in rows if json.loads(r["probs"])["t1"] >= th["t1"]]
|
|
precision = sum(1 for r in kept if r["true_tier"] == "T1") / len(kept)
|
|
assert precision >= 0.95
|
|
|
|
|
|
def test_min_labels_not_met_returns_not_ok():
|
|
rows = [{"probs": json.dumps({"t1": 0.9, "t3": 0.05}), "true_tier": "T1"}]
|
|
th = compute_thresholds(rows, alpha=0.05, min_labels=500)
|
|
assert th["ok"] is False and th["t1"] is None
|
|
|
|
|
|
def test_threshold_artifact_roundtrip_and_last_good(tmp_path):
|
|
"""工件读写 + active 切换 + last-good 回退链。"""
|
|
store = SenseStore.init_db(tmp_path / "s.sqlite3")
|
|
th = {"t1": 0.8, "t3": 0.85, "coverage": 0.9, "n": 500, "ok": True}
|
|
path = save_thresholds(store, tmp_path, "v1", th, activate=True)
|
|
assert "thresholds.json" in path
|
|
loaded = load_active(store, tmp_path)
|
|
assert loaded["t1"] == 0.8 and loaded["version"] == "v1"
|
|
# 保存 v2 不激活 -> 仍 v1；激活 v2 -> 切换；v2 文件损坏 -> 扫描回退 v1
|
|
save_thresholds(store, tmp_path, "v2", {"t1": 0.7, "t3": 0.7,
|
|
"coverage": 0.9, "n": 500})
|
|
assert load_active(store, tmp_path)["version"] == "v1" # v2 未激活
|
|
store.activate_artifact("v2", "thresholds")
|
|
assert load_active(store, tmp_path)["version"] == "v2"
|
|
(tmp_path / "v2" / "thresholds.json").unlink()
|
|
loaded = load_active(store, tmp_path)
|
|
assert loaded["version"] == "v1" and loaded["t1"] == 0.8 # last-good 扫描回退
|
|
# 全新 store（无任何工件登记）-> 保守值
|
|
fresh_store = SenseStore.init_db(tmp_path / "fresh.sqlite3")
|
|
fresh = load_active(fresh_store, tmp_path / "fresh")
|
|
assert fresh["version"] == "conservative"
|