"""标签+校准测试(T-G3):四条推导规则 / T3 保持 / 180d 清理 / conformal 覆盖 / last-good。""" import json import time from gateway.sense.calibrate import ( compute_thresholds, load_active, save_thresholds, ) from gateway.sense.labeler import derive_true_tier, derive_true_tiers from gateway.sense.store import SenseStore def _obs(rid, executed, outcome, probs=None, features=None, override="", true_tier=""): return {"request_id": rid, "consumer": "pipeline", "decided_tier": executed, "executed_tier": executed, "probs": probs or "{}", "outcome": outcome, "features": features or "{}", "human_override": override, "true_tier": true_tier} # ---------------- derive_true_tier 纯函数 ---------------- def test_rule1_success_t1(): assert derive_true_tier(_obs("r", "T1", "ok")) == "T1" assert derive_true_tier(_obs("r", "T1", "verified")) == "T1" def test_rule2_failure_next_tier_and_t3_stays(): assert derive_true_tier(_obs("r", "T1", "failed")) == "T2" assert derive_true_tier(_obs("r", "T2", "user_retry")) == "T3" assert derive_true_tier(_obs("r", "T3", "timeout")) == "T3" # T3 保持 T3 def test_rule3_plan_multi_signals_t3(): row = _obs("r", "T2", "ok", features=json.dumps({"plan_multi": True})) assert derive_true_tier(row) == "T3" def test_rule4_human_override_wins(): row = _obs("r", "T1", "failed", override="T3") assert derive_true_tier(row) == "T3" def test_empty_outcome_skipped(): assert derive_true_tier(_obs("r", "T1", "")) is None # ---------------- derive_true_tiers 批量 + 清理 ---------------- def test_batch_derive_and_purge(tmp_path): store = SenseStore.init_db(tmp_path / "s.sqlite3") now = time.time() for rid, outcome in [("a", "ok"), ("b", "failed"), ("c", "")]: store.insert_observation(_obs(rid, "T2", outcome)) # 一条 200 天前的旧数据(先推导后清理) store.insert_observation({**_obs("old", "T1", "ok"), "ts": now - 200 * 86400}) count = derive_true_tiers(store, now=now, retention_days=180) assert count == 3 # a→T2, b→T3, old→T1 rows = {r["request_id"]: r for r in store.all_observations()} assert rows["a"]["true_tier"] == "T2" assert rows["b"]["true_tier"] == "T3" assert "old" not in rows # 180d 清理生效(推导后删除) # ---------------- compute_thresholds ---------------- def test_conformal_threshold_coverage(): """合成分布:T1 高分样本真为 T1 -> τ1 满足精度 >= 1-α;τ3 同理(需真 T3 样本)。""" rows = [] for i in range(40): rows.append({"probs": json.dumps({"t1": 0.95 - i * 0.001, "t3": 0.01}), "true_tier": "T1"}) for i in range(10): rows.append({"probs": json.dumps({"t1": 0.3, "t3": 0.3}), "true_tier": "T2"}) for i in range(5): rows.append({"probs": json.dumps({"t3": 0.9, "t1": 0.02}), "true_tier": "T3"}) th = compute_thresholds(rows, alpha=0.05, min_labels=10) assert th["ok"] is True and th["n"] == 55 kept = [r for r in rows if json.loads(r["probs"])["t1"] >= th["t1"]] precision = sum(1 for r in kept if r["true_tier"] == "T1") / len(kept) assert precision >= 0.95 def test_min_labels_not_met_returns_not_ok(): rows = [{"probs": json.dumps({"t1": 0.9, "t3": 0.05}), "true_tier": "T1"}] th = compute_thresholds(rows, alpha=0.05, min_labels=500) assert th["ok"] is False and th["t1"] is None def test_threshold_artifact_roundtrip_and_last_good(tmp_path): """工件读写 + active 切换 + last-good 回退链。""" store = SenseStore.init_db(tmp_path / "s.sqlite3") th = {"t1": 0.8, "t3": 0.85, "coverage": 0.9, "n": 500, "ok": True} path = save_thresholds(store, tmp_path, "v1", th, activate=True) assert "thresholds.json" in path loaded = load_active(store, tmp_path) assert loaded["t1"] == 0.8 and loaded["version"] == "v1" # 保存 v2 不激活 -> 仍 v1;激活 v2 -> 切换;v2 文件损坏 -> 扫描回退 v1 save_thresholds(store, tmp_path, "v2", {"t1": 0.7, "t3": 0.7, "coverage": 0.9, "n": 500}) assert load_active(store, tmp_path)["version"] == "v1" # v2 未激活 store.activate_artifact("v2", "thresholds") assert load_active(store, tmp_path)["version"] == "v2" (tmp_path / "v2" / "thresholds.json").unlink() loaded = load_active(store, tmp_path) assert loaded["version"] == "v1" and loaded["t1"] == 0.8 # last-good 扫描回退 # 全新 store(无任何工件登记)-> 保守值 fresh_store = SenseStore.init_db(tmp_path / "fresh.sqlite3") fresh = load_active(fresh_store, tmp_path / "fresh") assert fresh["version"] == "conservative"