"""离线训练三分类线性头（T-G4）：softmax 回归（numpy），产出 JSON 工件。 用法： python scripts/train_tier_head.py --db data/sense.sqlite3 --version v1 python scripts/train_tier_head.py --synthetic # 合成数据自测（无需真实标签） 输入：sense.sqlite3 tier_observations 中 true_tier 非空且 embedding（int8 BLOB） 可解析的行；按时间 70/15/15 切分 train/val/calib。 输出：data/sense_models/{version}/head.json + metrics.json + 工件登记（active=0， 人工 /sense/admin/promote 切换）。 依赖：numpy（可选，requirements-ml.txt）；serving 推理不依赖 numpy（D-G5）。 """ from __future__ import annotations import argparse import json import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8") sys.stderr.reconfigure(encoding="utf-8") try: import numpy as np except ImportError: print("缺少 numpy：pip install numpy（或 requirements-ml.txt）；训练必需，serving 不需要。") sys.exit(1) from gateway.sense.store import SenseStore # noqa: E402 TIERS = ["t1", "t2", "t3"] def _deblob(blob) -> list[float]: return [b - 256 if b > 127 else b for b in blob] def load_rows(db_path: str): """true_tier 非空且 embedding 可解析的行 -> (X, y, versions)。""" store = SenseStore.init_db(db_path) X, y, vers = [], [], [] for row in store.labeled_rows(limit=200000): blob = row["embedding"] if not blob: continue X.append(_deblob(blob)) y.append(TIERS.index(row["true_tier"].lower())) vers.append(row.get("policy_version") or "") return np.array(X, dtype=float), np.array(y), vers def split(n: int, seed: int = 42): idx = np.random.default_rng(seed).permutation(n) t = int(n * 0.7) v = int(n * 0.85) return idx[:t], idx[t:v], idx[v:] def softmax(z): z = z - z.max(axis=1, keepdims=True) e = np.exp(z) return e / e.sum(axis=1, keepdims=True) def train(X, y, epochs=300, lr=0.5, seed: int = 42): """L2 正则 softmax 回归（全量梯度下降，类别权重均衡）。""" n, d = X.shape k = 3 Y = np.zeros((n, k)) for i, c in enumerate(y): Y[i, c] = 1.0 cls_weight = n / (k * np.array([(y == c).sum() or 1 for c in range(k)])) W = np.zeros((d, k)) b = np.zeros(k) sw = np.array([cls_weight[c] for c in y]) sw = sw / sw.sum() * n rng = np.random.default_rng(seed) for _ in range(epochs): probs = softmax(X @ W + b) grad = (probs - Y) * sw[:, None] W -= lr * (X.T @ grad / n + 0.01 * W) b -= lr * (grad.sum(axis=0) / n) return W, b def metrics(X, y, W, b) -> dict: probs = softmax(X @ W + b) pred = probs.argmax(axis=1) macro_f1s = [] for c in range(3): tp = float(((pred == c) & (y == c)).sum()) fp = float(((pred == c) & (y != c)).sum()) fn = float(((pred != c) & (y == c)).sum()) precision = tp / (tp + fp) if tp + fp else 0.0 recall = tp / (tp + fn) if tp + fn else 0.0 f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 macro_f1s.append(f1) acc = float((pred == y).mean()) # ordinal AUC 简化：P(判>=c 与 真>=c 一致) 均值 return {"accuracy": round(acc, 4), "macro_f1": round(sum(macro_f1s) / 3, 4), "per_tier_f1": [round(f, 4) for f in macro_f1s]} def run(db_path: str, version: str, models_dir: str, epochs: int) -> int: store = SenseStore.init_db(db_path) X, y, _ = load_rows(db_path) if len(X) < 30: print(f"[train] 标签样本不足（{len(X)} < 30），跳过训练。" "先以 collect 模式积累标签（夜间推导）。") return 1 tr, va, ca = split(len(X)) W, b = train(X[tr], y[tr], epochs=epochs) m_val = metrics(X[va], y[va], W, b) probs_cal = (softmax(X[ca] @ W + b) * 1000).round().astype(int) / 1000 calib_rows = [{"probs": json.dumps({TIERS[i]: float(probs_cal[j, i]) for i in range(3)}), "true_tier": TIERS[y[ca][j]]} for j in range(len(y[ca]))] out = Path(models_dir) / version out.mkdir(parents=True, exist_ok=True) (out / "head.json").write_text(json.dumps({ "version": version, "dim": int(X.shape[1]), "labels": TIERS, "weights": W.T.tolist(), "bias": b.tolist()}, ensure_ascii=False), encoding="utf-8") metrics_data = {"version": version, "n": int(len(X)), "val": m_val, "created": time.strftime("%Y-%m-%d %H:%M:%S")} (out / "metrics.json").write_text(json.dumps(metrics_data, ensure_ascii=False, indent=2), encoding="utf-8") # calib 阈值（用 T-G3 校准逻辑） from gateway.sense.calibrate import compute_thresholds, save_thresholds th = compute_thresholds(calib_rows, alpha=0.05, min_labels=5) save_thresholds(store, models_dir, version, th, activate=False) store.register_artifact(version, "head", str(out / "head.json"), metrics_data, active=False) print(f"[train] version={version} n={len(X)} val={m_val}") print(f"[train] 阈值={th}") print(f"[train] 工件已登记（active=0），/sense/admin/promote 切换") return 0 def run_synthetic(models_dir: str) -> int: """合成数据自测：三类可分高斯簇，验证训练-工件-LinearHead 推理链路。""" rng = np.random.default_rng(7) n, d = 300, 64 centers = [rng.normal(0, 1, d), rng.normal(4, 1, d), rng.normal(-4, 1, d)] X = np.vstack([c + rng.normal(0, 0.8, (n, d)) for c in centers]) y = np.repeat([0, 1, 2], n) W, b = train(X, y, epochs=200) m = metrics(X, y, W, b) assert m["macro_f1"] > 0.9, f"合成集 macro_F1 过低: {m}" version = "synthetic" out = Path(models_dir) / version out.mkdir(parents=True, exist_ok=True) (out / "head.json").write_text(json.dumps({ "version": version, "dim": d, "labels": TIERS, "weights": W.T.tolist(), "bias": b.tolist()}), encoding="utf-8") print(f"[synthetic] macro_f1={m['macro_f1']} accuracy={m['accuracy']}" f" -> {out/'head.json'}") return 0 def main() -> int: ap = argparse.ArgumentParser(description="训练三分类线性头（T-G4）") ap.add_argument("--db", default="data/sense.sqlite3") ap.add_argument("--version", default=time.strftime("v%Y%m%d")) ap.add_argument("--models-dir", default="data/sense_models") ap.add_argument("--epochs", type=int, default=300) ap.add_argument("--synthetic", action="store_true", help="合成数据自测训练链路（不需要真实标签）") args = ap.parse_args() if args.synthetic: return run_synthetic(args.models_dir) return run(args.db, args.version, args.models_dir, args.epochs) if __name__ == "__main__": sys.exit(main())