feat: 多专业小模型+路由模型系统 MVP（mock 全链路 + FastAPI 网关 + 论文调研）

2026-08-12 10:40:04 +08:00
commit 1e51167ea5
50 changed files with 49382 additions and 0 deletions
@@ -0,0 +1,80 @@
+"""迷你评估：对带标注的样例查询评估分类准确率、升级率、成本。
+
+用法：
+    python scripts/eval.py [--repeat 2] [--config path]
+--repeat 用于把样例跑 N 遍，验证语义缓存命中与降本效果。
+"""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+from collections import Counter
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from router_system.router import build_router
+
+# (query, 期望领域)
+BENCH = [
+    ("用 Python 实现二分查找", "code"),
+    ("这段 JavaScript 为什么报错：undefined is not a function", "code"),
+    ("帮我优化这个 SQL 查询的索引", "code"),
+    ("求解一元二次方程 ax^2+bx+c=0 的求根公式", "math"),
+    ("证明勾股定理", "math"),
+    ("计算 3x + 5 = 20，x 等于多少", "math"),
+    ("劳动合同到期不续签，公司需要支付经济补偿吗", "legal"),
+    ("在合同中约定违约金上限 30%，是否合规", "legal"),
+    ("专利申请的流程和费用大概是多少", "legal"),
+    ("高血压患者可以吃哪些降压药，副作用是什么", "medical"),
+    ("感冒发烧 38.5 度，需要吃退烧药吗", "medical"),
+    ("糖尿病患者的日常饮食建议", "medical"),
+    ("介绍一下 Transformer 架构", "general"),
+    ("写一封请假邮件", "general"),
+    ("为什么天空是蓝色的", "general"),
+]
+
+
+async def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--repeat", type=int, default=2, help="重复轮数（验证缓存）")
+    parser.add_argument("--config", type=str, default=None)
+    args = parser.parse_args()
+
+    router = build_router(args.config)
+    correct = Counter()
+    total = 0
+    upgraded = 0
+    cache_hits = 0
+
+    for round_i in range(args.repeat):
+        for q, expected in BENCH:
+            r = await router.route(q)
+            total += 1
+            if r.domain == expected:
+                correct["total"] += 1
+            else:
+                correct[f"misclass->{r.domain}"] += 1
+            if r.upgraded:
+                upgraded += 1
+            if r.cache_hit:
+                cache_hits += 1
+
+    acc = correct["total"] / total
+    print(f"样例数: {len(BENCH)} x {args.repeat} 轮 = {total} 次请求")
+    print(f"分类准确率: {acc:.1%}  ({correct['total']}/{total})")
+    print(f"升级率: {upgraded/total:.1%} ({upgraded}/{total})")
+    print(f"缓存命中率: {cache_hits/total:.1%} ({cache_hits}/{total})")
+    print()
+    print("运行指标:", router.stats.summary())
+    print("缓存统计:", router.cache.stats())
+    print()
+    if acc < 0.8:
+        print("⚠️  准确率低于 80%，请检查分类规则。")
+    else:
+        print("✅ 分类准确率达标（≥80%）。")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())␍