81 lines
2.9 KiB
Python
81 lines
2.9 KiB
Python
"""迷你评估：对带标注的样例查询评估分类准确率、升级率、成本。
|
||
|
||
用法：
|
||
python scripts/eval.py [--repeat 2] [--config path]
|
||
--repeat 用于把样例跑 N 遍，验证语义缓存命中与降本效果。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import asyncio
|
||
import sys
|
||
from collections import Counter
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||
|
||
from router_system.router import build_router
|
||
|
||
# (query, 期望领域)
|
||
BENCH = [
|
||
("用 Python 实现二分查找", "code"),
|
||
("这段 JavaScript 为什么报错：undefined is not a function", "code"),
|
||
("帮我优化这个 SQL 查询的索引", "code"),
|
||
("求解一元二次方程 ax^2+bx+c=0 的求根公式", "math"),
|
||
("证明勾股定理", "math"),
|
||
("计算 3x + 5 = 20，x 等于多少", "math"),
|
||
("劳动合同到期不续签，公司需要支付经济补偿吗", "legal"),
|
||
("在合同中约定违约金上限 30%，是否合规", "legal"),
|
||
("专利申请的流程和费用大概是多少", "legal"),
|
||
("高血压患者可以吃哪些降压药，副作用是什么", "medical"),
|
||
("感冒发烧 38.5 度，需要吃退烧药吗", "medical"),
|
||
("糖尿病患者的日常饮食建议", "medical"),
|
||
("介绍一下 Transformer 架构", "general"),
|
||
("写一封请假邮件", "general"),
|
||
("为什么天空是蓝色的", "general"),
|
||
]
|
||
|
||
|
||
async def main():
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument("--repeat", type=int, default=2, help="重复轮数（验证缓存）")
|
||
parser.add_argument("--config", type=str, default=None)
|
||
args = parser.parse_args()
|
||
|
||
router = build_router(args.config)
|
||
correct = Counter()
|
||
total = 0
|
||
upgraded = 0
|
||
cache_hits = 0
|
||
|
||
for round_i in range(args.repeat):
|
||
for q, expected in BENCH:
|
||
r = await router.route(q)
|
||
total += 1
|
||
if r.domain == expected:
|
||
correct["total"] += 1
|
||
else:
|
||
correct[f"misclass->{r.domain}"] += 1
|
||
if r.upgraded:
|
||
upgraded += 1
|
||
if r.cache_hit:
|
||
cache_hits += 1
|
||
|
||
acc = correct["total"] / total
|
||
print(f"样例数: {len(BENCH)} x {args.repeat} 轮 = {total} 次请求")
|
||
print(f"分类准确率: {acc:.1%} ({correct['total']}/{total})")
|
||
print(f"升级率: {upgraded/total:.1%} ({upgraded}/{total})")
|
||
print(f"缓存命中率: {cache_hits/total:.1%} ({cache_hits}/{total})")
|
||
print()
|
||
print("运行指标:", router.stats.summary())
|
||
print("缓存统计:", router.cache.stats())
|
||
print()
|
||
if acc < 0.8:
|
||
print("⚠️ 准确率低于 80%，请检查分类规则。")
|
||
else:
|
||
print("✅ 分类准确率达标（≥80%）。")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
asyncio.run(main())
|
||
|