chore: T-P-1 工作区收敛——并行会话成果与历史未入库文件整理入库
- 入库历史遗漏源码/测试：router_system 9 模块（agent/executors/inference/knowledge/ memory/planner/skills/trace）、tests 11 个测试文件、config/knowledge 领域知识 - 入库根目录方案文档（v2/v3/可行性×2）、references 文献（arxiv 14-18/cnki_open/ 参考文献清单）、research 论文素材（routerarena/paper/中文文献 PDF） - 前端构建产物刷新（新 hash）；webapp 误写文档删除 - gitignore 增补：deepseek-harness、research/_refs、.mimosa/.zcode、网关日志/pid、 临时调试脚本、tests/e2e/node_modules、AI代理功能开发/prefix - 基线确认：318 passed
This commit is contained in:
tzt
@@ -0,0 +1,410 @@
|
||||
"""本地 Runner：不依赖 RouterArena 完整仓库，验证接入方法学。
|
||||
|
||||
职责：
|
||||
1. 加载数据集（mock 或真实 sub_10，真实数据需从 HF 拉取）
|
||||
2. 跑 adapter.get_prediction 拿到 (global_index, prompt, prediction)
|
||||
3. 写入 RouterArena 协议预测文件
|
||||
4. 提供 mock 推理 + Arena Score 计算，验证方法学
|
||||
5. 输出路由分布报告与排行榜基线对比占位
|
||||
|
||||
不依赖：API key、RouterArena 仓库
|
||||
仅依赖：Python 标准库 + 本项目 router_system
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
# 项目根加入 path
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
|
||||
|
||||
# --- 模型价格（USD / 1M tokens）---
|
||||
# 依据 RouterArena model_cost/model_cost.json 公开快照（2026-07）
|
||||
# 这里给的是 input+output 平均近似；精确值在官方文件
|
||||
MODEL_PRICING: Dict[str, Dict[str, float]] = {
|
||||
"gpt-4o-mini": {"input": 0.15, "output": 0.60},
|
||||
"claude-3-haiku-20240307": {"input": 0.25, "output": 1.25},
|
||||
"gemini-2.0-flash-001": {"input": 0.075, "output": 0.30},
|
||||
"deepseek-chat": {"input": 0.14, "output": 0.28},
|
||||
"mistral-medium": {"input": 2.7, "output": 8.1},
|
||||
}
|
||||
|
||||
# 排行榜公开基线（2026-07 快照，来源 RouterArena README L40-66）
|
||||
LEADERBOARD_BASELINES: Dict[str, Dict[str, float]] = {
|
||||
"Cross-Router": {"arena_score": 75.75, "accuracy": 78.14, "cost_per_1k": 0.40},
|
||||
"Hybrid Router": {"arena_score": 72.08, "accuracy": 71.38, "cost_per_1k": 0.04},
|
||||
"R2-Router": {"arena_score": 71.60, "accuracy": 71.23, "cost_per_1k": 0.06},
|
||||
"GPT-5": {"arena_score": 64.32, "accuracy": 73.96, "cost_per_1k": 10.02},
|
||||
"MIRT-BERT": {"arena_score": 66.89, "accuracy": 66.88, "cost_per_1k": 0.15},
|
||||
"NotDiamond": {"arena_score": 57.29, "accuracy": 60.83, "cost_per_1k": 4.10},
|
||||
"RouteLLM": {"arena_score": 48.07, "accuracy": 47.04, "cost_per_1k": 0.27},
|
||||
"RouterDC": {"arena_score": 33.75, "accuracy": 32.01, "cost_per_1k": 0.07},
|
||||
}
|
||||
|
||||
|
||||
# --- Arena Score 公式（依据 RouterArena llm_evaluation/run.py L65-87）---
|
||||
def compute_arena_score(
|
||||
cost_per_1k: float,
|
||||
accuracy: float,
|
||||
beta: float = 0.1,
|
||||
c_max: float = 200.0,
|
||||
c_min: float = 0.0044,
|
||||
) -> float:
|
||||
if cost_per_1k is None or cost_per_1k <= 0:
|
||||
raise ValueError("cost_per_1k must be positive")
|
||||
if accuracy is None or not (0 <= accuracy <= 1):
|
||||
raise ValueError("accuracy must be in [0, 1]")
|
||||
cost_clamped = max(c_min, min(cost_per_1k, c_max))
|
||||
C = (math.log2(c_max) - math.log2(cost_clamped)) / (math.log2(c_max) - math.log2(c_min))
|
||||
return ((1 + beta) * accuracy * C) / (beta * accuracy + C)
|
||||
|
||||
|
||||
# --- Mock 数据集生成 ---
|
||||
# 模仿 RouterArena sub_10 的 9 领域结构：9 domains × ~90 queries = 810 ≈ 809
|
||||
MOCK_DOMAIN_QUERIES: Dict[str, List[str]] = {
|
||||
"code": [
|
||||
"Implement quicksort in Python",
|
||||
"Write a function to reverse a linked list",
|
||||
"Debug this TypeError: undefined is not a function",
|
||||
"Optimize SQL query with index hints",
|
||||
"Implement binary search in Java",
|
||||
"用 Python 写一个快速排序函数",
|
||||
"解释这段 JavaScript 代码的 TypeError",
|
||||
"帮我优化 SQL 索引",
|
||||
"Implement merge sort",
|
||||
"Convert JSON to CSV in Python",
|
||||
],
|
||||
"math": [
|
||||
"Solve x^2 - 5x + 6 = 0",
|
||||
"Prove the Pythagorean theorem",
|
||||
"Calculate the integral of x^2 from 0 to 1",
|
||||
"求方程 x^2+3x+2=0 的根",
|
||||
"证明勾股定理",
|
||||
"计算 3x+5=20 中 x 的值",
|
||||
"Find eigenvalues of a 2x2 matrix",
|
||||
"Differentiate sin(x) * cos(x)",
|
||||
"求 ∫ x^2 dx",
|
||||
"Compute dot product of two vectors",
|
||||
],
|
||||
"legal": [
|
||||
"Is a non-compete clause for 2 years enforceable?",
|
||||
"How to calculate severance pay",
|
||||
"劳动合同到期不续签是否要给补偿金",
|
||||
"加班费怎么计算",
|
||||
"违约金上限 30% 合法吗",
|
||||
"What counts as wrongful termination",
|
||||
"Can I sue my employer for unpaid wages",
|
||||
"劳动合同里约定竞业限制是否有效",
|
||||
"Statute of limitations for breach of contract",
|
||||
"How does arbitration work in employment disputes",
|
||||
],
|
||||
"medical": [
|
||||
"What foods should hypertensive patients avoid",
|
||||
"First aid for burns",
|
||||
"高血压患者日常饮食",
|
||||
"感冒发烧 38.5 度需要吃退烧药吗",
|
||||
"Side effects of common blood pressure medications",
|
||||
"When to go to ER for chest pain",
|
||||
"烫伤后怎么处理",
|
||||
"感冒初期如何缓解症状",
|
||||
"How to treat a sprained ankle",
|
||||
"What are warning signs of diabetes",
|
||||
],
|
||||
"finance": [
|
||||
"How to calculate ROI on a fund",
|
||||
"What to do when credit card is overdue",
|
||||
"基金定投收益率怎么计算",
|
||||
"信用卡逾期怎么办",
|
||||
"房贷利率是 LPR 加多少",
|
||||
"Should I refinance my mortgage",
|
||||
"Best way to save for retirement",
|
||||
"理财产品和基金的区别",
|
||||
"How to read a stock balance sheet",
|
||||
"What is dollar-cost averaging",
|
||||
],
|
||||
"life": [
|
||||
"Travel itinerary for Japan in 7 days",
|
||||
"Beginner muscle building plan",
|
||||
"日本旅行攻略",
|
||||
"健身增肌计划",
|
||||
"家常菜推荐",
|
||||
"How to meal prep for a week",
|
||||
"Best hiking trails near San Francisco",
|
||||
"减脂餐怎么搭配",
|
||||
"How to start running for beginners",
|
||||
"室内绿植推荐",
|
||||
],
|
||||
"education": [
|
||||
"How to prepare for graduate English exam",
|
||||
"Effective study techniques",
|
||||
"考研英语怎么备考",
|
||||
"高效学习方法",
|
||||
"面试技巧有哪些",
|
||||
"How to write a research paper",
|
||||
"GRE quantitative prep strategy",
|
||||
"如何准备技术面试",
|
||||
"Best resources for learning Python",
|
||||
"时间管理方法",
|
||||
],
|
||||
"general": [
|
||||
"Why is the sky blue",
|
||||
"Explain the Transformer architecture",
|
||||
"为什么天空是蓝色的",
|
||||
"介绍 Transformer 架构",
|
||||
"Write a vacation request email",
|
||||
"What is quantum entanglement",
|
||||
"请写一封请假邮件",
|
||||
"Explain CRISPR in simple terms",
|
||||
"What is blockchain",
|
||||
"简单介绍下黑洞",
|
||||
],
|
||||
"creative": [
|
||||
"Write a haiku about autumn",
|
||||
"Suggest a name for a coffee shop",
|
||||
"Plot twist ideas for a mystery novel",
|
||||
"Write a short poem about the ocean",
|
||||
"Ideas for a 5-year-old's birthday party",
|
||||
"Story opening for a sci-fi short",
|
||||
"Suggest tagline for eco-friendly brand",
|
||||
"Lyrics for an upbeat summer song",
|
||||
"Title ideas for a romance novel",
|
||||
"Concept art description for a fantasy creature",
|
||||
],
|
||||
# 注：RouterArena 有 9 domains，creative 是第 9 类的代表（写作/创意）
|
||||
}
|
||||
|
||||
|
||||
def build_mock_dataset() -> List[Dict[str, Any]]:
|
||||
"""生成与 RouterArena sub_10 协议对齐的 mock 数据集。"""
|
||||
data: List[Dict[str, Any]] = []
|
||||
idx = 0
|
||||
for domain, queries in MOCK_DOMAIN_QUERIES.items():
|
||||
for i, q in enumerate(queries):
|
||||
data.append({
|
||||
"global index": f"mock_{idx:04d}",
|
||||
"prompt": q,
|
||||
"prompt_formatted": q,
|
||||
"domain": domain, # 仅用于本地诊断，不暴露给路由决策
|
||||
"difficulty": ["easy", "medium", "hard"][i % 3],
|
||||
})
|
||||
idx += 1
|
||||
return data
|
||||
|
||||
|
||||
# --- 加载器（支持 mock + 真实 sub_10 JSON）---
|
||||
def load_dataset(source: str, path: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""source ∈ {'mock', 'sub_10_file'}；path 为 None 时按 source 推断。"""
|
||||
if source == "mock":
|
||||
return build_mock_dataset()
|
||||
if source == "sub_10_file":
|
||||
if not path or not os.path.exists(path):
|
||||
raise FileNotFoundError(f"sub_10 dataset not found at {path}")
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
# RouterArena 协议字段：global index, prompt_formatted
|
||||
normalized = []
|
||||
for entry in data:
|
||||
normalized.append({
|
||||
"global index": entry.get("global index") or entry.get("global_index"),
|
||||
"prompt": entry.get("prompt_formatted") or entry.get("prompt"),
|
||||
"domain": None, # 真实数据无 ground truth
|
||||
"difficulty": None,
|
||||
})
|
||||
return normalized
|
||||
raise ValueError(f"Unknown source: {source}")
|
||||
|
||||
|
||||
# --- 模拟推理（不调 API）：用 L0 router 自身的 response 当 generated_answer ---
|
||||
def mock_inference(router, query: str) -> Dict[str, Any]:
|
||||
"""用本系统 L0 router 自身的 response 模拟目标 LLM 的输出。
|
||||
|
||||
注意：这只是验证"路由层 + 协议格式"正确，不替代真实 LLM 推理。
|
||||
真实评测需要 RouterArena 的 llm_inference/run.py + 目标模型 API key。
|
||||
"""
|
||||
import asyncio
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
if loop.is_running():
|
||||
# 极少见兜底
|
||||
return {"generated_answer": "[sync-fallback]", "success": True,
|
||||
"token_usage": {"output_tokens": 50}}
|
||||
result = loop.run_until_complete(router._router.route(query))
|
||||
except RuntimeError:
|
||||
result = asyncio.run(router._router.route(query))
|
||||
|
||||
answer = result.response or ""
|
||||
# 估算 token 数（中英文 1 token ≈ 1.5 字符）
|
||||
output_tokens = max(1, int(len(answer) / 1.5))
|
||||
return {
|
||||
"generated_answer": answer,
|
||||
"success": True,
|
||||
"model_used": result.model_used,
|
||||
"token_usage": {"output_tokens": output_tokens, "input_tokens": int(len(query) / 1.5)},
|
||||
}
|
||||
|
||||
|
||||
# --- 主流程 ---
|
||||
def estimate_cost(model_name: str, token_usage: Dict[str, int]) -> float:
|
||||
"""按 MODEL_PRICING 估算单条推理成本（USD）。"""
|
||||
p = MODEL_PRICING.get(model_name)
|
||||
if not p:
|
||||
return 0.0
|
||||
in_tok = token_usage.get("input_tokens", 0)
|
||||
out_tok = token_usage.get("output_tokens", 0)
|
||||
return (in_tok * p["input"] + out_tok * p["output"]) / 1_000_000
|
||||
|
||||
|
||||
def run_local(
|
||||
source: str = "mock",
|
||||
dataset_path: Optional[str] = None,
|
||||
router_name: str = "es-expert",
|
||||
config_path: Optional[str] = None,
|
||||
do_mock_inference: bool = True,
|
||||
output_dir: str = "research/routerarena/output",
|
||||
) -> Dict[str, Any]:
|
||||
"""跑本地端到端流程，输出预测文件 + 诊断报告。"""
|
||||
from .adapter import ESExpertRouter
|
||||
|
||||
if config_path is None:
|
||||
here = Path(__file__).resolve().parent
|
||||
config_path = str(here / "config" / f"{router_name}.json")
|
||||
|
||||
router = ESExpertRouter(router_name=router_name, config_path=config_path)
|
||||
dataset = load_dataset(source, dataset_path)
|
||||
print(f"[local_runner] router models = {router.models}")
|
||||
print(f"[local_runner] dataset size = {len(dataset)}")
|
||||
|
||||
# 1) 路由决策（按 RouterArena 协议）
|
||||
predictions: List[Dict[str, Any]] = []
|
||||
diagnostics: List[Dict[str, Any]] = []
|
||||
t0 = time.perf_counter()
|
||||
for entry in dataset:
|
||||
gi = entry["global index"]
|
||||
prompt = entry["prompt"]
|
||||
selected = router.get_prediction(prompt)
|
||||
# 同时记一份诊断（科研用，不影响协议）
|
||||
diag = router.diagnostics(prompt)
|
||||
diag["global_index"] = gi
|
||||
diag["ground_truth_domain"] = entry.get("domain") # 仅 mock 数据有
|
||||
diagnostics.append(diag)
|
||||
predictions.append({
|
||||
"global index": gi,
|
||||
"prompt": prompt,
|
||||
"prediction": selected,
|
||||
"generated_result": None,
|
||||
"cost": None,
|
||||
"accuracy": None,
|
||||
"for_optimality": False,
|
||||
})
|
||||
routing_latency_ms = (time.perf_counter() - t0) * 1000 / len(dataset)
|
||||
|
||||
# 2) 模拟推理（mock generated_result）
|
||||
if do_mock_inference:
|
||||
for pred, diag in zip(predictions, diagnostics):
|
||||
gen = mock_inference(router, pred["prompt"])
|
||||
pred["generated_result"] = gen
|
||||
cost = estimate_cost(pred["prediction"], gen["token_usage"])
|
||||
pred["cost"] = cost
|
||||
# mock 准确率：仅用于方法学验证（真实评测 RouterArena 用 ground truth）
|
||||
# 这里如果 router 选对了 ground truth domain → 1.0，否则按简单启发式
|
||||
if diag.get("ground_truth_domain") and diag.get("domain"):
|
||||
# 把 L0 的 8 域映射到 RouterArena 的 9 域（creative 算 general）
|
||||
gt = diag["ground_truth_domain"]
|
||||
pred_domain = diag["domain"]
|
||||
# RouterArena creative → 本系统 general 域
|
||||
gt_mapped = "general" if gt == "creative" else gt
|
||||
pred["accuracy"] = 1.0 if pred_domain == gt_mapped else 0.0
|
||||
else:
|
||||
pred["accuracy"] = None # 真实数据无 domain 标签，跳过
|
||||
|
||||
# 3) 写预测文件（RouterArena 协议）
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
pred_path = os.path.join(output_dir, f"{router_name}.json")
|
||||
with open(pred_path, "w", encoding="utf-8") as f:
|
||||
json.dump(predictions, f, ensure_ascii=False, indent=2)
|
||||
diag_path = os.path.join(output_dir, f"{router_name}_diagnostics.json")
|
||||
with open(diag_path, "w", encoding="utf-8") as f:
|
||||
json.dump(diagnostics, f, ensure_ascii=False, indent=2)
|
||||
|
||||
# 4) 算指标
|
||||
n = len(predictions)
|
||||
acc_vals = [p["accuracy"] for p in predictions if p["accuracy"] is not None]
|
||||
cost_vals = [p["cost"] for p in predictions if p["cost"] is not None and p["cost"] > 0]
|
||||
avg_acc = sum(acc_vals) / len(acc_vals) if acc_vals else 0.0
|
||||
total_cost = sum(cost_vals) if cost_vals else 0.0
|
||||
cost_per_1k = (total_cost / n * 1000) if n > 0 else 0.0
|
||||
try:
|
||||
arena_score = compute_arena_score(cost_per_1k, avg_acc) if cost_per_1k > 0 else None
|
||||
except ValueError:
|
||||
arena_score = None
|
||||
|
||||
# 路由分布
|
||||
from collections import Counter
|
||||
routing_dist = Counter(p["prediction"] for p in predictions)
|
||||
domain_dist = Counter(d["domain"] for d in diagnostics)
|
||||
confidence_dist = {
|
||||
"min": min(d["confidence"] for d in diagnostics),
|
||||
"max": max(d["confidence"] for d in diagnostics),
|
||||
"mean": sum(d["confidence"] for d in diagnostics) / len(diagnostics),
|
||||
}
|
||||
|
||||
summary = {
|
||||
"router_name": router_name,
|
||||
"n_queries": n,
|
||||
"routing_latency_ms_per_query": routing_latency_ms,
|
||||
"domain_distribution": dict(domain_dist),
|
||||
"routing_distribution": dict(routing_dist),
|
||||
"confidence": confidence_dist,
|
||||
"mock_accuracy": avg_acc,
|
||||
"total_cost_usd": total_cost,
|
||||
"cost_per_1k_usd": cost_per_1k,
|
||||
"arena_score_mock": arena_score,
|
||||
"prediction_file": pred_path,
|
||||
"diagnostics_file": diag_path,
|
||||
}
|
||||
summary_path = os.path.join(output_dir, f"{router_name}_summary.json")
|
||||
with open(summary_path, "w", encoding="utf-8") as f:
|
||||
json.dump(summary, f, ensure_ascii=False, indent=2)
|
||||
return summary
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="RouterArena 本地接入烟测（无需 API）")
|
||||
parser.add_argument("--source", choices=["mock", "sub_10_file"], default="mock")
|
||||
parser.add_argument("--dataset-path", default=None)
|
||||
parser.add_argument("--router", default="es-expert")
|
||||
parser.add_argument("--config", default=None)
|
||||
parser.add_argument("--no-mock-inference", action="store_true",
|
||||
help="只跑路由不模拟推理（用于纯路由层验证）")
|
||||
parser.add_argument("--output-dir", default="research/routerarena/output")
|
||||
args = parser.parse_args()
|
||||
|
||||
summary = run_local(
|
||||
source=args.source,
|
||||
dataset_path=args.dataset_path,
|
||||
router_name=args.router,
|
||||
config_path=args.config,
|
||||
do_mock_inference=not args.no_mock_inference,
|
||||
output_dir=args.output_dir,
|
||||
)
|
||||
# 打印关键指标（确保终端 GBK 安全：写到文件再读）
|
||||
out_path = os.path.join(args.output_dir, f"{args.router}_summary.json")
|
||||
with open(out_path, "r", encoding="utf-8") as f:
|
||||
s = json.load(f)
|
||||
print("\n========== RouterArena Local Run Summary ==========")
|
||||
for k, v in s.items():
|
||||
print(f" {k}: {v}")
|
||||
print("====================================================")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user