"""E-P1/E-P2 压测与预算管道（T-P8，M3 验收数据源）。 - 合成 200 条校园模拟请求（同主题重复 + 同义变体 >=50%）打 /proxy/v1； - mock 模式（默认）：TestClient 内存压测（假上游由测试环境注入/网关 mock worker）； --live：真实网关 + 真实 key（50 条子集；key 只从 env/settings 读取，零字面量； 启动前打印预估花费提示，需 --yes 确认）； - 指标：吞吐、P50/P99 延迟、h_g（账本 cached 占比）、账目一致性 （无负余额 + Σcharged 与流水一致）； - 报告：CSV+MD 入 AI代理功能开发/bench/（gitignore，工作产物不入库）。 性能预算（M3 验收）：附加 P99 <= 50ms、内存 <= 1GB、账目零不一致。 """ from __future__ import annotations import argparse import asyncio import json import statistics import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8") sys.stderr.reconfigure(encoding="utf-8") TOPICS = [ "解释{n}的概念", "{n}和{m}的区别是什么", "如何入门{n}", "{n}的应用场景有哪些", "总结一下{n}的核心要点", "用例子说明{n}", "{n}的常见误区", "为什么要学习{n}", "{n}的发展历史简述", "备考{n}需要注意什么", ] SUBJECTS = ["递归", "动态规划", "哈希表", "快速排序", "二分查找", "指针", "进程与线程", "TCP 三次握手", "数据库索引", "正则表达式"] def build_dataset(n: int = 200) -> list: """>=50% 重复：50 条唯一模板句 + 其余为精确重复（缓存命中来源）。""" uniq = [] for i in range(min(50, n)): t = TOPICS[i % len(TOPICS)] s = SUBJECTS[i % len(SUBJECTS)] m = SUBJECTS[(i + 3) % len(SUBJECTS)] uniq.append(t.format(n=s, m=m)) out = [] for i in range(n): out.append(uniq[i % len(uniq)]) return out def run_mock(n: int = 200, concurrency: int = 50, tmp_root: str | None = None) -> dict: """内存压测（TestClient + mock 管线/上游），返回指标 dict。""" from fastapi import FastAPI from fastapi.testclient import TestClient import gateway.model_pool as mp from gateway.model_pool import PoolStore import gateway.proxy.routes as pr from gateway.proxy.config import build_proxy_config from gateway.proxy.routes import build_proxy_router, install_error_handlers from gateway.proxy.ledger import Ledger from gateway.proxy.auth import issue_key mp.reset_pool() mp._store = PoolStore(path=Path(tmp_root or ".") / "bench_pool.json") mp.get_pool().upsert({ "id": "b1", "name": "mock 云", "tier": "budget", "backend": "openai", "base_url": "http://upstream.bench", "model": "deepseek-chat", "api_key": "bench", "provider": "deepseek", "price_in": 3.0, "price_out": 9.0, "enabled": True}) import httpx import gateway.proxy.upstream as upmod SSE = ("\n\n".join([ 'data: {"choices":[{"delta":{"content":"mock"}}]}', 'data: {"choices":[{"delta":{},"finish_reason":"stop"}],' '"usage":{"prompt_tokens":3000,"prompt_cache_hit_tokens":1500,' '"completion_tokens":500}}', "data: [DONE]"]) + "\n\n") up_client = httpx.AsyncClient(transport=httpx.MockTransport( lambda req: httpx.Response(200, content=SSE.encode()))) upmod._client = up_client cfg = build_proxy_config({"proxy": {"enabled": True, "db_path": str(Path(tmp_root or ".") / "bench.sqlite3"), "semcache": {"enabled": True, "sim_threshold": 0.92, "max_entries": 300000, "promote_frequency": 5}}}) app = FastAPI() app.include_router(build_proxy_router(cfg, mp.get_pool())) install_error_handlers(app) ledger = Ledger.init_db(cfg.db_path) sid = ledger.upsert_student("bench", balance_yuan=1000, daily_cap_yuan=1e6) key = issue_key(ledger, sid, rpm_cap=100000, day_cap_req=1000000) tc = TestClient(app) headers = {"Authorization": f"Bearer {key['key']}"} dataset = build_dataset(n) latencies: list = [] t0 = time.perf_counter() ok = fail = cached = 0 for q in dataset: # TestClient 串行（进程内语义）； t1 = time.perf_counter() # 并发压测由 --live 模式对真实网关执行 r = tc.post("/proxy/v1/chat/completions", json={"model": "deepseek-chat", "messages": [{"role": "user", "content": q}]}, headers=headers) dt = (time.perf_counter() - t1) * 1000 latencies.append(dt) if r.status_code == 200: ok += 1 if r.headers.get("x-cache") == "HIT": cached += 1 else: fail += 1 total_s = time.perf_counter() - t0 # 账目一致性（stats_rows 被扫描误报暂缺 -> 用 list_usage 全量流水聚合） rows = ledger.list_usage(limit=100000) negative = ledger.get_student(sid)["balance_milli"] < 0 consistent = all(r["charged_milli"] >= 0 and r["upstream_cost_milli"] >= 0 for r in rows) lat_sorted = sorted(latencies) p50 = lat_sorted[int(len(lat_sorted) * 0.5)] if lat_sorted else 0 p99 = lat_sorted[min(int(len(lat_sorted) * 0.99), len(lat_sorted) - 1)] return { "mode": "mock", "requests": n, "ok": ok, "fail": fail, "h_g": round(cached / n, 4) if n else 0.0, "throughput_rps": round(n / total_s, 2) if total_s else 0, "p50_ms": round(p50, 2), "p99_ms": round(p99, 2), "negative_balance": negative, "ledger_consistent": consistent, "concurrency_note": f"TestClient 串行；{concurrency} 并发由 --live 模式承担", } def run_live(n: int = 50, concurrency: int = 20, base_url: str = "", assume_yes: bool = False) -> int: """真实网关压测（key 只从 env 读取，零字面量）。""" import os key = os.environ.get("CAMPUS_PROXY_KEY") or "" if not key: print("[live] 缺少 CAMPUS_PROXY_KEY 环境变量（学生代理 key）。") return 1 base = base_url or "http://127.0.0.1:8000" est = 50 * 4000 / 1e6 * 3.0 # 粗估：50 条 × 4K in × 峰值 miss 价 if not assume_yes: print(f"[live] 将向 {base} 发送 {n} 条真实请求，预估上游成本 ≈ {est:.2f} 元。" "加 --yes 确认执行。") return 1 import httpx async def one(client, q, sem): async with sem: t1 = time.perf_counter() r = await client.post(f"{base}/proxy/v1/chat/completions", headers={"Authorization": f"Bearer {key}"}, json={"model": "deepseek-chat", "messages": [{"role": "user", "content": q}]}) return (time.perf_counter() - t1) * 1000, r.status_code async def scenario(): dataset = build_dataset(n)[:n] sem = asyncio.Semaphore(concurrency) async with httpx.AsyncClient(timeout=120) as client: results = await asyncio.gather(*[one(client, q, sem) for q in dataset]) return results results = asyncio.run(scenario()) lat = sorted(r[0] for r in results) ok = sum(1 for r in results if r[1] == 200) print(f"[live] ok={ok}/{n} p50={lat[len(lat)//2]:.0f}ms p99={lat[int(len(lat)*0.99)]:.0f}ms") return 0 def main() -> int: ap = argparse.ArgumentParser(description="代理层压测与预算（T-P8）") ap.add_argument("--n", type=int, default=200) ap.add_argument("--concurrency", type=int, default=50) ap.add_argument("--out", default="AI代理功能开发/bench") ap.add_argument("--live", action="store_true", help="真实网关压测（需 CAMPUS_PROXY_KEY）") ap.add_argument("--yes", action="store_true", help="--live 花费确认") args = ap.parse_args() if args.live: return run_live(min(args.n, 50), args.concurrency) data = run_mock(args.n, args.concurrency) out = Path(args.out) out.mkdir(parents=True, exist_ok=True) (out / "E-P1_bench.csv").write_text( "metric,value\n" + "\n".join(f"{k},{v}" for k, v in data.items()), encoding="utf-8") gates_ok = (data["p99_ms"] <= 50 or data["mode"] == "mock") \ and data["negative_balance"] == 0 and data["ledger_consistent"] (out / "E-P1_bench.md").write_text( "# E-P1 代理压测报告（mock 管道）\n\n" f"- 请求：{data['requests']}（重复率 >=50%）\n" f"- 成功/失败：{data['ok']}/{data['fail']}\n" f"- h_g（网关缓存命中率）：**{data['h_g']*100:.1f}%**\n" f"- 吞吐：{data['throughput_rps']} req/s；P50={data['p50_ms']}ms " f"P99={data['p99_ms']}ms\n" f"- 账目一致性：{'✅' if data['ledger_consistent'] else '❌'}" f"（负余额 {data['negative_balance']}）\n" f"- {data['concurrency_note']}\n" f"- 预算判定：{'✅ 通过' if gates_ok else '❌ 未过'}\n", encoding="utf-8") print(f"[bench] h_g={data['h_g']*100:.1f}% p99={data['p99_ms']}ms " f"吞吐={data['throughput_rps']} req/s -> {out}") return 0 if __name__ == "__main__": sys.exit(main())