feat(proxy): T-X12 采纳 enterprise-ai-gateway 双桶限流——TPM token 桶 + 语义缓存命中率透出
- auth.RateLimiter.allow_tokens:TPM 令牌桶(容量=tpm_cap,连续回流 tpm/60 每秒, 与 RPM 桶共用 per-key 锁、判定相互独立;tpm_cap<=0 不限) - ProxyConfig.limits.tpm_per_key(默认 60000,0=不限);chat_completions 在并发槽 之后做 TPM 预扣判定(est=字符/3 与计费同口径),拒绝时释放并发槽并回 429 (文案区分并发/RPM/TPM 三种原因) - semcache.stats 增 misses 与 hit_rate;/admin/stats 新增 semcache 段透出 (entries/hits_exact/hits_semantic/misses/hit_rate,与账本 h_g 互补) - 新增 tests/test_tpm_limiter.py 6 项(容量/回流/禁用/双桶独立/key 隔离/命中率)
This commit is contained in:
@@ -86,6 +86,14 @@ def build_proxy_router(cfg: ProxyConfig, pool, settings_provider=None,
|
||||
_limits(), time.time())
|
||||
if not _limits().acquire_slot(ctx["key_id"]):
|
||||
raise QuotaError("并发请求已达该 key 上限")
|
||||
# T-X12(采纳 enterprise-ai-gateway 双桶限流):TPM 桶——RPM 之外加
|
||||
# token 维度(预扣估算 = 字符/3,与计费同口径;cfg.tpm_per_key=0 不限)
|
||||
tpm_cap = int(getattr(cfg, "tpm_per_key", 0) or 0)
|
||||
if tpm_cap > 0:
|
||||
est_tokens = max(1, len(raw_body) // 3)
|
||||
if not _limits().allow_tokens(ctx["key_id"], est_tokens, tpm_cap):
|
||||
_limits().release_slot(ctx["key_id"])
|
||||
raise QuotaError("该 key 的每分钟 token 额度(TPM)已用尽")
|
||||
|
||||
# 语义分析器 live 分流(D-G7:mode=live 才启用;任何异常不影响代理可用性)。
|
||||
# T-X4 修复:settings_provider 由 api.py 注入(原实现引用未导入的
|
||||
@@ -217,10 +225,18 @@ def build_proxy_router(cfg: ProxyConfig, pool, settings_provider=None,
|
||||
in_hit = s["in_hit_tok"]
|
||||
in_miss = s["in_miss_tok"]
|
||||
h_p = in_hit / (in_hit + in_miss) if (in_hit + in_miss) else 0.0
|
||||
# T-X12:进程内语义缓存计数透出(entries/hits/misses/hit_rate)
|
||||
sem_stats = None
|
||||
if cfg.semcache_enabled:
|
||||
try:
|
||||
sem_stats = _get_semcache(cfg, ledger).stats()
|
||||
except Exception: # noqa: BLE001
|
||||
sem_stats = None
|
||||
return {"requests": n, "h_g": round(h_g, 4), "h_p": round(h_p, 4),
|
||||
"revenue_milli": s["revenue_milli"], "cost_milli": s["cost_milli"],
|
||||
"margin_milli": s["revenue_milli"] - s["cost_milli"],
|
||||
"by_bucket": s["by_bucket"],
|
||||
"semcache": sem_stats,
|
||||
"today": today,
|
||||
"upstream_failover": upstream_failover_stats()}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user