feat(proxy): T-X12 采纳 enterprise-ai-gateway 双桶限流——TPM token 桶 + 语义缓存命中率透出

- auth.RateLimiter.allow_tokens:TPM 令牌桶(容量=tpm_cap,连续回流 tpm/60 每秒,
  与 RPM 桶共用 per-key 锁、判定相互独立;tpm_cap<=0 不限)
- ProxyConfig.limits.tpm_per_key(默认 60000,0=不限);chat_completions 在并发槽
  之后做 TPM 预扣判定(est=字符/3 与计费同口径),拒绝时释放并发槽并回 429
  (文案区分并发/RPM/TPM 三种原因)
- semcache.stats 增 misses 与 hit_rate;/admin/stats 新增 semcache 段透出
  (entries/hits_exact/hits_semantic/misses/hit_rate,与账本 h_g 互补)
- 新增 tests/test_tpm_limiter.py 6 项(容量/回流/禁用/双桶独立/key 隔离/命中率)
This commit is contained in:
tzt
2026-09-19 10:09:56 +08:00
parent f04a6c4c43
commit b2b7a64cb3
5 changed files with 117 additions and 1 deletions
+70
View File
@@ -0,0 +1,70 @@
"""TPM 双桶限流与缓存命中率透出(T-X12,采纳 enterprise-ai-gateway 设计)。"""
import time
from gateway.proxy.auth import RateLimiter
from gateway.proxy.semcache import SemanticCache
def test_tpm_bucket_capacity_and_consume():
"""容量内放行并扣减;超过余量拒绝。"""
rl = RateLimiter()
assert rl.allow_tokens(1, 100, 1000) is True
assert rl.allow_tokens(1, 500, 1000) is True
# 已扣 600,剩 400 + 回流(瞬时近似 0):700 > 400+eps -> 拒
assert rl.allow_tokens(1, 700, 1000) is False
def test_tpm_bucket_continuous_refill(monkeypatch):
"""连续时间回流:elapsed * tpm/60 补充(容量封顶)。"""
rl = RateLimiter()
clock = {"t": 100.0}
monkeypatch.setattr(time, "monotonic", lambda: clock["t"])
assert rl.allow_tokens(2, 1000, 2000) is True # 剩 1000
clock["t"] = 130.0 # 30s 回流 1000 -> 满 2000
assert rl.allow_tokens(2, 1500, 2000) is True # 扣 1500 剩 500
clock["t"] = 145.0 # 15s 回流 500 -> 1000
assert rl.allow_tokens(2, 1000, 2000) is True # 精确扣空
assert rl.allow_tokens(2, 1, 2000) is False # 空桶拒绝
def test_tpm_disabled_when_cap_zero():
"""tpm_cap <= 0 = 不限流(恒放行)。"""
rl = RateLimiter()
assert rl.allow_tokens(3, 10**9, 0) is True
def test_tpm_independent_from_rpm_bucket():
"""双桶相互独立:打满 TPM 不影响 RPM 放行,RPM 打空不影响 TPM 记账。"""
rl = RateLimiter()
assert rl.allow_tokens(4, 5000, 5000) is True # TPM 打空
assert rl.allow(4, 10) is True # RPM 仍放行
for _ in range(9): # 共 10 次 = 打满 RPM
assert rl.allow(4, 10) is True
assert rl.allow(4, 10) is False # RPM 空桶
assert rl.allow_tokens(4, 1, 5000) is False # TPM 仍空(未被 RPM 动过)
def test_tpm_per_key_isolation():
"""按 key 隔离:一个 key 打满不影响另一个。"""
rl = RateLimiter()
assert rl.allow_tokens(5, 5000, 5000) is True
assert rl.allow_tokens(6, 5000, 5000) is True
assert rl.allow_tokens(5, 5000, 5000) is False
def test_semcache_stats_hit_rate():
"""stats 透出 misses 与 hit_rate(命中率口径 = hits / (hits + misses))。"""
class _S:
def semcache_rows(self, limit=0):
return []
def put_semcache(self, *a, **k):
pass
c = SemanticCache(_S(), max_entries=10, sim_threshold=0.5, promote_frequency=5)
c.put("k1", "请解释一下递归函数的概念", "答案", "m", doc_version=1)
assert c.lookup("k1", "请解释一下递归函数的概念", doc_version=1) is not None
assert c.lookup("k9", "完全无关的问题内容呢", doc_version=1) is None
s = c.stats()
assert s["hits_exact"] == 1 and s["misses"] == 1
assert s["hit_rate"] == 0.5