feat(proxy): T-X12 采纳 enterprise-ai-gateway 双桶限流——TPM token 桶 + 语义缓存命中率透出
- auth.RateLimiter.allow_tokens:TPM 令牌桶(容量=tpm_cap,连续回流 tpm/60 每秒, 与 RPM 桶共用 per-key 锁、判定相互独立;tpm_cap<=0 不限) - ProxyConfig.limits.tpm_per_key(默认 60000,0=不限);chat_completions 在并发槽 之后做 TPM 预扣判定(est=字符/3 与计费同口径),拒绝时释放并发槽并回 429 (文案区分并发/RPM/TPM 三种原因) - semcache.stats 增 misses 与 hit_rate;/admin/stats 新增 semcache 段透出 (entries/hits_exact/hits_semantic/misses/hit_rate,与账本 h_g 互补) - 新增 tests/test_tpm_limiter.py 6 项(容量/回流/禁用/双桶独立/key 隔离/命中率)
This commit is contained in:
@@ -0,0 +1,70 @@
|
||||
"""TPM 双桶限流与缓存命中率透出(T-X12,采纳 enterprise-ai-gateway 设计)。"""
|
||||
import time
|
||||
|
||||
from gateway.proxy.auth import RateLimiter
|
||||
from gateway.proxy.semcache import SemanticCache
|
||||
|
||||
|
||||
def test_tpm_bucket_capacity_and_consume():
|
||||
"""容量内放行并扣减;超过余量拒绝。"""
|
||||
rl = RateLimiter()
|
||||
assert rl.allow_tokens(1, 100, 1000) is True
|
||||
assert rl.allow_tokens(1, 500, 1000) is True
|
||||
# 已扣 600,剩 400 + 回流(瞬时近似 0):700 > 400+eps -> 拒
|
||||
assert rl.allow_tokens(1, 700, 1000) is False
|
||||
|
||||
|
||||
def test_tpm_bucket_continuous_refill(monkeypatch):
|
||||
"""连续时间回流:elapsed * tpm/60 补充(容量封顶)。"""
|
||||
rl = RateLimiter()
|
||||
clock = {"t": 100.0}
|
||||
monkeypatch.setattr(time, "monotonic", lambda: clock["t"])
|
||||
assert rl.allow_tokens(2, 1000, 2000) is True # 剩 1000
|
||||
clock["t"] = 130.0 # 30s 回流 1000 -> 满 2000
|
||||
assert rl.allow_tokens(2, 1500, 2000) is True # 扣 1500 剩 500
|
||||
clock["t"] = 145.0 # 15s 回流 500 -> 1000
|
||||
assert rl.allow_tokens(2, 1000, 2000) is True # 精确扣空
|
||||
assert rl.allow_tokens(2, 1, 2000) is False # 空桶拒绝
|
||||
|
||||
|
||||
def test_tpm_disabled_when_cap_zero():
|
||||
"""tpm_cap <= 0 = 不限流(恒放行)。"""
|
||||
rl = RateLimiter()
|
||||
assert rl.allow_tokens(3, 10**9, 0) is True
|
||||
|
||||
|
||||
def test_tpm_independent_from_rpm_bucket():
|
||||
"""双桶相互独立:打满 TPM 不影响 RPM 放行,RPM 打空不影响 TPM 记账。"""
|
||||
rl = RateLimiter()
|
||||
assert rl.allow_tokens(4, 5000, 5000) is True # TPM 打空
|
||||
assert rl.allow(4, 10) is True # RPM 仍放行
|
||||
for _ in range(9): # 共 10 次 = 打满 RPM
|
||||
assert rl.allow(4, 10) is True
|
||||
assert rl.allow(4, 10) is False # RPM 空桶
|
||||
assert rl.allow_tokens(4, 1, 5000) is False # TPM 仍空(未被 RPM 动过)
|
||||
|
||||
|
||||
def test_tpm_per_key_isolation():
|
||||
"""按 key 隔离:一个 key 打满不影响另一个。"""
|
||||
rl = RateLimiter()
|
||||
assert rl.allow_tokens(5, 5000, 5000) is True
|
||||
assert rl.allow_tokens(6, 5000, 5000) is True
|
||||
assert rl.allow_tokens(5, 5000, 5000) is False
|
||||
|
||||
|
||||
def test_semcache_stats_hit_rate():
|
||||
"""stats 透出 misses 与 hit_rate(命中率口径 = hits / (hits + misses))。"""
|
||||
class _S:
|
||||
def semcache_rows(self, limit=0):
|
||||
return []
|
||||
|
||||
def put_semcache(self, *a, **k):
|
||||
pass
|
||||
|
||||
c = SemanticCache(_S(), max_entries=10, sim_threshold=0.5, promote_frequency=5)
|
||||
c.put("k1", "请解释一下递归函数的概念", "答案", "m", doc_version=1)
|
||||
assert c.lookup("k1", "请解释一下递归函数的概念", doc_version=1) is not None
|
||||
assert c.lookup("k9", "完全无关的问题内容呢", doc_version=1) is None
|
||||
s = c.stats()
|
||||
assert s["hits_exact"] == 1 and s["misses"] == 1
|
||||
assert s["hit_rate"] == 0.5
|
||||
Reference in New Issue
Block a user