feat(proxy): T-X12 采纳 enterprise-ai-gateway 双桶限流——TPM token 桶 + 语义缓存命中率透出
- auth.RateLimiter.allow_tokens:TPM 令牌桶(容量=tpm_cap,连续回流 tpm/60 每秒, 与 RPM 桶共用 per-key 锁、判定相互独立;tpm_cap<=0 不限) - ProxyConfig.limits.tpm_per_key(默认 60000,0=不限);chat_completions 在并发槽 之后做 TPM 预扣判定(est=字符/3 与计费同口径),拒绝时释放并发槽并回 429 (文案区分并发/RPM/TPM 三种原因) - semcache.stats 增 misses 与 hit_rate;/admin/stats 新增 semcache 段透出 (entries/hits_exact/hits_semantic/misses/hit_rate,与账本 h_g 互补) - 新增 tests/test_tpm_limiter.py 6 项(容量/回流/禁用/双桶独立/key 隔离/命中率)
This commit is contained in:
@@ -89,6 +89,7 @@ class RateLimiter:
|
||||
|
||||
def __init__(self, concurrent_per_key: int = 2):
|
||||
self._tokens: Dict[int, tuple[float, float]] = {} # key_id -> (tokens, last_ts)
|
||||
self._tpm: Dict[int, tuple[float, float]] = {} # key_id -> (tokens, last_ts)
|
||||
self._locks: Dict[int, threading.Lock] = {}
|
||||
self._sems: Dict[int, threading.Semaphore] = {}
|
||||
self._global = threading.Lock()
|
||||
@@ -108,6 +109,27 @@ class RateLimiter:
|
||||
self._tokens[key_id] = (tokens - 1.0, now)
|
||||
return True
|
||||
|
||||
def allow_tokens(self, key_id: int, est_tokens: int, tpm_cap: int) -> bool:
|
||||
"""TPM 令牌桶放行判定(T-X12,采纳 enterprise-ai-gateway 双桶设计)。
|
||||
|
||||
容量 = tpm_cap,回流速率 = tpm/60 每秒(连续时间回流补充);
|
||||
est_tokens 为本次预扣估算(字符/3,与预扣计费同口径)。
|
||||
tpm_cap <= 0 视为不限流;与 RPM 桶共用 per-key 锁,判定相互独立。
|
||||
"""
|
||||
if tpm_cap <= 0:
|
||||
return True
|
||||
now = time.monotonic()
|
||||
with self._global:
|
||||
lock = self._locks.setdefault(key_id, threading.Lock())
|
||||
with lock:
|
||||
tokens, last = self._tpm.get(key_id, (float(tpm_cap), now))
|
||||
tokens = min(float(tpm_cap), tokens + (now - last) * (tpm_cap / 60.0))
|
||||
if tokens < est_tokens:
|
||||
self._tpm[key_id] = (tokens, now)
|
||||
return False
|
||||
self._tpm[key_id] = (tokens - est_tokens, now)
|
||||
return True
|
||||
|
||||
def acquire_slot(self, key_id: int) -> bool:
|
||||
"""并发槽(非阻塞);返回 False = 超并发上限(429)。"""
|
||||
with self._global:
|
||||
|
||||
Reference in New Issue
Block a user