feat(proxy): T-X2 上游有序降级链 + failover 三元组如实上报(采纳 cortiq tier 链)
- routes._fallback_chain:主条目之外的启用真实后端按(档位升序,单价和升序) 构成有序候选链(≤3),排除 mock/停用/同模型,仅作 D-P4 首 token 前 failover - _stream_response/_json_response 接入链式候选(原 [entry,entry] 同条目重试升级为真降级链) - upstream.stream:failover 发生即记 sink[upstream_fallback] 与模块级统计; JSON 路径经 X-Upstream-Fallback/Original/Used/Reason 四头如实上报 (流式路径头已发出不可追溯,由统计聚合暴露) - /proxy/admin/stats 新增 upstream_failover 块(total + recent 20 条) pytest 447 passed(T-X1 后 443 + 4)
This commit is contained in:
@@ -11,7 +11,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, AsyncIterator, Dict, List, Optional
|
||||
import time
|
||||
from collections import deque
|
||||
from typing import Any, AsyncIterator, Deque, Dict, List, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
@@ -42,6 +44,36 @@ class UpstreamAborted(Exception):
|
||||
"""首 token 已下发后上游失败(D-P4:不可 failover,按已收 usage 计费)。"""
|
||||
|
||||
|
||||
# ---------------- failover 统计(T-X2:如实上报,进程内,D-P9 单进程) ----------------
|
||||
_FAILOVER_EVENTS_MAX = 100
|
||||
_failover_events: Deque[Dict[str, Any]] = deque(maxlen=_FAILOVER_EVENTS_MAX)
|
||||
_failover_total = 0
|
||||
|
||||
|
||||
def _record_failover(original: str, failed_model: str, reason: str) -> None:
|
||||
"""记录一次「首 token 前失败 -> 切换候选」事件。"""
|
||||
global _failover_total
|
||||
_failover_total += 1
|
||||
_failover_events.append({
|
||||
"ts": int(time.time()), "original": original,
|
||||
"failed_model": failed_model, "reason": reason,
|
||||
})
|
||||
|
||||
|
||||
def failover_stats(recent: int = 20) -> Dict[str, Any]:
|
||||
"""failover 统计(/proxy/admin/stats 聚合暴露)。"""
|
||||
events = list(_failover_events)
|
||||
return {"total": _failover_total,
|
||||
"recent": events[-max(0, int(recent)):][::-1]}
|
||||
|
||||
|
||||
def reset_failover_stats() -> None:
|
||||
"""测试用:清空 failover 统计。"""
|
||||
global _failover_total
|
||||
_failover_events.clear()
|
||||
_failover_total = 0
|
||||
|
||||
|
||||
def normalize_usage(provider: str, usage_dict: Dict[str, Any]) -> Dict[str, int]:
|
||||
"""三家 usage 字段 -> 统一 {in_miss, in_hit, out}。
|
||||
|
||||
@@ -89,17 +121,22 @@ def filter_usage_chunk(raw_line: str, client_wants_usage: bool) -> Optional[str]
|
||||
async def stream(body: dict, entry: Dict[str, Any], usage_sink: Dict[str, Any],
|
||||
fallback_entries: Optional[List[Dict[str, Any]]] = None
|
||||
) -> AsyncIterator[bytes]:
|
||||
"""流式派发(§6 签名扩展 failover 链)。
|
||||
"""流式派发(§6 签名扩展 failover 链;T-X2 扩展降级上报)。
|
||||
|
||||
- 始终注入 include_usage;usage chunk 交给 usage_sink(透传过滤由调用方用
|
||||
filter_usage_chunk 决定)。
|
||||
- D-P4:首 token 前(未 yield 任何字节)失败 -> 切换 fallback_entries;
|
||||
已 yield 后失败 -> 抛 UpstreamAborted。
|
||||
- usage_sink["usage"] 收敛为归一化 dict;sink["ttfb_ms"] 记录首字节耗时。
|
||||
- T-X2 如实上报:failover 发生时记 sink["upstream_fallback"]
|
||||
{used, original, used_model, reason},并计入模块级 failover 统计
|
||||
(failover_stats / reset_failover_stats),/proxy/admin/stats 聚合暴露。
|
||||
"""
|
||||
import time as _time
|
||||
candidates: List[Dict[str, Any]] = [entry] + list(fallback_entries or [])
|
||||
last_err: Optional[Exception] = None
|
||||
head_model = str(candidates[0].get("model") or "")
|
||||
first_failure: Optional[Dict[str, Any]] = None
|
||||
|
||||
for cand in candidates:
|
||||
provider = str(cand.get("provider") or "openai")
|
||||
@@ -130,10 +167,18 @@ async def stream(body: dict, entry: Dict[str, Any], usage_sink: Dict[str, Any],
|
||||
first = True
|
||||
usage_sink["ttfb_ms"] = int((_time.perf_counter() - t0) * 1000)
|
||||
yield (raw + "\n\n").encode("utf-8")
|
||||
if first_failure is not None:
|
||||
first_failure["used_model"] = str(cand.get("model") or "")
|
||||
usage_sink["upstream_fallback"] = dict(first_failure)
|
||||
return # 正常结束
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if first:
|
||||
raise UpstreamAborted(str(exc)) from exc
|
||||
last_err = exc
|
||||
if first_failure is None:
|
||||
first_failure = {"used": True, "original": head_model,
|
||||
"used_model": "", "reason": f"{type(exc).__name__}: {exc}"}
|
||||
_record_failover(head_model, str(cand.get("model") or ""),
|
||||
f"{type(exc).__name__}: {exc}")
|
||||
continue # 首 token 前失败 -> failover
|
||||
raise UpstreamError(f"上游均不可用: {type(last_err).__name__}: {last_err}")
|
||||
|
||||
Reference in New Issue
Block a user