feat(proxy): T-X2 上游有序降级链 + failover 三元组如实上报(采纳 cortiq tier 链)

- routes._fallback_chain:主条目之外的启用真实后端按(档位升序,单价和升序)
  构成有序候选链(≤3),排除 mock/停用/同模型,仅作 D-P4 首 token 前 failover
- _stream_response/_json_response 接入链式候选(原 [entry,entry] 同条目重试升级为真降级链)
- upstream.stream:failover 发生即记 sink[upstream_fallback] 与模块级统计;
  JSON 路径经 X-Upstream-Fallback/Original/Used/Reason 四头如实上报
  (流式路径头已发出不可追溯,由统计聚合暴露)
- /proxy/admin/stats 新增 upstream_failover 块(total + recent 20 条)

pytest 447 passed(T-X1 后 443 + 4)
This commit is contained in:
tzt
2026-09-18 22:35:53 +08:00
parent aa7cb0704c
commit a043493548
5 changed files with 230 additions and 21 deletions
+47 -2
View File
@@ -11,7 +11,9 @@
from __future__ import annotations
import json
from typing import Any, AsyncIterator, Dict, List, Optional
import time
from collections import deque
from typing import Any, AsyncIterator, Deque, Dict, List, Optional
import httpx
@@ -42,6 +44,36 @@ class UpstreamAborted(Exception):
"""首 token 已下发后上游失败(D-P4:不可 failover,按已收 usage 计费)。"""
# ---------------- failover 统计(T-X2:如实上报,进程内,D-P9 单进程) ----------------
_FAILOVER_EVENTS_MAX = 100
_failover_events: Deque[Dict[str, Any]] = deque(maxlen=_FAILOVER_EVENTS_MAX)
_failover_total = 0
def _record_failover(original: str, failed_model: str, reason: str) -> None:
"""记录一次「首 token 前失败 -> 切换候选」事件。"""
global _failover_total
_failover_total += 1
_failover_events.append({
"ts": int(time.time()), "original": original,
"failed_model": failed_model, "reason": reason,
})
def failover_stats(recent: int = 20) -> Dict[str, Any]:
"""failover 统计(/proxy/admin/stats 聚合暴露)。"""
events = list(_failover_events)
return {"total": _failover_total,
"recent": events[-max(0, int(recent)):][::-1]}
def reset_failover_stats() -> None:
"""测试用:清空 failover 统计。"""
global _failover_total
_failover_events.clear()
_failover_total = 0
def normalize_usage(provider: str, usage_dict: Dict[str, Any]) -> Dict[str, int]:
"""三家 usage 字段 -> 统一 {in_miss, in_hit, out}。
@@ -89,17 +121,22 @@ def filter_usage_chunk(raw_line: str, client_wants_usage: bool) -> Optional[str]
async def stream(body: dict, entry: Dict[str, Any], usage_sink: Dict[str, Any],
fallback_entries: Optional[List[Dict[str, Any]]] = None
) -> AsyncIterator[bytes]:
"""流式派发(§6 签名扩展 failover 链)。
"""流式派发(§6 签名扩展 failover 链T-X2 扩展降级上报)。
- 始终注入 include_usageusage chunk 交给 usage_sink(透传过滤由调用方用
filter_usage_chunk 决定)。
- D-P4:首 token 前(未 yield 任何字节)失败 -> 切换 fallback_entries
已 yield 后失败 -> 抛 UpstreamAborted。
- usage_sink["usage"] 收敛为归一化 dictsink["ttfb_ms"] 记录首字节耗时。
- T-X2 如实上报:failover 发生时记 sink["upstream_fallback"]
{used, original, used_model, reason},并计入模块级 failover 统计
failover_stats / reset_failover_stats),/proxy/admin/stats 聚合暴露。
"""
import time as _time
candidates: List[Dict[str, Any]] = [entry] + list(fallback_entries or [])
last_err: Optional[Exception] = None
head_model = str(candidates[0].get("model") or "")
first_failure: Optional[Dict[str, Any]] = None
for cand in candidates:
provider = str(cand.get("provider") or "openai")
@@ -130,10 +167,18 @@ async def stream(body: dict, entry: Dict[str, Any], usage_sink: Dict[str, Any],
first = True
usage_sink["ttfb_ms"] = int((_time.perf_counter() - t0) * 1000)
yield (raw + "\n\n").encode("utf-8")
if first_failure is not None:
first_failure["used_model"] = str(cand.get("model") or "")
usage_sink["upstream_fallback"] = dict(first_failure)
return # 正常结束
except Exception as exc: # noqa: BLE001
if first:
raise UpstreamAborted(str(exc)) from exc
last_err = exc
if first_failure is None:
first_failure = {"used": True, "original": head_model,
"used_model": "", "reason": f"{type(exc).__name__}: {exc}"}
_record_failover(head_model, str(cand.get("model") or ""),
f"{type(exc).__name__}: {exc}")
continue # 首 token 前失败 -> failover
raise UpstreamError(f"上游均不可用: {type(last_err).__name__}: {last_err}")