tzt/projectAIpopular
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(proxy): T-X2 上游有序降级链 + failover 三元组如实上报（采纳 cortiq tier 链）

Browse Source 
- routes._fallback_chain：主条目之外的启用真实后端按（档位升序，单价和升序）
  构成有序候选链（≤3），排除 mock/停用/同模型，仅作 D-P4 首 token 前 failover
- _stream_response/_json_response 接入链式候选（原 [entry,entry] 同条目重试升级为真降级链）
- upstream.stream：failover 发生即记 sink[upstream_fallback] 与模块级统计；
  JSON 路径经 X-Upstream-Fallback/Original/Used/Reason 四头如实上报
  （流式路径头已发出不可追溯，由统计聚合暴露）
- /proxy/admin/stats 新增 upstream_failover 块（total + recent 20 条）

pytest 447 passed（T-X1 后 443 + 4）
This commit is contained in:
tzt
2026-09-18 22:35:53 +08:00
parent aa7cb0704c
commit a043493548
5 changed files with 230 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gateway/proxy/routes.py
+42 -9
View File
@@ -30,7 +30,8 @@ from gateway.proxy.errors import (
UpstreamError,
)
from gateway.proxy.pricing import compute
from gateway.proxy.upstream import UpstreamAborted, filter_usage_chunk, stream as upstream_stream
from gateway.proxy.upstream import (UpstreamAborted, failover_stats as upstream_failover_stats,
filter_usage_chunk, stream as upstream_stream)
def install_error_handlers(app) -> None:
@@ -204,7 +205,8 @@ def build_proxy_router(cfg: ProxyConfig, pool) -> APIRouter:
return {"requests": n, "h_g": round(h_g, 4), "h_p": round(h_p, 4),
"revenue_milli": revenue, "cost_milli": cost,
"margin_milli": revenue - cost, "by_bucket": by_bucket,
"today": today}
"today": today,
"upstream_failover": upstream_failover_stats()}
@router.get("/admin/ledger", tags=["proxy-admin"])
async def admin_ledger(request: Request, student_id: int = 0,
@@ -272,6 +274,24 @@ def _budget_headers(budget_mode: str) -> Dict[str, str]:
return {"X-Budget-Mode": budget_mode} if budget_mode and budget_mode != "normal" else {}
def _fallback_chain(pool, entry: Dict[str, Any], max_total: int = 3) -> List[Dict[str, Any]]:
"""上游有序降级链（T-X2，采纳 cortiq tier 链思路）。
主条目之外，取池内其他启用真实后端条目，按 (档位升序, 单价和升序) 排列
——便宜的先顶上；总链长 <= max_total。仅作首 token 前 failover 候选
（D-P4），不做负载均衡（单写者模型）。
"""
from gateway.model_pool import TIERS
tier_rank = {t: i for i, t in enumerate(TIERS)}
rest = [e for e in pool.list().get("entries", [])
if (e.get("enabled") and e.get("id") != entry.get("id")
and e.get("backend") not in ("mock",) and e.get("base_url")
and e.get("model") != entry.get("model"))]
rest.sort(key=lambda e: (tier_rank.get(e.get("tier"), 99),
float(e.get("price_in") or 0) + float(e.get("price_out") or 0)))
return rest[:max(0, int(max_total) - 1)]
def _downgrade_entry(pool, entry: Dict[str, Any], mode: str) -> Optional[Dict[str, Any]]:
"""预算降档（T-X1）：在池内找恰好低一档/最低档的启用条目。
@@ -427,16 +447,18 @@ async def _run_chat(body: dict, headers: Dict[str, str], ctx: Dict[str, Any],
t0 = time.perf_counter()
sink: Dict[str, Any] = {}
chain = _fallback_chain(pool, entry)
try:
if is_stream:
return await _stream_response(body, entry, sink, headers, client_wants_usage,
request_id, ctx, cfg, ledger, model, est, t0,
cache=cache, cacheable=cacheable,
budget_mode=budget_mode)
budget_mode=budget_mode, chain=chain)
return await _json_response(body, entry, sink, request_id, ctx, cfg,
ledger, model, est, t0,
cache=cache, cacheable=cacheable,
headers=headers, budget_mode=budget_mode)
headers=headers, budget_mode=budget_mode,
chain=chain)
except UpstreamAborted as e:
# 流中失败：按已收 usage 结算（无 usage 按字符估算），不缓存（D-P4）
usage = sink.get("usage") or _estimate_usage_from_sink(sink)
@@ -465,7 +487,8 @@ def _estimate_usage_from_sink(sink: Dict[str, Any]) -> Dict[str, int]:
async def _stream_response(body, entry, sink, headers, client_wants_usage,
request_id, ctx, cfg, ledger, model, est, t0,
cache=None, cacheable=False, budget_mode: str = "normal"):
cache=None, cacheable=False, budget_mode: str = "normal",
chain: Optional[List[Dict[str, Any]]] = None):
usage = {"in_miss": 0, "in_hit": 0, "out": 0}
async def gen():
@@ -473,7 +496,8 @@ async def _stream_response(body, entry, sink, headers, client_wants_usage,
chunk_id = f"chatcmpl-{request_id}"
created = int(time.time())
try:
async for raw_bytes in upstream_stream(body, entry, sink, [entry]):
async for raw_bytes in upstream_stream(body, entry, sink,
chain if chain is not None else [entry]):
line = raw_bytes.decode("utf-8").strip()
if not line:
continue
@@ -518,9 +542,11 @@ async def _stream_response(body, entry, sink, headers, client_wants_usage,
async def _json_response(body, entry, sink, request_id, ctx, cfg, ledger,
model, est, t0, cache=None, cacheable=False,
headers=None, budget_mode: str = "normal"):
headers=None, budget_mode: str = "normal",
chain: Optional[List[Dict[str, Any]]] = None):
parts = []
async for raw_bytes in upstream_stream(body, entry, sink, [entry]):
async for raw_bytes in upstream_stream(body, entry, sink,
chain if chain is not None else [entry]):
line = raw_bytes.decode("utf-8").strip()
if not line:
continue
@@ -562,6 +588,13 @@ async def _json_response(body, entry, sink, request_id, ctx, cfg, ledger,
ttl_hours=bucket_cfg.ttl_hours)
except Exception:
pass
fb = sink.get("upstream_fallback") or {}
fb_headers: Dict[str, str] = {}
if fb.get("used"):
fb_headers = {"X-Upstream-Fallback": "1",
"X-Upstream-Original": str(fb.get("original") or ""),
"X-Upstream-Used": str(fb.get("used_model") or ""),
"X-Upstream-Reason": str(fb.get("reason") or "")[:200]}
return JSONResponse({
"id": f"chatcmpl-{request_id}",
"object": "chat.completion",
@@ -576,4 +609,4 @@ async def _json_response(body, entry, sink, request_id, ctx, cfg, ledger,
"total_tokens": usage.get("in_miss", 0) + usage.get("in_hit", 0)
+ usage.get("out", 0)},
}, headers={"X-Request-Id": request_id,
**_budget_headers(budget_mode)})
**_budget_headers(budget_mode), **fb_headers})
gateway/proxy/upstream.py
+47 -2
View File
@@ -11,7 +11,9 @@
from __future__ import annotations
import json
from typing import Any, AsyncIterator, Dict, List, Optional
import time
from collections import deque
from typing import Any, AsyncIterator, Deque, Dict, List, Optional
import httpx
@@ -42,6 +44,36 @@ class UpstreamAborted(Exception):
"""首 token 已下发后上游失败（D-P4：不可 failover，按已收 usage 计费）。"""
# ---------------- failover 统计（T-X2：如实上报，进程内，D-P9 单进程） ----------------
_FAILOVER_EVENTS_MAX = 100
_failover_events: Deque[Dict[str, Any]] = deque(maxlen=_FAILOVER_EVENTS_MAX)
_failover_total = 0
def _record_failover(original: str, failed_model: str, reason: str) -> None:
"""记录一次「首 token 前失败 -> 切换候选」事件。"""
global _failover_total
_failover_total += 1
_failover_events.append({
"ts": int(time.time()), "original": original,
"failed_model": failed_model, "reason": reason,
})
def failover_stats(recent: int = 20) -> Dict[str, Any]:
"""failover 统计（/proxy/admin/stats 聚合暴露）。"""
events = list(_failover_events)
return {"total": _failover_total,
"recent": events[-max(0, int(recent)):][::-1]}
def reset_failover_stats() -> None:
"""测试用：清空 failover 统计。"""
global _failover_total
_failover_events.clear()
_failover_total = 0
def normalize_usage(provider: str, usage_dict: Dict[str, Any]) -> Dict[str, int]:
"""三家 usage 字段 -> 统一 {in_miss, in_hit, out}。
@@ -89,17 +121,22 @@ def filter_usage_chunk(raw_line: str, client_wants_usage: bool) -> Optional[str]
async def stream(body: dict, entry: Dict[str, Any], usage_sink: Dict[str, Any],
fallback_entries: Optional[List[Dict[str, Any]]] = None
) -> AsyncIterator[bytes]:
"""流式派发（§6 签名扩展 failover 链）。
"""流式派发（§6 签名扩展 failover 链；T-X2 扩展降级上报）。
- 始终注入 include_usage；usage chunk 交给 usage_sink（透传过滤由调用方用
filter_usage_chunk 决定）。
- D-P4：首 token 前（未 yield 任何字节）失败 -> 切换 fallback_entries；
已 yield 后失败 -> 抛 UpstreamAborted。
- usage_sink["usage"] 收敛为归一化 dict；sink["ttfb_ms"] 记录首字节耗时。
- T-X2 如实上报：failover 发生时记 sink["upstream_fallback"]
{used, original, used_model, reason}，并计入模块级 failover 统计
（failover_stats / reset_failover_stats），/proxy/admin/stats 聚合暴露。
"""
import time as _time
candidates: List[Dict[str, Any]] = [entry] + list(fallback_entries or [])
last_err: Optional[Exception] = None
head_model = str(candidates[0].get("model") or "")
first_failure: Optional[Dict[str, Any]] = None
for cand in candidates:
provider = str(cand.get("provider") or "openai")
@@ -130,10 +167,18 @@ async def stream(body: dict, entry: Dict[str, Any], usage_sink: Dict[str, Any],
first = True
usage_sink["ttfb_ms"] = int((_time.perf_counter() - t0) * 1000)
yield (raw + "\n\n").encode("utf-8")
if first_failure is not None:
first_failure["used_model"] = str(cand.get("model") or "")
usage_sink["upstream_fallback"] = dict(first_failure)
return # 正常结束
except Exception as exc: # noqa: BLE001
if first:
raise UpstreamAborted(str(exc)) from exc
last_err = exc
if first_failure is None:
first_failure = {"used": True, "original": head_model,
"used_model": "", "reason": f"{type(exc).__name__}: {exc}"}
_record_failover(head_model, str(cand.get("model") or ""),
f"{type(exc).__name__}: {exc}")
continue # 首 token 前失败 -> failover
raise UpstreamError(f"上游均不可用: {type(last_err).__name__}: {last_err}")