tzt/projectAIpopular
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(proxy): T-X2 上游有序降级链 + failover 三元组如实上报（采纳 cortiq tier 链）

Browse Source 
- routes._fallback_chain：主条目之外的启用真实后端按（档位升序，单价和升序）
  构成有序候选链（≤3），排除 mock/停用/同模型，仅作 D-P4 首 token 前 failover
- _stream_response/_json_response 接入链式候选（原 [entry,entry] 同条目重试升级为真降级链）
- upstream.stream：failover 发生即记 sink[upstream_fallback] 与模块级统计；
  JSON 路径经 X-Upstream-Fallback/Original/Used/Reason 四头如实上报
  （流式路径头已发出不可追溯，由统计聚合暴露）
- /proxy/admin/stats 新增 upstream_failover 块（total + recent 20 条）

pytest 447 passed（T-X1 后 443 + 4）
This commit is contained in:
tzt
2026-09-18 22:35:53 +08:00
parent aa7cb0704c
commit a043493548
5 changed files with 230 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gateway/proxy/routes.py
+42 -9
View File
@@ -30,7 +30,8 @@ from gateway.proxy.errors import (
UpstreamError,
)
from gateway.proxy.pricing import compute
from gateway.proxy.upstream import UpstreamAborted, filter_usage_chunk, stream as upstream_stream
from gateway.proxy.upstream import (UpstreamAborted, failover_stats as upstream_failover_stats,
filter_usage_chunk, stream as upstream_stream)
def install_error_handlers(app) -> None:
@@ -204,7 +205,8 @@ def build_proxy_router(cfg: ProxyConfig, pool) -> APIRouter:
return {"requests": n, "h_g": round(h_g, 4), "h_p": round(h_p, 4),
"revenue_milli": revenue, "cost_milli": cost,
"margin_milli": revenue - cost, "by_bucket": by_bucket,
"today": today}
"today": today,
"upstream_failover": upstream_failover_stats()}
@router.get("/admin/ledger", tags=["proxy-admin"])
async def admin_ledger(request: Request, student_id: int = 0,
@@ -272,6 +274,24 @@ def _budget_headers(budget_mode: str) -> Dict[str, str]:
return {"X-Budget-Mode": budget_mode} if budget_mode and budget_mode != "normal" else {}
def _fallback_chain(pool, entry: Dict[str, Any], max_total: int = 3) -> List[Dict[str, Any]]:
"""上游有序降级链（T-X2，采纳 cortiq tier 链思路）。
主条目之外，取池内其他启用真实后端条目，按 (档位升序, 单价和升序) 排列
——便宜的先顶上；总链长 <= max_total。仅作首 token 前 failover 候选
（D-P4），不做负载均衡（单写者模型）。
"""
from gateway.model_pool import TIERS
tier_rank = {t: i for i, t in enumerate(TIERS)}
rest = [e for e in pool.list().get("entries", [])
if (e.get("enabled") and e.get("id") != entry.get("id")
and e.get("backend") not in ("mock",) and e.get("base_url")
and e.get("model") != entry.get("model"))]
rest.sort(key=lambda e: (tier_rank.get(e.get("tier"), 99),
float(e.get("price_in") or 0) + float(e.get("price_out") or 0)))
return rest[:max(0, int(max_total) - 1)]
def _downgrade_entry(pool, entry: Dict[str, Any], mode: str) -> Optional[Dict[str, Any]]:
"""预算降档（T-X1）：在池内找恰好低一档/最低档的启用条目。
@@ -427,16 +447,18 @@ async def _run_chat(body: dict, headers: Dict[str, str], ctx: Dict[str, Any],
t0 = time.perf_counter()
sink: Dict[str, Any] = {}
chain = _fallback_chain(pool, entry)
try:
if is_stream:
return await _stream_response(body, entry, sink, headers, client_wants_usage,
request_id, ctx, cfg, ledger, model, est, t0,
cache=cache, cacheable=cacheable,
budget_mode=budget_mode)
budget_mode=budget_mode, chain=chain)
return await _json_response(body, entry, sink, request_id, ctx, cfg,
ledger, model, est, t0,
cache=cache, cacheable=cacheable,
headers=headers, budget_mode=budget_mode)
headers=headers, budget_mode=budget_mode,
chain=chain)
except UpstreamAborted as e:
# 流中失败：按已收 usage 结算（无 usage 按字符估算），不缓存（D-P4）
usage = sink.get("usage") or _estimate_usage_from_sink(sink)
@@ -465,7 +487,8 @@ def _estimate_usage_from_sink(sink: Dict[str, Any]) -> Dict[str, int]:
async def _stream_response(body, entry, sink, headers, client_wants_usage,
request_id, ctx, cfg, ledger, model, est, t0,
cache=None, cacheable=False, budget_mode: str = "normal"):
cache=None, cacheable=False, budget_mode: str = "normal",
chain: Optional[List[Dict[str, Any]]] = None):
usage = {"in_miss": 0, "in_hit": 0, "out": 0}
async def gen():
@@ -473,7 +496,8 @@ async def _stream_response(body, entry, sink, headers, client_wants_usage,
chunk_id = f"chatcmpl-{request_id}"
created = int(time.time())
try:
async for raw_bytes in upstream_stream(body, entry, sink, [entry]):
async for raw_bytes in upstream_stream(body, entry, sink,
chain if chain is not None else [entry]):
line = raw_bytes.decode("utf-8").strip()
if not line:
continue
@@ -518,9 +542,11 @@ async def _stream_response(body, entry, sink, headers, client_wants_usage,
async def _json_response(body, entry, sink, request_id, ctx, cfg, ledger,
model, est, t0, cache=None, cacheable=False,
headers=None, budget_mode: str = "normal"):
headers=None, budget_mode: str = "normal",
chain: Optional[List[Dict[str, Any]]] = None):
parts = []
async for raw_bytes in upstream_stream(body, entry, sink, [entry]):
async for raw_bytes in upstream_stream(body, entry, sink,
chain if chain is not None else [entry]):
line = raw_bytes.decode("utf-8").strip()
if not line:
continue
@@ -562,6 +588,13 @@ async def _json_response(body, entry, sink, request_id, ctx, cfg, ledger,
ttl_hours=bucket_cfg.ttl_hours)
except Exception:
pass
fb = sink.get("upstream_fallback") or {}
fb_headers: Dict[str, str] = {}
if fb.get("used"):
fb_headers = {"X-Upstream-Fallback": "1",
"X-Upstream-Original": str(fb.get("original") or ""),
"X-Upstream-Used": str(fb.get("used_model") or ""),
"X-Upstream-Reason": str(fb.get("reason") or "")[:200]}
return JSONResponse({
"id": f"chatcmpl-{request_id}",
"object": "chat.completion",
@@ -576,4 +609,4 @@ async def _json_response(body, entry, sink, request_id, ctx, cfg, ledger,
"total_tokens": usage.get("in_miss", 0) + usage.get("in_hit", 0)
+ usage.get("out", 0)},
}, headers={"X-Request-Id": request_id,
**_budget_headers(budget_mode)})
**_budget_headers(budget_mode), **fb_headers})