186 lines
6.5 KiB
Python
186 lines
6.5 KiB
Python
"""大模型回退层（最后处理者）：Mock / 降级模板 / 本地模型 / OpenAI 兼容 API。
|
||
|
||
- NoneFallback ：降级模板（零参数，明确告知超出知识库范围）—— L0 最小可用
|
||
- MockFallback ：确定性 mock 大模型（零参数，测试升级路径用）
|
||
- LocalFallback ：本地小模型（≤8B，Q4 量化，OpenAI 兼容端点如 Ollama/vLLM）—— 按需加载
|
||
- APIFallback ：远程 OpenAI 兼容 API（可选，默认关闭）
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
from typing import Dict, Optional
|
||
|
||
from .models import ExpertResponse
|
||
|
||
|
||
class FallbackProvider:
|
||
name: str = "fallback"
|
||
|
||
async def generate(self, query: str) -> ExpertResponse:
|
||
raise NotImplementedError
|
||
|
||
|
||
class NoneFallback(FallbackProvider):
|
||
"""降级模板：零参数兜底，明确告知查询超出知识库范围。"""
|
||
|
||
def __init__(self, model: str = "none"):
|
||
self.model = model
|
||
self.name = "fallback-none"
|
||
|
||
async def generate(self, query: str) -> ExpertResponse:
|
||
body = (
|
||
f"（降级响应）「{query}」\n\n"
|
||
"当前查询超出本地知识库可处理范围（低置信度或质量校验未通过）。\n"
|
||
"可选处理：\n"
|
||
"1. 换个更明确的问法重试\n"
|
||
"2. 启用 L2 本地小模型或配置最后处理者（fallback.type: local）\n"
|
||
)
|
||
return ExpertResponse(
|
||
text=body,
|
||
model_used=self.model,
|
||
latency_ms=0.0,
|
||
tokens=80,
|
||
cost_est=0.0,
|
||
)
|
||
|
||
|
||
class MockFallback(FallbackProvider):
|
||
"""确定性 mock 大模型：标识为 fallback，便于测试升级路径。"""
|
||
|
||
def __init__(self, model: str = "mock-large"):
|
||
self.model = model
|
||
self.name = f"fallback-{model}"
|
||
|
||
async def generate(self, query: str) -> ExpertResponse:
|
||
await asyncio.sleep(0.002)
|
||
body = (
|
||
f"（大模型回退）「{query}」\n\n"
|
||
"这是一条来自大模型回退路径的完整回答。\n"
|
||
"要点：\n"
|
||
"1. 对复杂/跨域任务给出综合推理\n"
|
||
"2. 补充领域专家未覆盖的上下文\n"
|
||
"3. 给出可执行的后续建议\n"
|
||
)
|
||
return ExpertResponse(
|
||
text=body,
|
||
model_used=self.model,
|
||
latency_ms=2.0,
|
||
tokens=120,
|
||
cost_est=2.0 * 120 / 1_000_000,
|
||
)
|
||
|
||
|
||
class LocalFallback(FallbackProvider):
|
||
"""本地小模型最后处理者（≤8B，如 DeepSeek-R1-Distill-Qwen-7B Q4）。
|
||
|
||
通过 OpenAI 兼容端点调用（Ollama 默认 11434/v1，vLLM 默认 8001/v1），
|
||
模型按需加载、用完即卸载（由本地推理服务管理），不常驻显存。
|
||
"""
|
||
|
||
def __init__(self, model: str, base_url: str = "http://127.0.0.1:11434/v1",
|
||
api_key: str = ""):
|
||
self.model = model
|
||
self.base_url = base_url.rstrip("/")
|
||
self.api_key = api_key
|
||
self.name = f"fallback-local-{model}"
|
||
self._client = None
|
||
|
||
def _get_client(self):
|
||
if self._client is None:
|
||
import httpx
|
||
self._client = httpx.AsyncClient(timeout=120.0)
|
||
return self._client
|
||
|
||
async def generate(self, query: str) -> ExpertResponse:
|
||
client = self._get_client()
|
||
headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
|
||
resp = await client.post(
|
||
f"{self.base_url}/chat/completions",
|
||
headers=headers,
|
||
json={
|
||
"model": self.model,
|
||
"messages": [{"role": "user", "content": query}],
|
||
"temperature": 0.3,
|
||
"max_tokens": 2048,
|
||
},
|
||
)
|
||
resp.raise_for_status()
|
||
data = resp.json()
|
||
body = data["choices"][0]["message"]["content"]
|
||
usage = data.get("usage", {})
|
||
tokens = usage.get("completion_tokens", int(len(body) / 2.2))
|
||
return ExpertResponse(
|
||
text=body,
|
||
model_used=self.model,
|
||
latency_ms=0.0,
|
||
tokens=tokens,
|
||
cost_est=0.0, # 本地推理成本按电费计，模型层成本记为 0（相对 API）
|
||
)
|
||
|
||
|
||
class APIFallback(FallbackProvider):
|
||
"""OpenAI 兼容大模型 API（如 DeepSeek / OpenAI / 本地 vLLM）。"""
|
||
|
||
def __init__(self, model: str, base_url: str, api_key: str):
|
||
self.model = model
|
||
self.base_url = base_url.rstrip("/")
|
||
self.api_key = api_key
|
||
self.name = f"fallback-{model}"
|
||
self._client = None
|
||
|
||
def _get_client(self):
|
||
if self._client is None:
|
||
import httpx
|
||
self._client = httpx.AsyncClient(timeout=90.0)
|
||
return self._client
|
||
|
||
async def generate(self, query: str) -> ExpertResponse:
|
||
client = self._get_client()
|
||
resp = await client.post(
|
||
f"{self.base_url}/chat/completions",
|
||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||
json={
|
||
"model": self.model,
|
||
"messages": [{"role": "user", "content": query}],
|
||
"temperature": 0.3,
|
||
"max_tokens": 2048,
|
||
},
|
||
)
|
||
resp.raise_for_status()
|
||
data = resp.json()
|
||
body = data["choices"][0]["message"]["content"]
|
||
usage = data.get("usage", {})
|
||
tokens = usage.get("completion_tokens", int(len(body) / 2.2))
|
||
return ExpertResponse(
|
||
text=body,
|
||
model_used=self.model,
|
||
latency_ms=0.0,
|
||
tokens=tokens,
|
||
cost_est=2.0 * tokens / 1_000_000,
|
||
)
|
||
|
||
|
||
def build_fallback(cfg: Dict) -> FallbackProvider:
|
||
"""cfg 为 fallback 段配置。"""
|
||
ftype = cfg.get("type", "mock")
|
||
model = cfg.get("model", "deepseek-v4-flash")
|
||
if ftype == "none":
|
||
return NoneFallback(model=model)
|
||
if ftype == "mock":
|
||
return MockFallback(model=model)
|
||
if ftype == "local":
|
||
return LocalFallback(
|
||
model,
|
||
cfg.get("base_url", "http://127.0.0.1:11434/v1"),
|
||
cfg.get("api_key", ""),
|
||
)
|
||
if ftype == "api":
|
||
import os
|
||
api_key = cfg.get("api_key") or os.environ.get(cfg.get("api_key_env", "API_KEY"))
|
||
if not api_key:
|
||
raise RuntimeError(
|
||
f"APIFallback 缺少 API Key：请设置环境变量 {cfg.get('api_key_env')} 或配置 api_key"
|
||
)
|
||
return APIFallback(model, cfg.get("base_url", "https://api.deepseek.com"), api_key)
|
||
raise ValueError(f"未知 fallback 类型: {ftype}（支持 none | mock | local | api）")
|