"""质量控制器(Judge):评估专家输出,决定是否升级大模型。 - RuleJudge:零依赖启发式(内容覆盖度 / 长度充分性 / 领域格式 / 安全提示), 稳定可测,适合 MVP 与离线演示。 - LLMJudge:可选,基于 transformers 小模型或 API 的 LLM-as-Judge。 设计对齐实现方案:overall_score < judge_fallback_threshold -> 升级大模型。 """ from __future__ import annotations from dataclasses import dataclass, field from typing import Dict, List from .experts import extract_content_terms # 各领域期望的响应长度范围(字符数) _EXPECTED_LEN = { "code": (60, 2000), "math": (60, 2000), "legal": (80, 3000), "medical": (80, 3000), "general": (40, 2000), } # 领域格式检查:响应应包含的标记 _DOMAIN_FORMAT_HINTS = { "code": ["```", "def ", "function", "class "], "math": ["步骤", "推导", "=", "解", "step"], "legal": ["⚠", "法律", "意见", "合规", "contract", "law"], "medical": ["⚠", "就医", "医生", "症状", "诊断", "symptom"], "general": [], } @dataclass class QualityEvaluation: overall_score: float scores: Dict[str, float] = field(default_factory=dict) needs_fallback: bool = False reasons: List[str] = field(default_factory=list) def to_dict(self) -> Dict: return { "overall_score": round(self.overall_score, 4), "scores": {k: round(v, 4) for k, v in self.scores.items()}, "needs_fallback": self.needs_fallback, "reasons": self.reasons, } class BaseJudge: async def evaluate(self, query: str, response: str, domain: str) -> QualityEvaluation: raise NotImplementedError class RuleJudge(BaseJudge): """启发式质量评估(零依赖)。""" def __init__(self, fallback_threshold: float = 0.70): self.fallback_threshold = fallback_threshold async def evaluate(self, query: str, response: str, domain: str) -> QualityEvaluation: scores: Dict[str, float] = {} reasons: List[str] = [] # 1) 内容覆盖度:查询中的内容词有多少出现在响应里 terms = extract_content_terms(query) if terms: lowered = response.lower() # lowercase 一次,避免逐词重复复制长响应 hit = sum(1 for t in terms if t in lowered) coverage = hit / len(terms) scores["coverage"] = coverage if coverage < 0.4: reasons.append(f"内容覆盖度低 ({coverage:.0%})") else: scores["coverage"] = 1.0 # 2) 长度充分性 lo, hi = _EXPECTED_LEN.get(domain, (40, 2000)) n = len(response) if n < lo: scores["length"] = max(0.0, n / lo) reasons.append(f"响应过短 ({n} 字符)") elif n > hi: scores["length"] = 0.8 reasons.append(f"响应过长 ({n} 字符)") else: scores["length"] = 1.0 # 3) 领域格式检查 hints = _DOMAIN_FORMAT_HINTS.get(domain, []) if hints: hit_hints = sum(1 for h in hints if h in response) scores["format"] = min(1.0, 0.4 + 0.2 * hit_hints) if hit_hints == 0: reasons.append("缺少领域格式特征") else: scores["format"] = 1.0 # 4) 安全/免责提示(法律、医疗领域应有警示语) if domain in ("legal", "medical") and ("⚠" not in response and "提示" not in response): scores["safety"] = 0.6 reasons.append("缺少免责提示") else: scores["safety"] = 1.0 weights = {"coverage": 0.4, "length": 0.2, "format": 0.2, "safety": 0.2} overall = sum(scores.get(k, 0.0) * w for k, w in weights.items()) needs = overall < self.fallback_threshold if needs: reasons.append("质量分低于阈值,建议升级大模型") return QualityEvaluation( overall_score=round(overall, 4), scores=scores, needs_fallback=needs, reasons=reasons, ) class LLMJudge(BaseJudge): """可选:LLM-as-Judge(API 后端)。""" def __init__(self, model: str, base_url: str, api_key: str, fallback_threshold: float = 0.70): from .llm_client import OpenAICompatClient self.model = model self.fallback_threshold = fallback_threshold # 请求体与旧实现一致:不下发 temperature / max_tokens self._client = OpenAICompatClient(base_url, api_key, timeout=60.0) async def evaluate(self, query: str, response: str, domain: str) -> QualityEvaluation: prompt = ( f"你是质量评审员。评估以下回答对查询的满足程度,输出 0-1 分(相关性/正确性/完整性)。\n" f"查询: {query}\n领域: {domain}\n回答: {response[:2000]}\n" f"只输出一个 0 到 1 之间的数字。" ) data = await self._client.chat( self.model, [{"role": "user", "content": prompt}], temperature=None, max_tokens=None, ) try: score = float(self._client.completion_text(data).strip()) score = max(0.0, min(1.0, score)) except Exception: score = 0.5 return QualityEvaluation( overall_score=score, scores={"llm_judge": score}, needs_fallback=score < self.fallback_threshold, ) def build_judge(cfg: Dict, fallback_threshold: float = 0.70) -> BaseJudge: """cfg 为 judge 段配置。""" jtype = cfg.get("type", "rule") if jtype == "rule": return RuleJudge(fallback_threshold=fallback_threshold) if jtype == "llm": from .config import get_api_key api_key = cfg.get("api_key") or get_api_key(cfg) return LLMJudge( cfg.get("model", "deepseek-chat"), cfg.get("base_url", "https://api.deepseek.com/v1"), api_key or "", fallback_threshold=fallback_threshold, ) raise ValueError(f"未知 judge 类型: {jtype}(支持 rule | llm)")