tzt
b2fa8c3c81
算法： - RouterCache：语义条目写入时预计算向量范数、语义查找单遍完成（消除命中后二次 O(N) 查找）、 相似度=1.0 提前终止；微基准（3000 条目×200 查询）：3986ms -> 1685ms，2.37x - TaskGraph.topo_order：O(V²logV) 重排序/成员扫描 -> 邻接表+deque 的 O(V+E) Kahn， 输出顺序契约不变（初始就绪层按插入序、循环依赖按插入序兜底、未知依赖忽略） - RuleClassifier：同分决胜按领域名字典序（与规则表排列无关），次高分 O(n) 扫描 工程卫生： - .mimosa/（扫描器工作目录）加入 .gitignore 并移出索引 - test_review 抽样测试改用内联确定性 LCG，消除 2 个低危（不安全随机数） 测试：新增 11 项（topo 契约 6 + 缓存回归 3 + 分类器 2） pytest 230 passed（基线 219 全绿 + 11）
264 lines
13 KiB
Python
264 lines
13 KiB
Python
"""意图分类器：识别查询领域（code/math/legal/medical/general）与难度。
|
||
|
||
- RuleClassifier：关键词/正则规则打分，纯标准库，零依赖，可离线运行。
|
||
- HuggingFaceClassifier：可选，基于 transformers 的分类模型（需安装 ML 依赖）。
|
||
|
||
置信度设计：每个领域有一组 (关键词, 权重)。命中权重求和得原始分 s，
|
||
confidence = 1 - exp(-s)，保证 s=1 -> 0.63，s=2 -> 0.86，s=3 -> 0.95。
|
||
无领域命中（或最高分领域为 general）时置信度低，触发 should_fallback。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import math
|
||
from typing import Dict, List, Optional, Tuple
|
||
|
||
from .difficulty import estimate_difficulty
|
||
from .models import Classification
|
||
|
||
# ---------------------------------------------------------------
|
||
# 领域关键词规则： (关键词, 权重)
|
||
# ---------------------------------------------------------------
|
||
DOMAIN_RULES: Dict[str, List[Tuple[str, float]]] = {
|
||
"code": [
|
||
# 中文
|
||
("python", 1.2), ("java", 1.2), ("javascript", 1.2), ("typescript", 1.2),
|
||
("代码", 1.2), ("编程", 1.2), ("函数", 0.9), ("接口", 0.8), ("报错", 0.9),
|
||
("调试", 0.9), ("部署", 0.8), ("算法", 0.8), ("数组", 0.8), ("排序", 0.9),
|
||
("正则", 0.8), ("数据库", 0.7), ("sql", 0.8), ("git", 0.7), ("api", 0.7),
|
||
("变量", 0.7), ("循环", 0.7), ("递归", 0.8), ("重构", 0.8), ("编译", 0.9),
|
||
("测试", 0.6), ("前端", 0.8), ("后端", 0.8), ("爬虫", 0.8), ("脚本", 0.7),
|
||
# 英文
|
||
("function", 0.9), ("class", 0.8), ("bug", 0.9), ("debug", 0.9),
|
||
("compile", 0.9), ("error", 0.6), ("code", 0.7), ("script", 0.7),
|
||
("algorithm", 0.8), ("sort", 0.7), ("array", 0.7), ("regex", 0.8),
|
||
("import", 0.7), ("loop", 0.7), ("recursion", 0.8), ("refactor", 0.8),
|
||
("deploy", 0.8), ("docker", 0.8), ("kubernetes", 0.8),
|
||
("async", 0.7), ("flask", 0.7), ("django", 0.7), ("api", 0.7),
|
||
("索引", 0.8), ("优化", 0.7), ("查询", 0.6),
|
||
],
|
||
"math": [
|
||
("数学", 1.2), ("方程", 1.0), ("求解", 0.8), ("导数", 1.0), ("积分", 1.0),
|
||
("矩阵", 0.9), ("概率", 0.9), ("统计", 0.8), ("证明", 0.8), ("定理", 0.9),
|
||
("微积分", 1.1), ("代数", 0.9), ("几何", 0.9), ("不等式", 0.9),
|
||
("equation", 1.0), ("derivative", 1.0), ("integral", 1.0), ("calculus", 1.1),
|
||
("matrix", 0.9), ("probability", 0.9), ("statistics", 0.8), ("proof", 0.8),
|
||
("theorem", 0.9), ("algebra", 0.9), ("geometry", 0.9), ("sqrt", 0.8),
|
||
("gcd", 0.8), ("lim", 0.8), ("polynomial", 0.9), ("summation", 0.7),
|
||
("math", 0.7), ("解", 0.6), ("计算", 0.8), ("等于", 0.6), ("求值", 0.7), ("函数", 0.6),
|
||
],
|
||
"legal": [
|
||
("法律", 1.2), ("合同", 1.0), ("法条", 1.0), ("合规", 1.0), ("诉讼", 1.0),
|
||
("知识产权", 1.1), ("版权", 0.9), ("专利", 0.9), ("违约", 0.9), ("赔偿", 0.8),
|
||
("仲裁", 0.9), ("劳动法", 1.0), ("刑法", 1.0), ("民法典", 1.0),
|
||
("法规", 0.8), ("条款", 0.7), ("律师", 0.8), ("起诉", 0.9), ("判决", 0.9),
|
||
# 劳动法
|
||
("加班", 0.9), ("加班费", 1.0), ("工资", 0.8), ("辞退", 0.9), ("裁员", 0.9),
|
||
("试用期", 0.9), ("社保", 0.8), ("公积金", 0.8), ("年假", 0.9), ("离职", 0.8),
|
||
("解除劳动合同", 1.1), ("经济补偿", 1.0), ("竞业", 1.0),
|
||
# 房产/婚姻/消费者
|
||
("租房", 0.9), ("买房", 0.9), ("购房", 0.9), ("押金", 0.8), ("房贷", 0.9),
|
||
("离婚", 1.0), ("继承", 0.9), ("遗产", 0.9), ("抚养权", 0.9), ("遗嘱", 0.9),
|
||
("退款", 0.9), ("退货", 0.8), ("消费者", 0.8), ("七天无理由", 1.0), ("维权", 0.8),
|
||
("law", 1.0), ("legal", 1.1), ("contract", 1.0), ("compliance", 1.0),
|
||
("litigation", 1.0), ("copyright", 0.9), ("patent", 0.9), ("trademark", 0.9),
|
||
("liability", 0.9), ("regulatory", 0.8), ("jurisdiction", 0.9),
|
||
("clause", 0.8), ("agreement", 0.7), ("申请", 0.6),
|
||
],
|
||
"medical": [
|
||
("医疗", 1.2), ("药物", 1.0), ("症状", 1.0), ("诊断", 1.0), ("治疗", 0.9),
|
||
("医生", 0.9), ("血压", 0.9), ("高血压", 1.0), ("糖尿病", 1.0), ("感冒", 0.9),
|
||
("剂量", 0.9), ("副作用", 0.9), ("手术", 0.9), ("患者", 0.9),
|
||
("吃药", 0.9), ("发烧", 1.0), ("疫苗", 0.9), ("感染", 0.9), ("体检", 0.7),
|
||
# 急救/消化/心理/营养/儿科
|
||
("烫伤", 1.0), ("烧伤", 1.0), ("止血", 0.9), ("扭伤", 0.9), ("中暑", 1.0),
|
||
("急救", 0.9), ("腹泻", 0.9), ("拉肚子", 0.9), ("便秘", 0.9), ("胃", 0.7),
|
||
("失眠", 0.9), ("焦虑", 0.9), ("抑郁", 0.9), ("压力", 0.6), ("睡眠", 0.7),
|
||
("减肥", 0.8), ("营养", 0.7), ("卡路里", 0.9), ("儿童", 0.8), ("婴儿", 0.9),
|
||
("宝宝", 0.8), ("抗生素", 0.9), ("止咳", 0.9),
|
||
("medical", 1.0), ("patient", 0.9), ("symptom", 1.0), ("disease", 0.9),
|
||
("diagnosis", 1.0), ("treatment", 0.8), ("prescription", 1.0),
|
||
("dosage", 0.9), ("side effect", 0.9), ("hypertension", 1.0),
|
||
("diabetes", 1.0), ("surgery", 0.8), ("clinic", 0.7), ("vaccine", 0.9),
|
||
("infection", 0.9), ("first aid", 0.9), ("insomnia", 0.9),
|
||
],
|
||
"general": [
|
||
("总结", 0.4), ("翻译", 0.4), ("介绍", 0.4), ("解释", 0.3),
|
||
("summarize", 0.4), ("translate", 0.4), ("explain", 0.3),
|
||
("introduce", 0.3), ("what is", 0.3), ("tell me", 0.3),
|
||
("write an essay", 0.4), ("邮件", 0.4), ("email", 0.3),
|
||
("推荐", 0.3), ("评价", 0.3),
|
||
],
|
||
"finance": [
|
||
("理财", 1.0), ("投资", 1.0), ("基金", 1.0), ("股票", 1.0), ("债券", 0.9),
|
||
("存款", 0.9), ("储蓄", 0.8), ("利率", 0.8), ("利息", 0.8), ("贷款", 1.0),
|
||
("房贷", 1.0), ("月供", 0.9), ("保险", 0.9), ("理赔", 0.9), ("保费", 0.8),
|
||
("信用卡", 1.0), ("征信", 0.9), ("逾期", 0.9), ("分期", 0.8), ("记账", 0.7),
|
||
("预算", 0.7), ("理财规划", 1.0), ("收益率", 0.9), ("定投", 0.9),
|
||
("invest", 0.8), ("fund", 0.8), ("stock", 0.9), ("loan", 0.9),
|
||
("mortgage", 0.9), ("insurance", 0.9), ("credit card", 0.9),
|
||
("finance", 0.8), ("money", 0.6), ("lpr", 0.9), ("投资理财", 1.1),
|
||
],
|
||
"life": [
|
||
("菜谱", 0.9), ("做饭", 0.8), ("烹饪", 0.9), ("美食", 0.8), ("做法", 0.7),
|
||
("旅行", 0.9), ("旅游", 0.9), ("攻略", 0.8), ("机票", 0.8), ("酒店", 0.7),
|
||
("签证", 0.9), ("景点", 0.8), ("自驾", 0.8),
|
||
("装修", 0.9), ("收纳", 0.8), ("家居", 0.7), ("清洁", 0.7), ("打扫", 0.7),
|
||
("宠物", 0.9), ("猫", 0.7), ("狗", 0.7), ("猫粮", 0.9), ("驱虫", 0.9),
|
||
("健身", 0.9), ("锻炼", 0.8), ("跑步", 0.8), ("增肌", 0.9), ("减脂", 0.9),
|
||
("瑜伽", 0.8), ("天气", 0.7), ("气温", 0.7),
|
||
("recipe", 0.8), ("travel", 0.9), ("trip", 0.8), ("pet", 0.8),
|
||
("workout", 0.9), ("gym", 0.8), ("weather", 0.7), ("cook", 0.8),
|
||
],
|
||
"education": [
|
||
("学习方法", 1.0), ("怎么学", 0.7), ("高效学习", 1.0), ("记忆", 0.6), ("复习", 0.7),
|
||
("预习", 0.7), ("笔记", 0.6), ("专注", 0.6), ("拖延", 0.7), ("学习效率", 0.9),
|
||
("考试", 0.9), ("备考", 1.0), ("刷题", 0.9), ("模拟考", 0.9), ("中考", 0.9),
|
||
("高考", 0.9), ("考研", 0.9), ("考前", 0.7),
|
||
("英语", 0.8), ("单词", 0.7), ("口语", 0.8), ("听力", 0.7), ("雅思", 1.0),
|
||
("托福", 1.0), ("四级", 0.9), ("六级", 0.9), ("背单词", 0.9),
|
||
("选课", 0.9), ("课程", 0.6), ("专业选择", 0.9), ("报班", 0.8), ("网课", 0.7),
|
||
("自学", 0.7), ("职业规划", 1.0), ("求职", 0.9), ("面试", 0.8), ("简历", 0.8),
|
||
("实习", 0.7), ("跳槽", 0.8), ("转行", 0.9),
|
||
("study", 0.8), ("exam", 0.9), ("language", 0.7), ("career", 0.8),
|
||
("interview", 0.8), ("education", 0.7), ("learn", 0.6),
|
||
],
|
||
}
|
||
|
||
_STOPWORDS = {
|
||
"的", "了", "吗", "呢", "啊", "是", "在", "有", "和", "与", "或", "及", "一个", "如何",
|
||
"the", "a", "an", "is", "are", "to", "of", "in", "on", "for", "with", "and",
|
||
"or", "do", "does", "can", "could", "would", "should", "please", "me", "my",
|
||
}
|
||
|
||
|
||
class BaseClassifier:
|
||
def classify(self, query: str) -> Classification:
|
||
raise NotImplementedError
|
||
|
||
def should_fallback(self, classification: Classification, threshold: float) -> bool:
|
||
return classification.confidence < threshold
|
||
|
||
|
||
class RuleClassifier(BaseClassifier):
|
||
"""基于关键词规则的分类器（零依赖）。
|
||
|
||
domains 参数（可选）：限定只对部分领域打分 —— 两级路由中，
|
||
每个大领域的组内路由模型用 RuleClassifier(domains=组内领域)，
|
||
只认识本组领域，体积与匹配开销约为统一分类器的 1/4。
|
||
"""
|
||
|
||
def __init__(self, confidence_floor: float = 0.55,
|
||
domains: Optional[List[str]] = None):
|
||
self.confidence_floor = confidence_floor
|
||
if domains is None:
|
||
self.rules = DOMAIN_RULES
|
||
else:
|
||
self.rules = {d: DOMAIN_RULES[d] for d in domains if d in DOMAIN_RULES}
|
||
self.domains = list(self.rules.keys())
|
||
|
||
def _score(self, query: str) -> Tuple[Dict[str, float], Dict[str, List[str]]]:
|
||
q = query.lower()
|
||
scores: Dict[str, float] = {}
|
||
matched: Dict[str, List[str]] = {}
|
||
for domain, rules in self.rules.items():
|
||
s = 0.0
|
||
hits = []
|
||
for kw, w in rules:
|
||
if kw in q:
|
||
s += w
|
||
hits.append(kw)
|
||
if s > 0:
|
||
scores[domain] = s
|
||
matched[domain] = hits
|
||
return scores, matched
|
||
|
||
def classify(self, query: str) -> Classification:
|
||
raw, matched = self._score(query)
|
||
if not raw:
|
||
# 完全无命中 -> general，低置信度
|
||
diff, ds = estimate_difficulty(query)
|
||
return Classification(
|
||
domain="general",
|
||
confidence=0.50,
|
||
difficulty=diff,
|
||
difficulty_score=ds,
|
||
raw_scores={},
|
||
matched_rules=[],
|
||
)
|
||
|
||
# 同分决胜：按领域名字典序，保证与规则表排列顺序无关的确定性
|
||
best_domain = max(sorted(raw), key=lambda d: raw[d])
|
||
best_score = raw[best_domain]
|
||
confidence = 1.0 - math.exp(-best_score)
|
||
|
||
# general 领域天然置信度压低
|
||
if best_domain == "general":
|
||
confidence = min(confidence, self.confidence_floor + 0.05)
|
||
|
||
# 与次高分的差距影响置信度（区分度）
|
||
if len(raw) > 1:
|
||
second = max(v for d, v in raw.items() if d != best_domain)
|
||
if second > 0.7 * best_score:
|
||
confidence *= 0.85
|
||
|
||
diff, ds = estimate_difficulty(query)
|
||
return Classification(
|
||
domain=best_domain,
|
||
confidence=round(min(0.99, confidence), 4),
|
||
difficulty=diff,
|
||
difficulty_score=ds,
|
||
raw_scores={k: round(v, 3) for k, v in raw.items()},
|
||
matched_rules=matched.get(best_domain, []),
|
||
)
|
||
|
||
|
||
class HuggingFaceClassifier(BaseClassifier):
|
||
"""可选：基于 transformers 的序列分类模型。
|
||
|
||
仅当安装 torch+transformers 且模型可加载时可用；否则抛错提示。
|
||
"""
|
||
|
||
def __init__(self, model_name: str, num_labels: int = 8, confidence_floor: float = 0.55):
|
||
try:
|
||
from transformers import AutoModelForSequenceClassification, AutoTokenizer
|
||
except ImportError as e:
|
||
raise RuntimeError(
|
||
"HuggingFaceClassifier 需要安装 ML 依赖：pip install -r requirements-ml.txt"
|
||
) from e
|
||
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||
self.model = AutoModelForSequenceClassification.from_pretrained(
|
||
model_name, num_labels=num_labels
|
||
)
|
||
self.labels = ["code", "math", "legal", "medical", "general",
|
||
"finance", "life", "education"]
|
||
self.confidence_floor = confidence_floor
|
||
|
||
def classify(self, query: str) -> Classification:
|
||
import torch # type: ignore
|
||
|
||
inputs = self.tokenizer(query, return_tensors="pt", truncation=True, max_length=256)
|
||
with torch.no_grad():
|
||
logits = self.model(**inputs).logits
|
||
probs = torch.softmax(logits, dim=-1)[0]
|
||
idx = int(probs.argmax())
|
||
diff, ds = estimate_difficulty(query)
|
||
return Classification(
|
||
domain=self.labels[idx],
|
||
confidence=round(float(probs[idx]), 4),
|
||
difficulty=diff,
|
||
difficulty_score=ds,
|
||
raw_scores={self.labels[i]: round(float(probs[i]), 3) for i in range(len(self.labels))},
|
||
)
|
||
|
||
|
||
def build_classifier(cfg: Dict) -> BaseClassifier:
|
||
"""根据配置构建分类器。cfg 为 classifier 段配置。"""
|
||
ctype = cfg.get("type", "rule")
|
||
floor = cfg.get("confidence_floor", 0.55)
|
||
if ctype == "rule":
|
||
return RuleClassifier(confidence_floor=floor)
|
||
if ctype == "hf":
|
||
return HuggingFaceClassifier(cfg.get("model", "Qwen/Qwen3-0.6B"), confidence_floor=floor)
|
||
raise ValueError(f"未知分类器类型: {ctype}（支持 rule | hf）")
|
||
|