feat(v2): T5 WorkerLoop + 接地验证（代码沙箱/facts对照/结构检查）
This commit is contained in:
tzt
@@ -0,0 +1,135 @@
|
||||
"""接地验证器（D4）—— Worker 自验证的"接地"来源。
|
||||
|
||||
验证分层优先级（D4）：
|
||||
可执行验证（跑代码/跑测试） > facts 对照 > 结构检查 > 模型自由判断（最后手段）
|
||||
|
||||
- 可执行验证：code 域 .py 工件在临时目录子进程沙箱运行（timeout、-I 隔离、捕获输出）。
|
||||
- facts 对照：把回答/工件与知识库 facts（statement/keywords）比对覆盖度。
|
||||
- 结构检查：工件非空、长度达标、不含纯占位。
|
||||
|
||||
说明：沙箱目前做"临时目录 + 超时 + 解释器隔离"，Windows 下真正禁网需系统级工具，
|
||||
此处以超时与隔离为主要护栏（文档如实记录）。验证器为纯逻辑，可注入 runner 便于单测。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
# 判定为"含代码"的启发标记
|
||||
_CODE_HINTS = ("def ", "class ", "import ", "return ", "if __name__", "print(")
|
||||
|
||||
|
||||
def detect_artifact_language(name: str) -> str:
|
||||
"""按文件名推断工件语言：python / json / text。"""
|
||||
suffix = Path(name).suffix.lower()
|
||||
if suffix in (".py", ".pyw"):
|
||||
return "python"
|
||||
if suffix in (".json",):
|
||||
return "json"
|
||||
return "text"
|
||||
|
||||
|
||||
def extract_code_block(text: str) -> str:
|
||||
"""从模型输出提取 python 代码块（剥除 markdown 围栏），无则返回原文本。"""
|
||||
t = text.strip()
|
||||
fence = chr(96) * 3 # 三个反引号
|
||||
marker = fence + "python"
|
||||
start = t.find(marker)
|
||||
if start == -1:
|
||||
return t
|
||||
body_start = start + len(marker)
|
||||
end = t.find(fence, body_start)
|
||||
if end == -1:
|
||||
return t[body_start:].strip()
|
||||
return t[body_start:end].strip()
|
||||
|
||||
|
||||
def run_code_sandbox(code: str, timeout_s: float = 10.0,
|
||||
runner: Optional[Callable[[List[str], Dict[str, str], float], Tuple[int, str, str]]] = None
|
||||
) -> Tuple[int, str, str]:
|
||||
"""在临时目录子进程运行 python 代码。返回 (returncode, stdout, stderr)。
|
||||
|
||||
隔离措施：临时工作目录、-I 隔离模式、timeout 超时强杀、捕获输出。
|
||||
runner 可注入（测试用假执行器，避免真跑任意代码）。
|
||||
"""
|
||||
if runner is not None:
|
||||
return runner([sys.executable, "-I"], {}, timeout_s)
|
||||
with tempfile.TemporaryDirectory(prefix="v2_sandbox_") as tmp:
|
||||
script = Path(tmp) / "main.py"
|
||||
script.write_text(code, encoding="utf-8")
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "-I", str(script)],
|
||||
capture_output=True, text=True, timeout=timeout_s,
|
||||
cwd=tmp,
|
||||
creationflags=subprocess.CREATE_NO_WINDOW,
|
||||
)
|
||||
return proc.returncode, proc.stdout or "", proc.stderr or ""
|
||||
except subprocess.TimeoutExpired:
|
||||
return -1, "", "timeout exceeded"
|
||||
|
||||
|
||||
class Verifier:
|
||||
"""按 D4 分层执行接地验证。"""
|
||||
|
||||
def __init__(self, code_timeout_s: float = 10.0,
|
||||
sandbox_runner: Optional[Callable[..., Tuple[int, str, str]]] = None):
|
||||
self.code_timeout_s = code_timeout_s
|
||||
self._sandbox_runner = sandbox_runner
|
||||
|
||||
def verify(self, domain: str, artifact_name: str, artifact_text: str,
|
||||
query: str, kb: Any = None) -> Tuple[bool, List[str]]:
|
||||
"""返回 (passed, details)。kb 为 KnowledgeBase（可 None，跳过 facts 层）。"""
|
||||
lang = detect_artifact_language(artifact_name)
|
||||
details: List[str] = []
|
||||
|
||||
# 1) 可执行验证（D4 最高优先级）：code + python 工件且含代码
|
||||
if lang == "python" and any(h in artifact_text for h in _CODE_HINTS):
|
||||
rc, out, err = run_code_sandbox(artifact_text, self.code_timeout_s,
|
||||
runner=self._sandbox_runner)
|
||||
if rc == 0:
|
||||
details.append("代码沙箱运行通过 (rc=0)")
|
||||
return True, details
|
||||
# 可执行验证失败 -> 直接拒绝（不落回结构检查，避免"假通过"）
|
||||
details.append(f"代码沙箱运行失败 rc={rc}: {(err or out)[:120]}")
|
||||
return False, details
|
||||
elif lang == "python":
|
||||
details.append("工件不含可执行代码（跳过沙箱，进入 facts/结构检查）")
|
||||
|
||||
# 2) facts 对照
|
||||
if kb is not None:
|
||||
fact_hits = self._check_facts(domain, artifact_text, kb)
|
||||
if fact_hits:
|
||||
details.append(f"facts 对照命中 {fact_hits}")
|
||||
return True, details
|
||||
|
||||
# 3) 结构检查
|
||||
text = artifact_text.strip()
|
||||
if len(text) < 20:
|
||||
details.append(f"工件过短（{len(text)} 字符）")
|
||||
return False, details
|
||||
if text.lower() in ("pass", "none", "todo", "待实现", "略"):
|
||||
details.append("工件为占位内容")
|
||||
return False, details
|
||||
details.append("结构检查通过（非空、长度达标）")
|
||||
return True, details
|
||||
|
||||
def _check_facts(self, domain: str, text: str, kb: Any) -> int:
|
||||
"""统计工件文本命中知识库事实的数量。"""
|
||||
facts = kb.facts(domain) or []
|
||||
if not facts:
|
||||
return 0
|
||||
hits = 0
|
||||
for f in facts:
|
||||
kw = f.get("keywords") or []
|
||||
if any(str(k) in text for k in kw):
|
||||
hits += 1
|
||||
return hits
|
||||
|
||||
|
||||
def build_verifier(cfg: Dict[str, Any]) -> Verifier:
|
||||
"""cfg 为 config.worker 段。"""
|
||||
return Verifier(code_timeout_s=float(cfg.get("code_timeout_s", 10)))
|
||||
@@ -0,0 +1,180 @@
|
||||
"""WorkerLoop —— 小模型（本地 llama.cpp）的"实现/自验证"循环（端云协同的执行者）。
|
||||
|
||||
流程（对齐《实现方案_v2》5.1 T5 / 4.4）：
|
||||
读 brief+当前步 -> 模型生成工件 -> 接地验证（D4 分层）
|
||||
-> 通过：写 progress(done)
|
||||
-> 失败：自修 <= max_fix_attempts 次（把验证错误回喂重新生成）
|
||||
-> 仍失败：写 issue（增量、带锚点）
|
||||
|
||||
- generate 为可注入的文本生成器（真实为 llama-server 端点；测试用假实现）。
|
||||
- 工件落盘：runs/<request_id>/artifacts/<step>.py（由 pipeline 负责写盘，本模块只产出文本）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Awaitable, Callable, Dict, List, Optional
|
||||
|
||||
from .verifier import Verifier, detect_artifact_language, extract_code_block
|
||||
from .workspace import Workspace, build_anchor
|
||||
|
||||
# 按领域推断默认工件扩展名
|
||||
_DOMAIN_EXT = {
|
||||
"code": ".py",
|
||||
"math": ".md",
|
||||
"legal": ".md",
|
||||
"medical": ".md",
|
||||
"finance": ".md",
|
||||
"life": ".md",
|
||||
"education": ".md",
|
||||
"general": ".md",
|
||||
}
|
||||
|
||||
|
||||
def artifact_name_for(step_id: str, domain: str) -> str:
|
||||
"""为 step 生成工件文件名。"""
|
||||
ext = _DOMAIN_EXT.get(domain, ".md")
|
||||
return f"{step_id}{ext}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class StepOutcome:
|
||||
"""单步执行结果。"""
|
||||
step_id: str
|
||||
status: str # done | issue
|
||||
summary: str = ""
|
||||
model_used: str = "local"
|
||||
attempts: int = 0
|
||||
issue_id: Optional[str] = None
|
||||
artifact_name: Optional[str] = None
|
||||
artifact_text: str = ""
|
||||
details: List[str] = field(default_factory=list)
|
||||
|
||||
|
||||
class WorkerLoop:
|
||||
"""小模型 Worker：实现 -> 验证 -> 自修 -> issue。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
generate: Callable[[str], Awaitable[str]],
|
||||
verifier: Optional[Verifier] = None,
|
||||
kb: Any = None,
|
||||
max_fix_attempts: int = 2,
|
||||
model_used: str = "local-llama",
|
||||
):
|
||||
self.generate = generate
|
||||
self.verifier = verifier or Verifier()
|
||||
self.kb = kb
|
||||
self.max_fix_attempts = max_fix_attempts
|
||||
self.model_used = model_used
|
||||
|
||||
def _domain_from(self, ws: Workspace) -> str:
|
||||
tags = (ws.get("brief") or {}).get("tags") or []
|
||||
for t in tags:
|
||||
if t != "safety":
|
||||
return t
|
||||
return "general"
|
||||
|
||||
async def run_step(self, ws: Workspace, step_id: str,
|
||||
existing_artifact: str = "") -> StepOutcome:
|
||||
"""执行单个 step。existing_artifact 为该步当前已有工件全文（若有）。"""
|
||||
domain = self._domain_from(ws)
|
||||
brief = ws.get("brief") or {}
|
||||
plan = brief.get("plan") or []
|
||||
step_def = next((p for p in plan if p.get("id") == step_id), {})
|
||||
done_criteria = step_def.get("done_criteria", "")
|
||||
|
||||
artifact_name = artifact_name_for(step_id, domain)
|
||||
current = existing_artifact
|
||||
details: List[str] = []
|
||||
|
||||
for attempt in range(1, self.max_fix_attempts + 1):
|
||||
prompt = self._build_prompt(ws, step_id, current, attempt, done_criteria)
|
||||
out = await self.generate(prompt)
|
||||
if domain == "code":
|
||||
candidate = extract_code_block(out)
|
||||
else:
|
||||
candidate = out.strip()
|
||||
details.append(f"attempt{attempt}: 生成 {len(candidate)} 字符")
|
||||
|
||||
passed, v_details = self.verifier.verify(
|
||||
domain, artifact_name, candidate, ws["query"], kb=self.kb)
|
||||
details.extend(f" - {d}" for d in v_details)
|
||||
if passed:
|
||||
# 写回交流文本：progress(done) + 摘要
|
||||
ws.add_progress(step_id, "done", f"步骤完成（{attempt} 次尝试）",
|
||||
artifact=build_anchor(artifact_name, 1))
|
||||
return StepOutcome(
|
||||
step_id=step_id, status="done",
|
||||
summary=f"步骤完成（{attempt} 次尝试）",
|
||||
model_used=self.model_used, attempts=attempt,
|
||||
artifact_name=artifact_name, artifact_text=candidate,
|
||||
details=details,
|
||||
)
|
||||
# 未通过：带错误反馈重新生成（自修）
|
||||
current = candidate
|
||||
feedback = "；".join(v_details)
|
||||
details.append(f"attempt{attempt} 未通过，进入自修")
|
||||
|
||||
# 全部尝试失败 -> 写 issue
|
||||
anchor = build_anchor(artifact_name, 1, 30)
|
||||
iid = ws.add_issue(
|
||||
step=step_id,
|
||||
anchor=anchor,
|
||||
observed=f"验证未通过：{'；'.join(d for d in details if d.startswith(' - ')) or '未知'}",
|
||||
expected=done_criteria or "满足该步 done_criteria",
|
||||
tried=f"已自修 {self.max_fix_attempts} 次",
|
||||
ask="请裁决该步的实现方向或提供兜底实现",
|
||||
)
|
||||
return StepOutcome(
|
||||
step_id=step_id, status="issue", summary="未能通过验证，已上报 issue",
|
||||
model_used=self.model_used, attempts=self.max_fix_attempts,
|
||||
issue_id=iid, artifact_name=artifact_name, artifact_text=current,
|
||||
details=details,
|
||||
)
|
||||
|
||||
def _build_prompt(self, ws: Workspace, step_id: str, current: str,
|
||||
attempt: int, done_criteria: str) -> str:
|
||||
base = ws.render_for_worker(step_id, artifact_text=current or None)
|
||||
if attempt > 1:
|
||||
base += (
|
||||
"\n\n[注意] 上次生成的工件未通过接地验证。请修正以下问题后重新输出"
|
||||
f"完整工件。本次为第 {attempt} 次尝试。"
|
||||
)
|
||||
return base
|
||||
|
||||
|
||||
def build_worker(cfg: Dict[str, Any], kb: Any = None,
|
||||
generate: Optional[Callable[[str], Awaitable[str]]] = None) -> WorkerLoop:
|
||||
"""cfg 为 config.worker 段。generate 缺省时用 llama-server 端点客户端（惰性）。"""
|
||||
if generate is None:
|
||||
generate = _make_llama_generate(cfg)
|
||||
verifier = Verifier(code_timeout_s=float(cfg.get("code_timeout_s", 10)))
|
||||
return WorkerLoop(
|
||||
generate=generate,
|
||||
verifier=verifier,
|
||||
kb=kb,
|
||||
max_fix_attempts=int(cfg.get("max_fix_attempts", 2)),
|
||||
model_used=cfg.get("backend", "llama_server"),
|
||||
)
|
||||
|
||||
|
||||
def _make_llama_generate(cfg: Dict[str, Any]) -> Callable[[str], Awaitable[str]]:
|
||||
"""返回调用本地 llama-server（OpenAI 兼容 /v1/chat/completions）的生成器。"""
|
||||
base_url = cfg.get("base_url", f"http://127.0.0.1:{cfg.get('port', 8901)}/v1")
|
||||
model = cfg.get("model", "local")
|
||||
temperature = float(cfg.get("temperature", 0.3))
|
||||
timeout_s = float(cfg.get("per_step_timeout_s", 300))
|
||||
|
||||
async def _gen(prompt: str) -> str:
|
||||
import httpx
|
||||
async with httpx.AsyncClient(timeout=timeout_s) as client:
|
||||
resp = await client.post(
|
||||
f"{base_url}/chat/completions",
|
||||
json={"model": model, "messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": temperature, "max_tokens": 4096},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()["choices"][0]["message"]["content"]
|
||||
|
||||
return _gen
|
||||
Reference in New Issue
Block a user