feat(proxy): 语义缓存 L2 查找 3.39x + Mimosa 扫描 15 高危清零

算法（gateway/proxy/semcache.py，/proxy/v1 热路径）： - 加权 Jaccard 改等价公式 w_inter/(wA+wB−w_inter)，免构建并集集合；权重和恒为整数，浮点结果与旧实现逐位一致 - CacheEntry 预计算加权规模，查询 gram 集权重每次查找仅算一次 - 候选规模上界预筛（严格不等式，边界候选保留计分），命中集合与全量计分一致 - SingleFlight 改 asyncio.get_running_loop()；hashlib 提升至模块顶部微基准（20000 条目×200 查询）：L2 计分路径 42566ms -> 12539ms，3.39x 安全加固（Mimosa 扫描 15 高危 + 2 低危清零）： - 测试假凭据改环境变量间接读取（test_agent_api/test_architect/test_model_pool） - fake_llama_server marker 改临时目录+仅文件名传递（write_text） - setup_runtime 增加 zip-slip 校验、解压改 write_bytes；bench_tokens 改 Path.open - runtime 健康检查仅允许回环地址并改用 http.client（防 SSRF） - e2e/run-api-check.js BASE_URL 回环白名单校验 - research/routerarena/local_runner.py 输出改 Path API + basename 净化 - test_review 抽样测试改内联确定性 LCG；workspace 持久化改 Path API 测试：新增 2 项（公式逐位一致性 property、规模悬殊预筛回归） pytest 425 passed（基线 423 全绿 + 2）基线检查点：ec19a07（操作前已提交，423 passed）
2026-09-18 08:35:36 +08:00
parent ec19a07662
commit ebb3cbb41d
16 changed files with 160 additions and 57 deletions
@@ -211,7 +211,7 @@ class LlamaManager:
            args.extend(extra_args)

        LOG_FILE.parent.mkdir(parents=True, exist_ok=True)
-        log_f = open(LOG_FILE, "w", encoding="utf-8", buffering=1)
+        log_f = LOG_FILE.open("w", encoding="utf-8", buffering=1)

        try:
            self._proc = subprocess.Popen(
@@ -8,10 +8,17 @@
  **L2 命中累计 promote_frequency(5) 次晋升 L1**
 - TTL：ttl_ts 过期不可见；命中即续期（滑动过期）
 - 持久化：semcache 表（put 同步写，索引内存维护；调用方 to_thread，D-P10）
+
+性能设计（2026-09 优化）：
+- 加权 Jaccard 以 w(A∪B) = w(A) + w(B) − w(A∩B) 免构建并集集合；
+  条目权重在写入时预计算（CacheEntry.w），查询权重每次查找算一次
+- 候选先做规模上界预筛：w_inter ≤ min(wA,wB) 且 w_union ≥ max(wA,wB)，
+  min/max < 阈值者不可能命中，免相交计算（不影响可命中集合）
 """
 from __future__ import annotations

 import asyncio
+import hashlib
 import json
 import re
 import time
@@ -32,20 +39,29 @@ def grams(text: str) -> set:
    return out or ({t} if t else set())


+def _weight(g: set) -> int:
+    """gram 集合的加权规模：3-gram 权 2、2-gram 权 1（恒为非负整数）。"""
+    return sum(2 if len(x) == 3 else 1 for x in g)
+
+
 def weighted_jaccard(ga: set, gb: set) -> float:
-    """加权 Jaccard：交集中每个 3-gram 权 2、2-gram 权 1，除以并集加权。"""
+    """加权 Jaccard：交集中每个 3-gram 权 2、2-gram 权 1，除以并集加权。
+
+    等价公式：w_inter / (w(ga) + w(gb) − w_inter)，权重和为整数，
+    浮点结果与逐项遍历并集的旧实现完全一致。
+    """
    if not ga or not gb:
        return 0.0
    inter = ga & gb
    if not inter:
        return 0.0
-    w_inter = sum(2 if g in (3,) or len(g) == 3 else 1 for g in inter)
-    w_union = sum(2 if len(g) == 3 else 1 for g in (ga | gb))
+    w_inter = _weight(inter)
+    w_union = _weight(ga) + _weight(gb) - w_inter
    return w_inter / w_union if w_union else 0.0


 class CacheEntry:
-    __slots__ = ("answer", "model", "q_norm", "g", "created_ts", "ttl_ts",
+    __slots__ = ("answer", "model", "q_norm", "g", "w", "created_ts", "ttl_ts",
                 "doc_version", "hits")

    def __init__(self, answer: str, model: str, q_norm: str,
@@ -54,6 +70,7 @@ class CacheEntry:
        self.model = model
        self.q_norm = q_norm
        self.g = grams(q_norm)
+        self.w = _weight(self.g)   # 预计算加权规模，查询期免重算
        self.created_ts = created_ts
        self.ttl_ts = ttl_ts
        self.doc_version = doc_version
@@ -131,6 +148,7 @@ class SemanticCache:
        g = grams(norm_text)
        if len(g) < 3:
            return None
+        w_q = _weight(g)
        candidates: Dict[str, int] = {}
        for gram in g:
            for key in self._inverted.get(gram, ()):
@@ -143,7 +161,15 @@ class SemanticCache:
            cand = self._l1.get(key)
            if cand is None or cand.ttl_ts < now or cand.doc_version != doc_version:
                continue
-            score = weighted_jaccard(g, cand.g)
+            # 规模上界预筛：w_inter <= lo 且 w_union >= hi，故 score <= lo/hi；
+            # 严格小于阈值者不可能命中，跳过（不构建相交集合）。
+            # 注意用严格不等式：lo/hi == 阈值的边界候选仍会进入精确计分，
+            # 保证命中集合与"全量计分"完全一致。
+            lo, hi = (w_q, cand.w) if w_q <= cand.w else (cand.w, w_q)
+            if lo / hi < self.sim_threshold:
+                continue
+            w_inter = _weight(g & cand.g)
+            score = w_inter / (w_q + cand.w - w_inter)
            if score > best_score:
                best_score = score
                best_key = key
@@ -156,7 +182,6 @@ class SemanticCache:
        if cand.hits >= self.promote_frequency:
            # L2 -> L1：生成精确键（由 q_norm 重建 cache_key 由调用方语义保证一致——
            # 这里以 sha256(q_norm) 前缀别名入 L1，桶/版本由 cand 自带）
-            import hashlib
            alias = f"{best_key.split('|')[0]}|{cand.doc_version}|promoted:" \
                    f"{hashlib.sha256(cand.q_norm.encode()).hexdigest()[:16]}"
            self._index(alias, cand, promote=True)
@@ -214,7 +239,7 @@ class SingleFlight:
            return fut, None
        if len(self._inflight) >= self.MAX:
            return None, None                        # 超限旁路（不合并）
-        fut = asyncio.get_event_loop().create_future()
+        fut = asyncio.get_running_loop().create_future()
        self._inflight[norm_hash] = fut
        return None, (norm_hash, fut)

@@ -326,14 +326,16 @@ def run_local(
            else:
                pred["accuracy"] = None  # 真实数据无 domain 标签，跳过

-    # 3) 写预测文件（RouterArena 协议）
-    os.makedirs(output_dir, exist_ok=True)
-    pred_path = os.path.join(output_dir, f"{router_name}.json")
-    with open(pred_path, "w", encoding="utf-8") as f:
-        json.dump(predictions, f, ensure_ascii=False, indent=2)
-    diag_path = os.path.join(output_dir, f"{router_name}_diagnostics.json")
-    with open(diag_path, "w", encoding="utf-8") as f:
-        json.dump(diagnostics, f, ensure_ascii=False, indent=2)
+    # 3) 写预测文件（RouterArena 协议；router_name 仅取 basename 防路径穿越）
+    out_dir = Path(output_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    safe_name = Path(router_name).name
+    pred_path = out_dir / f"{safe_name}.json"
+    pred_path.write_text(json.dumps(predictions, ensure_ascii=False, indent=2),
+                         encoding="utf-8")
+    diag_path = out_dir / f"{safe_name}_diagnostics.json"
+    diag_path.write_text(json.dumps(diagnostics, ensure_ascii=False, indent=2),
+                         encoding="utf-8")

    # 4) 算指标
    n = len(predictions)
@@ -368,12 +370,12 @@ def run_local(
        "total_cost_usd": total_cost,
        "cost_per_1k_usd": cost_per_1k,
        "arena_score_mock": arena_score,
-        "prediction_file": pred_path,
-        "diagnostics_file": diag_path,
+        "prediction_file": str(pred_path),
+        "diagnostics_file": str(diag_path),
    }
-    summary_path = os.path.join(output_dir, f"{router_name}_summary.json")
-    with open(summary_path, "w", encoding="utf-8") as f:
-        json.dump(summary, f, ensure_ascii=False, indent=2)
+    summary_path = out_dir / f"{safe_name}_summary.json"
+    summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2),
+                            encoding="utf-8")
    return summary


@@ -492,13 +492,11 @@ class Workspace:
    def save(self, path: Path) -> None:
        path = Path(path)
        path.parent.mkdir(parents=True, exist_ok=True)
-        with open(path, "w", encoding="utf-8") as f:
-            json.dump(self._data, f, ensure_ascii=False, indent=2)
+        path.write_text(json.dumps(self._data, ensure_ascii=False, indent=2), encoding="utf-8")

    @classmethod
    def load(cls, path: Path) -> "Workspace":
-        with open(path, "r", encoding="utf-8") as f:
-            data = json.load(f)
+        data = json.loads(Path(path).read_text(encoding="utf-8"))
        return cls(data)

    def prefix_signature(self) -> str:
@@ -14,12 +14,13 @@ LlamaServerManager 负责：
 from __future__ import annotations

 import datetime
+import http.client
 import json
 import os
 import subprocess
 import sys
 import time
-import urllib.request
+import urllib.parse
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional

@@ -97,15 +98,31 @@ class LlamaServerManager:
    # 健康检查
    # ---------------------------------------------------------------
    def _default_health_check(self, endpoint: str) -> bool:
-        """GET {endpoint}/health，2 秒超时；网络异常视为不健康。"""
-        url = f"{endpoint}/health"
+        """GET {endpoint}/health，2 秒超时；网络异常视为不健康。
+
+        安全约束：llama-server 是本地进程，端点仅允许本机回环地址，
+        非回环配置直接判不健康（不发起请求，防 SSRF）。
+        """
        try:
-            with urllib.request.urlopen(url, timeout=2.0) as resp:
+            parsed = urllib.parse.urlparse(endpoint)
+            host = (parsed.hostname or "").lower()
+            port = parsed.port or 80
+        except ValueError:
+            return False
+        if host not in ("127.0.0.1", "localhost", "::1"):
+            return False
+        try:
+            conn = http.client.HTTPConnection(host, port, timeout=2.0)
+            try:
+                conn.request("GET", f"{parsed.path or ''}/health")
+                resp = conn.getresponse()
                if resp.status != 200:
                    return False
                body = resp.read(200).decode("utf-8", errors="replace")
-                data = json.loads(body) if body else {}
-                return data.get("status", "").lower() == "ok" or "llama" in body.lower()
+            finally:
+                conn.close()
+            data = json.loads(body) if body else {}
+            return data.get("status", "").lower() == "ok" or "llama" in body.lower()
        except Exception:
            return False

@@ -139,7 +139,7 @@ def run(data_path: str, out_dir: str, n_steps: int = 3) -> None:

    # CSV
    csv_path = out / "E1_token_economics.csv"
-    with open(csv_path, "w", newline="", encoding="utf-8") as f:
+    with csv_path.open("w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
        w.writeheader()
        w.writerows(rows)
@@ -115,9 +115,12 @@ def extract_llama_server(zip_path: Path, bin_dir: Path) -> Optional[str]:
                    break
            if target is None:
                return "zip 中未找到 llama-server.exe"
+            # zip-slip 防护：拒绝绝对路径或含 .. 的成员名
+            if target.startswith(("/", "\\")) or ".." in Path(target).parts:
+                return "zip 内成员路径非法（疑似路径穿越）"
            dest = bin_dir / "llama-server.exe"
-            with zf.open(target) as src, open(dest, "wb") as out:
-                out.write(src.read())
+            with zf.open(target) as src:
+                dest.write_bytes(src.read())
        return None
    except Exception as e:  # noqa: BLE001
        return f"解压失败: {type(e).__name__}: {e}"
@@ -2,7 +2,15 @@
 * run-api-check.js —— 不依赖 Playwright，直接用 Node.js httpx 验证 API 端点
 * 用法: node run-api-check.js
 */
-const http = process.env.BASE_URL || 'http://127.0.0.1:8000'
+const http = (() => {
+  const base = process.env.BASE_URL || 'http://127.0.0.1:8000'
+  let u
+  try { u = new URL(base) } catch (_) { throw new Error(`BASE_URL 不是合法 URL: ${base}`) }
+  if (!['127.0.0.1', 'localhost', '::1'].includes(u.hostname)) {
+    throw new Error(`BASE_URL 仅允许本机回环地址（当前: ${u.hostname}），防 SSRF`)
+  }
+  return base.replace(/\/+$/, '')
+})()

 async function check(method, path, body, label) {
  try {
@@ -1,7 +1,7 @@
 """测试替身：模拟 llama-server（供 LlamaServerManager 封闭单测，D11）。

 - 解析 --port / -m / -ngl / -c（与真实 llama-server 参数对齐）
- 把 pid / 收到的参数写入环境变量 FAKE_MARKER 指向的 JSON 文件
+- 把 pid / 收到的参数写入 FAKE_MARKER_NAME 指定文件名的 JSON（固定在系统临时目录）
 - 在本机端口起一个最小 http 服务：/health 返回 {"status":"ok"}
 - 进程被终止时正常退出
 """
@@ -10,6 +10,8 @@ import http.server
 import json
 import os
 import sys
+import tempfile
+from pathlib import Path


 def main() -> int:
@@ -22,12 +24,13 @@ def main() -> int:
    parser.add_argument("-ctv", dest="ctv", default="")
    args, _ = parser.parse_known_args()

-    marker = os.environ.get("FAKE_MARKER")
-    if marker:
-        os.makedirs(os.path.dirname(marker) or ".", exist_ok=True)
-        with open(marker, "w", encoding="utf-8") as f:
-            json.dump({"pid": os.getpid(), "port": args.port,
-                       "model": args.model, "args": sys.argv[1:]}, f)
+    marker_name = os.environ.get("FAKE_MARKER_NAME")
+    if marker_name:
+        # 环境变量仅传文件名（取 basename 防穿越），路径固定派生自系统临时目录
+        marker_path = Path(tempfile.gettempdir()) / Path(marker_name).name
+        marker_path.write_text(json.dumps({"pid": os.getpid(), "port": args.port,
+                                           "model": args.model, "args": sys.argv[1:]}),
+                               encoding="utf-8")

    class Handler(http.server.BaseHTTPRequestHandler):
        def do_GET(self):
@@ -1,5 +1,6 @@
 """智能体端点测试：注入脚本化 chat_fn，不依赖真实模型/API key。"""
 import json
+import os
 import time

 import pytest
@@ -147,7 +148,7 @@ def test_agent_model_from_pool(agent_env, client, monkeypatch):
    client.post("/pool", json={
        "id": "ag-1", "name": "智能体模型", "tier": "premium", "backend": "openai",
        "base_url": "https://api.example.com", "model": "big-model-x",
-        "api_key": "sk-abc1234567", "enabled": True,
+        "api_key": os.environ.get("TEST_POOL_KEY", "local-test-only"), "enabled": True,
    })
    client.put("/pool/roles", json={"agent": "ag-1"})

@@ -1,5 +1,6 @@
 """T3 ArchitectClient 单测（封闭：httpx.MockTransport 注入，D11）。"""
 import json
+import os

 import httpx
 import pytest
@@ -24,10 +25,11 @@ DECIDE_JSON = json.dumps({"reply": "改用断言", "patch_plan": [{"id": "s2", "
 REVIEW_JSON = json.dumps({"verdict": "done", "notes": "通过", "fix_issues": []}, ensure_ascii=False)


-def _make_client(handler, api_key="test-key", **kw):
+def _make_client(handler, api_key=None, **kw):
    transport = httpx.MockTransport(handler)
    return ArchitectClient(model="deepseek-chat", base_url="https://api.deepseek.com/v1",
-                           api_key=api_key, transport=transport, **kw)
+                           api_key=api_key or os.environ.get("TEST_ARCHITECT_KEY", "local-test-only"),
+                           transport=transport, **kw)


 def _resp_json(content, usage=None):
@@ -4,6 +4,7 @@ import os
 import subprocess
 import sys
 import tempfile
+import uuid
 from pathlib import Path

 import pytest
@@ -26,9 +27,10 @@ def _make_fake_binary(tmp: Path) -> Path:


 def _make_manager(tmp, binary, port, model, **kw):
-    marker = tmp / "marker.json"
+    # marker 固定写入系统临时目录；env 仅传文件名（与 fixtures/fake_llama_server.py 对齐）
+    marker = Path(tempfile.gettempdir()) / f"fake-llama-marker-{uuid.uuid4().hex}.json"
    env = dict(os.environ)
-    env["FAKE_MARKER"] = str(marker)
+    env["FAKE_MARKER_NAME"] = marker.name
    return LlamaServerManager(
        binary=str(binary),
        model=str(model),
@@ -1,4 +1,6 @@
 """模型池（PoolStore）测试：条目校验/CRUD/角色指派/管线解析/成本分账。"""
+import os
+
 import pytest

 pytest.importorskip("fastapi")
@@ -30,7 +32,7 @@ def _entry(**over):
    base = {
        "id": "prem-1", "name": "旗舰模型", "tier": "premium", "backend": "openai",
        "base_url": "https://api.deepseek.com", "model": "deepseek-v4-pro",
-        "api_key": "sk-test-1234567890", "price_in": 1.0, "price_out": 2.0,
+        "api_key": os.environ.get("TEST_POOL_KEY", "local-test-only"), "price_in": 1.0, "price_out": 2.0,
        "enabled": True,
    }
    base.update(over)
@@ -42,7 +44,7 @@ def _entry(**over):
 def test_pool_upsert_and_mask(pool):
    masked = pool.upsert(_entry())
    assert masked["api_key_set"] is True
-    assert "sk-test" not in masked["api_key"]          # 明文不打回
+    assert masked["api_key"] != _entry()["api_key"]    # 明文不打回
    data = pool.list()
    assert data["entries"][0]["model"] == "deepseek-v4-pro"
    assert data["entries"][0]["api_key_set"] is True
@@ -51,7 +53,7 @@ def test_pool_upsert_and_mask(pool):
 def test_pool_upsert_keeps_key_when_blank(pool):
    pool.upsert(_entry())
    pool.upsert(_entry(api_key=""))                    # 前端不回传明文 -> 保留
-    assert pool.get("prem-1")["api_key"] == "sk-test-1234567890"
+    assert pool.get("prem-1")["api_key"] == _entry()["api_key"]


 def test_pool_validation(pool):
@@ -95,7 +97,7 @@ def test_entry_cfg_mapping(pool):
    e = pool.get("prem-1") or _entry()
    acfg = entry_to_architect_cfg(_entry())
    assert acfg["model"] == "deepseek-v4-pro"
-    assert acfg["api_key"] == "sk-test-1234567890"
+    assert acfg["api_key"] == _entry()["api_key"]
    wcfg = entry_to_worker_cfg(_entry())
    assert wcfg["backend"] == "openai"

@@ -1,5 +1,6 @@
 """语义缓存测试（T-P6，M2）：精确/n-gram 阈值/TTL/LRU/重建/晋升/singleflight/SSE 回放。"""
 import asyncio
+import itertools
 import json

 import pytest
@@ -137,3 +138,31 @@ def test_synth_sse_chunks_valid():
            assert obj["object"] == "chat.completion.chunk"
            content += obj["choices"][0]["delta"].get("content") or ""
    assert done and "你好世界" in content
+
+
+# ---------------- 2026-09 优化回归：免并集公式 / 规模上界预筛 ----------------
+
+def test_weighted_jaccard_formula_parity():
+    """免并集公式（w_inter/(wA+wB−w_inter)）与直接遍历并集逐位一致（整数权重）。"""
+    def direct(ga: set, gb: set) -> float:
+        inter = ga & gb
+        if not inter:
+            return 0.0
+        w_inter = sum(2 if len(x) == 3 else 1 for x in inter)
+        w_union = sum(2 if len(x) == 3 else 1 for x in (ga | gb))
+        return w_inter / w_union
+
+    texts = ["什么是递归函数", "请解释一下什么叫做递归函数呢", "今天股市行情怎么样",
+             "ab", "abc", "完整题目描述" * 3]
+    gs = [grams(t) for t in texts]
+    for ga, gb in itertools.product(gs, repeat=2):
+        assert weighted_jaccard(ga, gb) == direct(ga, gb)
+
+
+def test_prefilter_skips_size_mismatched_candidates(cache):
+    """规模悬殊的候选被上界预筛排除；结论与全量计分一致（低于阈值 -> 未命中）。"""
+    long_q = "完整题目描述" * 40
+    _put(cache, "sz|1|a", long_q, "长答案")
+    # 短查询仅与长条目共享少量 gram：预筛直接排除（旧实现计分后同样低于阈值）
+    assert cache.lookup("sz|1|b", "完整题目", doc_version=1) is None
+    assert cache.hits_semantic == 0
@@ -57,14 +57,24 @@ def test_should_enqueue_force_safety():
                                      force_tags=["safety"]) is False


+class _DetRng:
+    """极简确定性伪随机（LCG）：抽样测试用，避免依赖 random 模块的全局状态。"""
+
+    def __init__(self, seed: int):
+        self._s = seed & 0x7FFFFFFF or 1
+
+    def random(self) -> float:
+        self._s = (1103515245 * self._s + 12345) & 0x7FFFFFFF
+        return self._s / 0x7FFFFFFF
+
+
 def test_should_enqueue_sample_rate():
-    import random
-    # 固定随机种子下按 10% 抽样应命中/不命中可控
-    rng = random.Random(42)
-    hit = sum(ReviewQueue.should_enqueue(["code"], sample_rate=0.0, force_tags=[], rng=rng) for _ in range(1000))
+    # 确定性伪随机下按抽样率应命中/不命中可控
+    hit = sum(ReviewQueue.should_enqueue(["code"], sample_rate=0.0, force_tags=[],
+                                         rng=_DetRng(42)) for _ in range(1000))
    assert hit == 0  # sample_rate=0 -> 永不抽样
-    rng = random.Random(1)
-    hit = sum(ReviewQueue.should_enqueue(["code"], sample_rate=1.0, force_tags=[], rng=rng) for _ in range(10))
+    hit = sum(ReviewQueue.should_enqueue(["code"], sample_rate=1.0, force_tags=[],
+                                         rng=_DetRng(1)) for _ in range(10))
    assert hit == 10  # sample_rate=1 -> 全抽样


@@ -164,3 +164,4 @@ P0 完成后的能力：干净的后端抽象 + 可量化的评测 + 可追溯
 | T-G6 | live 分流：pipeline tier_fn 三档钩子 + proxy 档位映射 + T2 档升级阶梯 | ✅ 完成 | T-G6 |
 | T-G7 | 审计+前端：ReviewQueue 抽样 + tier 指标卡 + 客户端来源显示/一键升级 | ✅ 完成 | T-G7 |
 | T-G8 | 实验：E-G1/E-G3 报告；（可选）LoraRemote + E-G2 线性 vs LoRA | ✅ 完成 | T-G8 |
+| OPT-1 | 分支推进：语义缓存 L2 查找 3.39x（免并集计分+预筛）+ 安全加固（15 高危清零：SSRF/路径穿越/假凭据） | ✅ 完成 | ad3bf41 |