infosave2007
7479cc51c2
Two "hard" probes (full suite only): a compiler-backend design and a security
protocol design, each dense enough for the router's complexity model to rate
them high. They are judged on the ROUTER's decision (judge: "route") — pass
means tier=high was reached and the request went to the big model; the answer
itself is not scored (open-ended essays aren't keyword-judgeable, and
reasoning models happily burn 3000+ thinking tokens on them — measured). With
max_tokens kept tiny the probes cost cents even on the priciest model, while
the cost column tells the whole story: $0.0148 for the high-tier request next
to $0.00005 for the easy ones.
Each run picks a random variant of the prompt: the router deliberately scores
familiar text as less novel, so a fixed wording would decay to medium over
repeated runs. In pinned-models mode route-judged tests are skipped.
Also: creative-writing budget 96→512 (deepseek was spending the whole cap
thinking and returning an empty poem), and when a model does burn its entire
budget inside <think> the table now says so ("budget spent on reasoning")
instead of the misleading "empty answer".
Server-side (cortiq-router prod, both replicas rolling-restarted): the high
band started at 0.66 while the complexity model tops out ~0.65 on the hardest
single prompts — high was unreachable. Band boundary moved to 0.50 based on
measured distribution (easy 0.25-0.36, medium 0.37-0.49, hard 0.50-0.65);
config backed up as config.prod.toml.bak-tiers.
475 lines
26 KiB
JavaScript
Vendored
475 lines
26 KiB
JavaScript
Vendored
// Benchmark — two modes over the live pipeline:
|
||
// 🧭 Smart routing (default): the product's core promise under test — every
|
||
// probe goes through cortiq-auto and the router's task/tier decision and
|
||
// chosen model are judged alongside the answer itself.
|
||
// 🎯 Individual models: the same suite pinned to selected models — compare
|
||
// quality and speed head-to-head.
|
||
// A redesign of the old standalone benchmark.html: same scoring ideas
|
||
// (keyword checks with | synonyms, think-tag stripping, anti-loop detection,
|
||
// pass/partial/fail), now embedded, localized and CDN-free.
|
||
import { h, mount, toast, ms, money, moneyEl } from "../ui.js";
|
||
import { t } from "../i18n.js";
|
||
import { api, testStream } from "../api.js";
|
||
|
||
// ── built-in suite ──────────────────────────────────────────────────────────
|
||
// Checks are language-neutral where possible (digits, identifiers, emails);
|
||
// `check` entries may hold |-separated synonyms — one hit per entry counts.
|
||
// `quick: true` marks the fast subset.
|
||
// `label`: router task labels accepted as a correct classification (null = not
|
||
// judged — the prompt has no natural label in the taxonomy).
|
||
const SUITE = [
|
||
{ id: "math-mult", cat: "math", label: ["math"], quick: true, max: 96, prompt: "What is 17 * 23? Answer with the number only.", check: ["391"] },
|
||
{ id: "math-eq", cat: "math", label: ["math"], max: 128, prompt: "Solve 2x + 6 = 14. Answer with the value of x only.", check: ["4"] },
|
||
{ id: "logic-syll", cat: "logic", label: ["qa", "math"], max: 96, prompt: "If all bloops are razzies and all razzies are lazzies, are all bloops lazzies? Answer yes or no.", check: ["yes|да|oui|ja|sí|si|evet|是"] },
|
||
{ id: "count-letters", cat: "logic", label: ["qa", "math"], quick: true, max: 64, prompt: "How many letters are in the word 'cat'? Answer with a digit.", check: ["3"] },
|
||
{ id: "apples", cat: "logic", label: ["math", "qa"], quick: true, max: 96, prompt: "John has 3 apples, buys 2 more, then eats 1. How many apples does he have? Answer with a digit.", check: ["4"] },
|
||
{ id: "code-fact", cat: "code", label: ["code"], max: 256, prompt: "Write a Python function factorial(n) that returns n!.", check: ["def", "factorial", "return"] },
|
||
{ id: "qa-capital", cat: "qa", label: ["qa"], quick: true, max: 48, prompt: "What is the capital of France? One word.", check: ["paris|париж"] },
|
||
{ id: "primes", cat: "qa", label: ["math", "qa"], max: 96, prompt: "List the first five prime numbers, comma-separated.", check: ["2", "3", "5", "7", "11"] },
|
||
{ id: "instr-ok", cat: "instruction", label: null, quick: true, max: 32, prompt: "Reply with exactly the word OK and nothing else.", check: ["ok"] },
|
||
{ id: "extract-email", cat: "extraction", label: ["extraction"], quick: true, max: 64, prompt: "Extract the email address from: 'Contact us at info@example.com today.' Reply with the address only.", check: ["info@example.com"] },
|
||
{ id: "translate", cat: "translation", label: ["translation"], max: 64, prompt: "Translate to English: «Доброе утро». Reply with the translation only.", check: ["good morning"] },
|
||
{ id: "format-json", cat: "instruction", label: null, max: 96, prompt: 'Return a JSON object with a single key "n" whose value is 7. JSON only.', check: ['"n"', "7"] },
|
||
{ id: "summarize", cat: "summarization", label: ["summarization"], max: 96, prompt: "Summarize in one short sentence: 'The cat sat on the warm mat by the window. After a while it curled up and fell asleep in the sun.'", check: ["cat|кот|кошк"] },
|
||
{ id: "creative", cat: "creative-writing", label: ["creative-writing"], max: 512, prompt: "Write a two-line poem about the sea.", check: ["sea|море|ocean"] },
|
||
{ id: "chitchat", cat: "chitchat", label: ["chitchat"], quick: true, max: 48, prompt: "Hi! How are you today?", check: [] },
|
||
// hard probes: dense enough that the router's complexity model rates them
|
||
// high tier — this is where tier-based routing actually earns its keep.
|
||
// Each run picks a random variant: the router treats familiar prompts as
|
||
// less novel (by design), so a fixed wording would decay to medium/low.
|
||
{ id: "hard-compiler", cat: "hard", label: ["code", "qa"], judge: "route", max: 8, check: [], variants: [
|
||
"Design an SSA-based optimizing compiler backend for an exposed-pipeline VLIW media processor: formulate instruction scheduling as an integer linear program with resource and hazard constraints, apply modulo scheduling to software-pipelined inner loops using rotating register files, do spill-aware register allocation via graph coloring with live-range splitting, add a translation-validated peephole pass, and explain how to prove the scheduler can never violate a structural hazard.",
|
||
"Build the code-generation backend of a JIT for GPU warp-specialized kernels: cost-model-driven instruction selection across divergent branches, software pipelining of memory latency with double-buffered shared memory, register allocation under occupancy pressure with rematerialization, a verified peephole pass over PTX, and a proof sketch that no schedule can deadlock the warp scheduler.",
|
||
"Construct an optimizing backend for a RISC-V vector coprocessor with configurable lane widths: vector-length-agnostic instruction selection as cost-model search, an integer-linear-program schedule around a chained load-store unit's structural hazards, VL-dependent spill-cost register allocation with live-range splitting, and a translation-validated peephole pass, plus the argument that the scheduler never issues into an occupied pipeline slot.",
|
||
] },
|
||
{ id: "hard-protocol", cat: "hard", label: ["qa", "code"], judge: "route", max: 8, check: [], variants: [
|
||
"Design and security-review an end-to-end encrypted coordination protocol for a federated fleet of 100000 IoT devices: lightweight key agreement under 32KB RAM budgets, per-group sender-key trees with post-compromise security, secure firmware-update attestation chained to a transparency log, metadata minimization against a network-level adversary, and revocation of physically captured devices without a round-trip to every peer. Analyze the trade-offs precisely, then specify the exact message flow for rotating a compromised region's keys.",
|
||
"Design an end-to-end encrypted messaging system for a disaster-relief mesh network that partitions for days: store-and-forward double ratchet with bounded state, group membership changes that commute across partitions, post-compromise security once connectivity returns, spam and sybil resistance without a central server, and forensic-resistant local storage. Weigh the trade-offs, then give the exact key schedule for merging two long-partitioned group histories.",
|
||
"Architect the security layer of a multi-tenant confidential-computing platform: remote attestation chained through TEE quotes to a transparency log, per-tenant key hierarchies with crypto-agile rotation, sealed storage that survives enclave upgrades, side-channel hardening policy for co-resident tenants, and revocation that cannot be rolled back by a malicious host. Analyze each trade-off, then specify the exact handshake for migrating a sealed workload between hosts.",
|
||
] },
|
||
];
|
||
|
||
const LS_CUSTOM = "allaigate_bench_custom";
|
||
const LS_HISTORY = "allaigate_bench_history";
|
||
|
||
function customTests() {
|
||
try {
|
||
return JSON.parse(localStorage.getItem(LS_CUSTOM) || "[]");
|
||
} catch {
|
||
return [];
|
||
}
|
||
}
|
||
function saveCustom(list) {
|
||
localStorage.setItem(LS_CUSTOM, JSON.stringify(list));
|
||
}
|
||
function history() {
|
||
try {
|
||
return JSON.parse(localStorage.getItem(LS_HISTORY) || "[]");
|
||
} catch {
|
||
return [];
|
||
}
|
||
}
|
||
function pushHistory(run) {
|
||
const l = history();
|
||
l.unshift(run);
|
||
localStorage.setItem(LS_HISTORY, JSON.stringify(l.slice(0, 12)));
|
||
}
|
||
function saveHistory(l) {
|
||
localStorage.setItem(LS_HISTORY, JSON.stringify(l));
|
||
}
|
||
|
||
// ── evaluation (ported from the old benchmark, trimmed) ─────────────────────
|
||
function stripThink(text) {
|
||
return text.replace(/<(?:\|)?(?:think|thought)(?:\|)?>[\s\S]*?(?:<\/(?:\|)?(?:think|thought)(?:\|)?>|$)/gi, "").trim();
|
||
}
|
||
// tail loop: the same 12+ char chunk repeated 3+ times back to back
|
||
function looksLooped(text) {
|
||
const tail = text.slice(-360);
|
||
for (let len = 12; len <= 60; len += 8) {
|
||
const probe = tail.slice(-len);
|
||
if (probe && tail.split(probe).length - 1 >= 3) return true;
|
||
}
|
||
return false;
|
||
}
|
||
function evaluate(raw, test) {
|
||
let text = stripThink(raw);
|
||
if (!text && raw) text = raw; // max_tokens cut inside <think> — judge the raw
|
||
const resp = text.toLowerCase();
|
||
if (!resp.trim()) return { status: "fail", score: 0, note: t("bm.emptyAnswer") };
|
||
if (looksLooped(resp)) return { status: "fail", score: 0, note: t("bm.loopDetected") };
|
||
if (!test.check.length) return { status: "pass", score: 1, note: "✓" };
|
||
let hit = 0;
|
||
for (const cond of test.check) {
|
||
const opts = cond.toLowerCase().split("|").map((s) => s.trim());
|
||
if (opts.some((o) => o && resp.includes(o))) hit++;
|
||
}
|
||
const frac = hit / test.check.length;
|
||
const status = frac >= 1 ? "pass" : frac >= 0.5 ? "partial" : "fail";
|
||
return { status, score: frac, note: `${hit}/${test.check.length}` };
|
||
}
|
||
|
||
export async function renderBenchmark() {
|
||
const modelsData = await api.listModels();
|
||
const ids = (modelsData.models || []).map((m) => m.id);
|
||
const root = h("div");
|
||
let running = false;
|
||
let stopFlag = false;
|
||
|
||
let mode = "routing"; // routing (cortiq-auto, the product's feature) | models
|
||
const modelCbs = ids.map((id) => ({ id, cb: h("input", { type: "checkbox", checked: ids.length === 1 ? true : null }) }));
|
||
const quickCb = h("input", { type: "checkbox", checked: true });
|
||
const resultsHost = h("div", {});
|
||
const progressHost = h("div", {});
|
||
const historyHost = h("div", {});
|
||
|
||
const modeDesc = h("p", { class: "muted", style: "margin:8px 0 0" }, t("bm.mode.routingDesc"));
|
||
const modelsPick = h("div", { class: "flex wrap", style: "gap:10px; margin-top:12px; display:none" },
|
||
...modelCbs.map(({ id, cb }) => h("label", { class: "check" }, cb, h("span", { class: "mono" }, id))));
|
||
const modeSeg = h("div", { class: "seg" },
|
||
h("button", { class: "active" }, "🧭 " + t("bm.mode.routing")),
|
||
h("button", {}, "🎯 " + t("bm.mode.models")));
|
||
modeSeg.querySelectorAll("button").forEach((b, i) =>
|
||
b.addEventListener("click", () => {
|
||
mode = i === 0 ? "routing" : "models";
|
||
modeSeg.querySelectorAll("button").forEach((x, j) => x.classList.toggle("active", j === i));
|
||
modelsPick.style.display = mode === "models" ? "" : "none";
|
||
modeDesc.textContent = t(mode === "routing" ? "bm.mode.routingDesc" : "bm.mode.modelsDesc");
|
||
}));
|
||
|
||
function suite() {
|
||
const base = SUITE.concat(customTests());
|
||
return quickCb.checked ? base.filter((x) => x.quick || x.custom) : base;
|
||
}
|
||
|
||
const runBtn = h("button", { class: "btn primary" }, "▶ " + t("bm.run"));
|
||
const stopBtn = h("button", { class: "btn ghost", style: "display:none" }, "✕ " + t("import.cancel"));
|
||
stopBtn.addEventListener("click", () => {
|
||
stopFlag = true;
|
||
stopBtn.disabled = true;
|
||
});
|
||
|
||
runBtn.addEventListener("click", async () => {
|
||
const models = mode === "routing" ? ["cortiq-auto"] : modelCbs.filter((m) => m.cb.checked).map((m) => m.id);
|
||
if (!models.length) {
|
||
toast(t("bm.pickModel"), "warn");
|
||
return;
|
||
}
|
||
running = true;
|
||
stopFlag = false;
|
||
runBtn.disabled = true;
|
||
stopBtn.style.display = "";
|
||
stopBtn.disabled = false;
|
||
const tests = suite().filter((x) => mode === "routing" || x.judge !== "route");
|
||
const run = { ts: Date.now(), mode, quick: quickCb.checked, models: {}, nTests: tests.length };
|
||
|
||
for (const model of models) {
|
||
const rec = { results: [], score: 0, tokS: [], ttft: [], failed: false };
|
||
run.models[model] = rec;
|
||
for (let i = 0; i < tests.length; i++) {
|
||
if (stopFlag) break;
|
||
const test = tests[i];
|
||
const prompt = test.variants ? test.variants[(Math.random() * test.variants.length) | 0] : test.prompt;
|
||
mount(
|
||
progressHost,
|
||
h("div", { class: "card" },
|
||
h("div", { class: "flex" },
|
||
h("span", { class: "spinner" }), " ",
|
||
h("b", { class: "mono" }, mode === "routing" ? t("bm.mode.routing") : model), " · ",
|
||
t("bm.testOf", { i: i + 1, n: tests.length }), " · ",
|
||
h("span", { class: "muted" }, test.id)),
|
||
h("div", { class: "muted small", style: "margin-top:6px; font-style:italic" }, "«" + prompt.slice(0, 110) + (prompt.length > 110 ? "…»" : "»")),
|
||
h("div", { class: "pbar", style: "margin-top:8px" },
|
||
h("div", { class: "pbar-fill det", style: `width:${Math.round((i / tests.length) * 100)}%` })))
|
||
);
|
||
const started = performance.now();
|
||
let text = "";
|
||
try {
|
||
const info = await testStream(
|
||
// +256: headroom for reasoning models — a tight cap cuts them
|
||
// mid-<think> and turns a correct model into a false ❌
|
||
{ model, messages: [{ role: "user", content: prompt }], temperature: 0, max_tokens: (test.max || 128) + 256 },
|
||
(d) => (text += d)
|
||
);
|
||
const latency = performance.now() - started;
|
||
let ev;
|
||
if (test.judge === "route") {
|
||
// the test asserts the ROUTER's decision: did it rate this high?
|
||
ev = info.tier === "high"
|
||
? { status: "pass", score: 1, note: "high ✓" }
|
||
: { status: "fail", score: 0, note: (info.tier || "—") + " ≠ high" };
|
||
} else {
|
||
ev = evaluate(text, test);
|
||
}
|
||
const compl = info.usage?.completion_tokens;
|
||
if (test.judge !== "route" && ev.status === "fail" && !stripThink(text).trim() && compl >= (test.max || 128) + 250) {
|
||
ev.note = t("bm.cutByCap"); // the whole budget went into <think>
|
||
}
|
||
if (compl && latency > 0) {
|
||
const gen = info.ttft_ms != null ? Math.max(1, latency - info.ttft_ms) : latency;
|
||
rec.tokS.push(compl / (gen / 1000));
|
||
}
|
||
if (info.ttft_ms != null) rec.ttft.push(info.ttft_ms);
|
||
const row = { id: test.id, cat: test.cat, ...ev, latency: Math.round(latency), cost: info.cost_usd || 0, answer: stripThink(text).slice(0, 300) };
|
||
if (mode === "routing") {
|
||
// router timed out / fell back — there was no classification to
|
||
// judge, so it must not count against accuracy
|
||
row.degraded = !info.task_label
|
||
|| info.task_label === "degraded"
|
||
|| String(info.route_source || "").includes("degraded");
|
||
row.task = info.task_label || "—";
|
||
row.tier = info.tier || "—";
|
||
row.chosen = info.selected_model || "—";
|
||
row.source = info.route_source || "—";
|
||
row.expected = test.label; // null = classification not judged
|
||
row.taskOk = test.label && !row.degraded ? test.label.includes(info.task_label) : null;
|
||
}
|
||
rec.results.push(row);
|
||
} catch (e) {
|
||
rec.results.push({ id: test.id, cat: test.cat, status: "fail", score: 0, note: String(e.message || e).slice(0, 80), latency: 0, answer: "" });
|
||
rec.failed = true;
|
||
}
|
||
renderResults(run, true); // live: строка появляется сразу после теста
|
||
}
|
||
if (stopFlag) break;
|
||
}
|
||
|
||
mount(progressHost);
|
||
running = false;
|
||
runBtn.disabled = false;
|
||
stopBtn.style.display = "none";
|
||
if (!stopFlag) {
|
||
pushHistory(run);
|
||
// финальный аккорд: короткий вердикт с процентами и деньгами
|
||
const first = run.models[Object.keys(run.models)[0]];
|
||
if (first) {
|
||
const st = stats(first);
|
||
const judged = first.results.filter((r) => r.taskOk != null);
|
||
const cls = judged.length ? ` · ${judged.filter((r) => r.taskOk).length}/${judged.length}` : "";
|
||
const total = Object.values(run.models).reduce((a, rec) => a + stats(rec).cost, 0);
|
||
toast(t("bm.verdict", { p: Math.round(st.score * 100) + "%" + cls, s: money(total) }), "ok", 6000);
|
||
}
|
||
}
|
||
renderResults(run);
|
||
renderHistory();
|
||
});
|
||
|
||
function statusIcon(s) {
|
||
return s === "pass" ? "✅" : s === "partial" ? "◐" : "❌";
|
||
}
|
||
|
||
function stats(rec) {
|
||
const n = rec.results.length || 1;
|
||
return {
|
||
score: rec.results.reduce((a, r) => a + r.score, 0) / n,
|
||
tokS: rec.tokS.length ? rec.tokS.reduce((a, b) => a + b, 0) / rec.tokS.length : null,
|
||
ttft: rec.ttft.length ? rec.ttft.reduce((a, b) => a + b, 0) / rec.ttft.length : null,
|
||
cost: rec.results.reduce((a, r) => a + (r.cost || 0), 0),
|
||
};
|
||
}
|
||
|
||
function renderResults(run, live) {
|
||
const models = Object.keys(run.models);
|
||
if (!models.length) {
|
||
mount(resultsHost);
|
||
return;
|
||
}
|
||
const routing = run.mode === "routing";
|
||
const detail = h("div", {});
|
||
|
||
let summary;
|
||
if (routing) {
|
||
const rec = run.models[models[0]];
|
||
const st = stats(rec);
|
||
const judged = rec.results.filter((r) => r.taskOk != null);
|
||
const okCount = judged.filter((r) => r.taskOk).length;
|
||
// какие модели реально выбирал роутер
|
||
const dist = {};
|
||
for (const r of rec.results) {
|
||
if (!r.chosen) continue;
|
||
dist[r.chosen] = dist[r.chosen] || { n: 0, cost: 0 };
|
||
dist[r.chosen].n++;
|
||
dist[r.chosen].cost += r.cost || 0;
|
||
}
|
||
const pct = Math.round(st.score * 100);
|
||
summary = h("div", {},
|
||
h("div", { class: "kpis", style: "grid-template-columns: repeat(auto-fit, minmax(150px, 1fr))" },
|
||
h("div", { class: "kpi" }, h("div", { class: "big" }, pct + "%"), h("div", { class: "lbl" }, t("bm.answersOk"))),
|
||
h("div", { class: "kpi" }, h("div", { class: "big" }, judged.length ? `${okCount}/${judged.length}` : "—"), h("div", { class: "lbl" }, t("bm.classAcc"))),
|
||
h("div", { class: "kpi" }, h("div", { class: "big" }, st.ttft != null ? ms(Math.round(st.ttft)) : "—"), h("div", { class: "lbl" }, t("pg.ttft"))),
|
||
h("div", { class: "kpi" }, h("div", { class: "big" }, st.tokS != null ? st.tokS.toFixed(1) : "—"), h("div", { class: "lbl" }, "tok/s")),
|
||
h("div", { class: "kpi" }, h("div", { class: "big" }, moneyEl(st.cost)), h("div", { class: "lbl" }, t("bm.spent")))),
|
||
h("div", { class: "flex wrap", style: "gap:6px; margin-top:4px" },
|
||
h("span", { class: "muted small" }, t("bm.modelDist") + ":"),
|
||
...Object.entries(dist).map(([m, d]) =>
|
||
h("span", { class: "badge" }, `${m} × ${d.n}`, d.cost > 0 ? " · " : "", d.cost > 0 ? moneyEl(d.cost) : ""))));
|
||
} else {
|
||
const sorted = models.slice().sort((a, b) => stats(run.models[b]).score - stats(run.models[a]).score);
|
||
summary = h("div", {},
|
||
...sorted.map((m) => {
|
||
const st = stats(run.models[m]);
|
||
const pct = Math.round(st.score * 100);
|
||
return h("div", { class: "bm-sum" },
|
||
h("div", { class: "flex" },
|
||
h("b", { class: "mono grow" }, m),
|
||
h("span", { class: "badge " + (pct >= 80 ? "ok" : pct >= 50 ? "" : "bad") }, pct + "%"),
|
||
st.tokS != null ? h("span", { class: "muted small", style: "margin-left:8px" }, st.tokS.toFixed(1) + " tok/s") : null,
|
||
st.ttft != null ? h("span", { class: "muted small", style: "margin-left:8px" }, "TTFT " + ms(Math.round(st.ttft))) : null,
|
||
st.cost > 0 ? h("span", { class: "muted small", style: "margin-left:8px" }, t("bm.spent") + " ", moneyEl(st.cost)) : null),
|
||
h("div", { class: "sysbar" }, h("i", { class: pct < 50 ? "bad" : "", style: `width:${pct}%` })));
|
||
}));
|
||
}
|
||
|
||
function answerCell(r, m) {
|
||
const cell = h("td", { class: "bm-cell", title: r.answer || r.note },
|
||
statusIcon(r.status) + " ",
|
||
h("span", { class: "muted small" }, r.note + (r.latency ? ` · ${(r.latency / 1000).toFixed(1)}s` : "")));
|
||
cell.addEventListener("click", () => {
|
||
mount(detail, h("div", { class: "card" },
|
||
h("div", { class: "card-title" }, `${r.id} · ${m} · ${statusIcon(r.status)}`),
|
||
h("pre", { class: "conv-log", style: "max-height:200px" }, r.answer || "—")));
|
||
});
|
||
return cell;
|
||
}
|
||
|
||
let header, rows;
|
||
if (routing) {
|
||
const rec = run.models[models[0]];
|
||
header = h("tr", {},
|
||
h("th", {}, t("bm.test")),
|
||
h("th", {}, t("pg.task")),
|
||
h("th", {}, t("pg.tier")),
|
||
h("th", {}, t("pg.model")),
|
||
h("th", {}, t("bm.answer")));
|
||
rows = rec.results.map((r) =>
|
||
h("tr", {},
|
||
h("td", {}, h("div", {}, r.id), h("div", { class: "muted small" }, r.cat)),
|
||
h("td", {},
|
||
r.degraded
|
||
? h("span", { class: "muted", title: t("bm.routerDegraded") }, "⚠ " + t("bm.degraded"))
|
||
: r.taskOk == null
|
||
? h("span", { class: "muted" }, r.task || "—")
|
||
: h("span", { class: r.taskOk ? "" : "err" },
|
||
(r.taskOk ? "✓ " : "✗ ") + r.task,
|
||
r.taskOk ? "" : h("div", { class: "muted small" }, t("bm.expectedLbl") + " " + (r.expected || []).join("/")))),
|
||
h("td", {}, r.tier ? h("span", { class: "badge tier-" + r.tier }, r.tier) : "—"),
|
||
h("td", {},
|
||
h("span", { class: "mono small" }, r.chosen || "—"),
|
||
r.cost > 0 ? h("div", { class: "muted small" }, moneyEl(r.cost)) : null,
|
||
r.source && r.source.includes("task-rule") ? h("div", { class: "badge", style: "margin-top:2px" }, "task-rule") : null),
|
||
answerCell(r, r.chosen || "cortiq-auto")));
|
||
} else {
|
||
header = h("tr", {}, h("th", {}, t("bm.test")), ...models.map((m) => h("th", { class: "mono" }, m)));
|
||
rows = (run.models[models[0]].results || []).map((_, i) =>
|
||
h("tr", {},
|
||
h("td", {},
|
||
h("div", {}, run.models[models[0]].results[i].id),
|
||
h("div", { class: "muted small" }, run.models[models[0]].results[i].cat)),
|
||
...models.map((m) => {
|
||
const r = run.models[m].results[i];
|
||
return r ? answerCell(r, m) : h("td", {}, "—");
|
||
})));
|
||
}
|
||
|
||
mount(resultsHost,
|
||
h("div", { class: "card" },
|
||
h("div", { class: "card-head" },
|
||
h("h3", {}, (routing ? "🧭 " : "🎯 ") + t("bm.results")),
|
||
h("span", { class: "card-head sub" },
|
||
(live ? t("bm.liveRunning") + " · " : "") + new Date(run.ts).toLocaleString() + (run.quick ? " · " + t("bm.quick") : ""))),
|
||
summary,
|
||
h("div", { class: "table-wrap", style: "margin-top:14px" }, h("table", {}, h("thead", {}, header), h("tbody", {}, ...rows))),
|
||
h("div", { class: "muted small", style: "margin-top:6px" }, t("bm.cellHint"))),
|
||
detail);
|
||
}
|
||
|
||
function renderHistory() {
|
||
const l = history();
|
||
if (!l.length) {
|
||
mount(historyHost);
|
||
return;
|
||
}
|
||
const clearAll = h("button", { class: "btn ghost sm danger" }, "🗑 " + t("bm.clearHistory"));
|
||
clearAll.addEventListener("click", () => {
|
||
saveHistory([]);
|
||
renderHistory();
|
||
});
|
||
mount(historyHost, h("div", { class: "card" },
|
||
h("div", { class: "card-head" },
|
||
h("h3", {}, t("bm.history")),
|
||
h("div", { class: "right" }, clearAll)),
|
||
...l.map((run, idx) => {
|
||
const models = Object.keys(run.models);
|
||
const scoreOf = (m) => {
|
||
const rs = run.models[m].results || [];
|
||
return rs.length ? rs.reduce((a, r) => a + r.score, 0) / rs.length : 0;
|
||
};
|
||
const best = models.slice().sort((a, b) => scoreOf(b) - scoreOf(a))[0];
|
||
const row = h("div", { class: "flex bench-row", style: "cursor:pointer" },
|
||
h("span", {}, run.mode === "routing" ? "🧭" : "🎯"),
|
||
h("span", { class: "muted small" }, new Date(run.ts).toLocaleString()),
|
||
h("span", { class: "grow mono small" }, run.mode === "routing" ? t("bm.mode.routing") : models.join(", ")),
|
||
run.quick ? h("span", { class: "badge" }, t("bm.quick")) : null,
|
||
best ? h("b", {}, Math.round(scoreOf(best) * 100) + "%") : null,
|
||
h("button", { class: "btn sm ghost danger", title: t("common.delete"), onClick: (e) => {
|
||
e.stopPropagation();
|
||
l.splice(idx, 1);
|
||
saveHistory(l);
|
||
renderHistory();
|
||
} }, "×"));
|
||
row.addEventListener("click", () => renderResults(run));
|
||
return row;
|
||
})));
|
||
}
|
||
|
||
// ── custom tests (stored in the browser) ──
|
||
function customCard() {
|
||
const host = h("div", { class: "card" });
|
||
function paint() {
|
||
const list = customTests();
|
||
const promptIn = h("input", { placeholder: t("bm.promptPh") });
|
||
const checkIn = h("input", { placeholder: t("bm.checkPh") });
|
||
const addBtn = h("button", { class: "btn ghost sm" }, "+ " + t("bm.addTest"));
|
||
addBtn.addEventListener("click", () => {
|
||
const prompt = promptIn.value.trim();
|
||
const check = checkIn.value.split(",").map((s) => s.trim()).filter(Boolean);
|
||
if (!prompt || !check.length) return;
|
||
list.push({ id: "custom-" + (list.length + 1), cat: "custom", custom: true, max: 128, prompt, check });
|
||
saveCustom(list);
|
||
paint();
|
||
});
|
||
mount(host,
|
||
h("div", { class: "card-title" }, t("bm.custom")),
|
||
h("div", { class: "muted small", style: "margin-bottom:8px" }, t("bm.customHint")),
|
||
...list.map((tst, i) =>
|
||
h("div", { class: "flex bench-row" },
|
||
h("span", { class: "grow small" }, tst.prompt.slice(0, 90)),
|
||
h("span", { class: "muted small mono" }, tst.check.join(", ").slice(0, 40)),
|
||
h("button", { class: "btn sm ghost danger", onClick: () => { list.splice(i, 1); saveCustom(list); paint(); } }, "×"))),
|
||
h("div", { class: "flex", style: "gap:8px; margin-top:8px" }, promptIn, checkIn, addBtn));
|
||
}
|
||
paint();
|
||
return host;
|
||
}
|
||
|
||
mount(root,
|
||
h("div", { class: "page-head" },
|
||
h("h2", {}, t("bm.title")),
|
||
h("p", {}, t("bm.subtitle"))),
|
||
ids.length === 0 ? h("div", { class: "callout" }, t("routing.empty")) : null,
|
||
h("div", { class: "card" },
|
||
modeSeg,
|
||
modeDesc,
|
||
modelsPick,
|
||
h("div", { class: "flex", style: "justify-content:space-between; margin-top:14px" },
|
||
h("label", { class: "check", style: "margin:0", title: t("bm.quickHint") }, quickCb, "⚡ " + t("bm.quickMode")),
|
||
h("div", { class: "flex", style: "gap:8px" }, stopBtn, runBtn)),
|
||
h("div", { class: "muted small", style: "margin-top:6px" },
|
||
t("bm.suiteInfo", { q: SUITE.filter((x) => x.quick).length, f: SUITE.length }))),
|
||
progressHost,
|
||
resultsHost,
|
||
customCard(),
|
||
historyHost);
|
||
renderHistory();
|
||
return root;
|
||
}
|