Files
cortiq-gateway/web/js/views/benchmark.js
T
infosave2007 ec92bdddb7 feat(benchmark): smart-routing mode is the point — live results, per-model cost
The benchmark now leads with the product's actual feature. Two explicit modes:

  🧭 Smart routing (default) — every probe goes through cortiq-auto; each row
     shows the router's task label judged against the expected one (✓/✗ with
     "expected: …"), the complexity tier, the chosen model with what that
     request cost, and a task-rule badge when a routing rule fired. KPIs:
     answer quality, classification accuracy, TTFT, tok/s and money spent,
     plus a "requests went to" distribution with per-model cost.
  🎯 Individual models — the old head-to-head comparison, now as the clearly
     separate option (model checkboxes appear only here).

The suite covers all 8 taxonomy labels (added summarization, creative-writing
and chitchat probes; expected labels annotated per test). Results are LIVE:
the table grows a row after every test (with a small fade-in), the progress
card shows the prompt being asked, and a 🏁 verdict toast lands at the end
with the score and the total spend.

12 new strings in all 7 languages.
2026-08-07 19:04:41 +03:00

424 lines
21 KiB
JavaScript
Vendored
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Benchmark — two modes over the live pipeline:
// 🧭 Smart routing (default): the product's core promise under test — every
// probe goes through cortiq-auto and the router's task/tier decision and
// chosen model are judged alongside the answer itself.
// 🎯 Individual models: the same suite pinned to selected models — compare
// quality and speed head-to-head.
// A redesign of the old standalone benchmark.html: same scoring ideas
// (keyword checks with | synonyms, think-tag stripping, anti-loop detection,
// pass/partial/fail), now embedded, localized and CDN-free.
import { h, mount, toast, ms, money } from "../ui.js";
import { t } from "../i18n.js";
import { api, testStream } from "../api.js";
// ── built-in suite ──────────────────────────────────────────────────────────
// Checks are language-neutral where possible (digits, identifiers, emails);
// `check` entries may hold |-separated synonyms — one hit per entry counts.
// `quick: true` marks the fast subset.
// `label`: router task labels accepted as a correct classification (null = not
// judged — the prompt has no natural label in the taxonomy).
const SUITE = [
{ id: "math-mult", cat: "math", label: ["math"], quick: true, max: 96, prompt: "What is 17 * 23? Answer with the number only.", check: ["391"] },
{ id: "math-eq", cat: "math", label: ["math"], max: 128, prompt: "Solve 2x + 6 = 14. Answer with the value of x only.", check: ["4"] },
{ id: "logic-syll", cat: "logic", label: ["qa", "math"], max: 96, prompt: "If all bloops are razzies and all razzies are lazzies, are all bloops lazzies? Answer yes or no.", check: ["yes|да|oui|ja|sí|si|evet|是"] },
{ id: "count-letters", cat: "logic", label: ["qa", "math"], quick: true, max: 64, prompt: "How many letters are in the word 'cat'? Answer with a digit.", check: ["3"] },
{ id: "apples", cat: "logic", label: ["math", "qa"], quick: true, max: 96, prompt: "John has 3 apples, buys 2 more, then eats 1. How many apples does he have? Answer with a digit.", check: ["4"] },
{ id: "code-fact", cat: "code", label: ["code"], max: 256, prompt: "Write a Python function factorial(n) that returns n!.", check: ["def", "factorial", "return"] },
{ id: "qa-capital", cat: "qa", label: ["qa"], quick: true, max: 48, prompt: "What is the capital of France? One word.", check: ["paris|париж"] },
{ id: "primes", cat: "qa", label: ["math", "qa"], max: 96, prompt: "List the first five prime numbers, comma-separated.", check: ["2", "3", "5", "7", "11"] },
{ id: "instr-ok", cat: "instruction", label: null, quick: true, max: 32, prompt: "Reply with exactly the word OK and nothing else.", check: ["ok"] },
{ id: "extract-email", cat: "extraction", label: ["extraction"], quick: true, max: 64, prompt: "Extract the email address from: 'Contact us at info@example.com today.' Reply with the address only.", check: ["info@example.com"] },
{ id: "translate", cat: "translation", label: ["translation"], max: 64, prompt: "Translate to English: «Доброе утро». Reply with the translation only.", check: ["good morning"] },
{ id: "format-json", cat: "instruction", label: null, max: 96, prompt: 'Return a JSON object with a single key "n" whose value is 7. JSON only.', check: ['"n"', "7"] },
{ id: "summarize", cat: "summarization", label: ["summarization"], max: 96, prompt: "Summarize in one short sentence: 'The cat sat on the warm mat by the window. After a while it curled up and fell asleep in the sun.'", check: ["cat|кот|кошк"] },
{ id: "creative", cat: "creative-writing", label: ["creative-writing"], max: 96, prompt: "Write a two-line poem about the sea.", check: ["sea|море|ocean"] },
{ id: "chitchat", cat: "chitchat", label: ["chitchat"], quick: true, max: 48, prompt: "Hi! How are you today?", check: [] },
];
const LS_CUSTOM = "allaigate_bench_custom";
const LS_HISTORY = "allaigate_bench_history";
function customTests() {
try {
return JSON.parse(localStorage.getItem(LS_CUSTOM) || "[]");
} catch {
return [];
}
}
function saveCustom(list) {
localStorage.setItem(LS_CUSTOM, JSON.stringify(list));
}
function history() {
try {
return JSON.parse(localStorage.getItem(LS_HISTORY) || "[]");
} catch {
return [];
}
}
function pushHistory(run) {
const l = history();
l.unshift(run);
localStorage.setItem(LS_HISTORY, JSON.stringify(l.slice(0, 12)));
}
// ── evaluation (ported from the old benchmark, trimmed) ─────────────────────
function stripThink(text) {
return text.replace(/<(?:\|)?(?:think|thought)(?:\|)?>[\s\S]*?(?:<\/(?:\|)?(?:think|thought)(?:\|)?>|$)/gi, "").trim();
}
// tail loop: the same 12+ char chunk repeated 3+ times back to back
function looksLooped(text) {
const tail = text.slice(-360);
for (let len = 12; len <= 60; len += 8) {
const probe = tail.slice(-len);
if (probe && tail.split(probe).length - 1 >= 3) return true;
}
return false;
}
function evaluate(raw, test) {
let text = stripThink(raw);
if (!text && raw) text = raw; // max_tokens cut inside <think> — judge the raw
const resp = text.toLowerCase();
if (!resp.trim()) return { status: "fail", score: 0, note: t("bm.emptyAnswer") };
if (looksLooped(resp)) return { status: "fail", score: 0, note: t("bm.loopDetected") };
if (!test.check.length) return { status: "pass", score: 1, note: "✓" };
let hit = 0;
for (const cond of test.check) {
const opts = cond.toLowerCase().split("|").map((s) => s.trim());
if (opts.some((o) => o && resp.includes(o))) hit++;
}
const frac = hit / test.check.length;
const status = frac >= 1 ? "pass" : frac >= 0.5 ? "partial" : "fail";
return { status, score: frac, note: `${hit}/${test.check.length}` };
}
export async function renderBenchmark() {
const modelsData = await api.listModels();
const ids = (modelsData.models || []).map((m) => m.id);
const root = h("div");
let running = false;
let stopFlag = false;
let mode = "routing"; // routing (cortiq-auto, the product's feature) | models
const modelCbs = ids.map((id) => ({ id, cb: h("input", { type: "checkbox", checked: ids.length === 1 ? true : null }) }));
const quickCb = h("input", { type: "checkbox", checked: true });
const resultsHost = h("div", {});
const progressHost = h("div", {});
const historyHost = h("div", {});
const modeDesc = h("p", { class: "muted", style: "margin:8px 0 0" }, t("bm.mode.routingDesc"));
const modelsPick = h("div", { class: "flex wrap", style: "gap:10px; margin-top:12px; display:none" },
...modelCbs.map(({ id, cb }) => h("label", { class: "check" }, cb, h("span", { class: "mono" }, id))));
const modeSeg = h("div", { class: "seg" },
h("button", { class: "active" }, "🧭 " + t("bm.mode.routing")),
h("button", {}, "🎯 " + t("bm.mode.models")));
modeSeg.querySelectorAll("button").forEach((b, i) =>
b.addEventListener("click", () => {
mode = i === 0 ? "routing" : "models";
modeSeg.querySelectorAll("button").forEach((x, j) => x.classList.toggle("active", j === i));
modelsPick.style.display = mode === "models" ? "" : "none";
modeDesc.textContent = t(mode === "routing" ? "bm.mode.routingDesc" : "bm.mode.modelsDesc");
}));
function suite() {
const base = SUITE.concat(customTests());
return quickCb.checked ? base.filter((x) => x.quick || x.custom) : base;
}
const runBtn = h("button", { class: "btn primary" }, "▶ " + t("bm.run"));
const stopBtn = h("button", { class: "btn ghost", style: "display:none" }, "✕ " + t("import.cancel"));
stopBtn.addEventListener("click", () => {
stopFlag = true;
stopBtn.disabled = true;
});
runBtn.addEventListener("click", async () => {
const models = mode === "routing" ? ["cortiq-auto"] : modelCbs.filter((m) => m.cb.checked).map((m) => m.id);
if (!models.length) {
toast(t("bm.pickModel"), "warn");
return;
}
running = true;
stopFlag = false;
runBtn.disabled = true;
stopBtn.style.display = "";
stopBtn.disabled = false;
const tests = suite();
const run = { ts: Date.now(), mode, quick: quickCb.checked, models: {}, nTests: tests.length };
for (const model of models) {
const rec = { results: [], score: 0, tokS: [], ttft: [], failed: false };
run.models[model] = rec;
for (let i = 0; i < tests.length; i++) {
if (stopFlag) break;
const test = tests[i];
mount(
progressHost,
h("div", { class: "card" },
h("div", { class: "flex" },
h("span", { class: "spinner" }), " ",
h("b", { class: "mono" }, mode === "routing" ? t("bm.mode.routing") : model), " · ",
t("bm.testOf", { i: i + 1, n: tests.length }), " · ",
h("span", { class: "muted" }, test.id)),
h("div", { class: "muted small", style: "margin-top:6px; font-style:italic" }, "«" + test.prompt.slice(0, 110) + (test.prompt.length > 110 ? "…»" : "»")),
h("div", { class: "pbar", style: "margin-top:8px" },
h("div", { class: "pbar-fill det", style: `width:${Math.round((i / tests.length) * 100)}%` })))
);
const started = performance.now();
let text = "";
try {
const info = await testStream(
{ model, messages: [{ role: "user", content: test.prompt }], temperature: 0, max_tokens: test.max || 128 },
(d) => (text += d)
);
const latency = performance.now() - started;
const ev = evaluate(text, test);
const compl = info.usage?.completion_tokens;
if (compl && latency > 0) {
const gen = info.ttft_ms != null ? Math.max(1, latency - info.ttft_ms) : latency;
rec.tokS.push(compl / (gen / 1000));
}
if (info.ttft_ms != null) rec.ttft.push(info.ttft_ms);
const row = { id: test.id, cat: test.cat, ...ev, latency: Math.round(latency), cost: info.cost_usd || 0, answer: stripThink(text).slice(0, 300) };
if (mode === "routing") {
row.task = info.task_label || "—";
row.tier = info.tier || "—";
row.chosen = info.selected_model || "—";
row.source = info.route_source || "—";
row.expected = test.label; // null = classification not judged
row.taskOk = test.label ? test.label.includes(info.task_label) : null;
}
rec.results.push(row);
} catch (e) {
rec.results.push({ id: test.id, cat: test.cat, status: "fail", score: 0, note: String(e.message || e).slice(0, 80), latency: 0, answer: "" });
rec.failed = true;
}
renderResults(run, true); // live: строка появляется сразу после теста
}
if (stopFlag) break;
}
mount(progressHost);
running = false;
runBtn.disabled = false;
stopBtn.style.display = "none";
if (!stopFlag) {
pushHistory(run);
// финальный аккорд: короткий вердикт с процентами и деньгами
const first = run.models[Object.keys(run.models)[0]];
if (first) {
const st = stats(first);
const judged = first.results.filter((r) => r.taskOk != null);
const cls = judged.length ? ` · ${judged.filter((r) => r.taskOk).length}/${judged.length}` : "";
const total = Object.values(run.models).reduce((a, rec) => a + stats(rec).cost, 0);
toast(t("bm.verdict", { p: Math.round(st.score * 100) + "%" + cls, s: money(total) }), "ok", 6000);
}
}
renderResults(run);
renderHistory();
});
function statusIcon(s) {
return s === "pass" ? "✅" : s === "partial" ? "◐" : "❌";
}
function stats(rec) {
const n = rec.results.length || 1;
return {
score: rec.results.reduce((a, r) => a + r.score, 0) / n,
tokS: rec.tokS.length ? rec.tokS.reduce((a, b) => a + b, 0) / rec.tokS.length : null,
ttft: rec.ttft.length ? rec.ttft.reduce((a, b) => a + b, 0) / rec.ttft.length : null,
cost: rec.results.reduce((a, r) => a + (r.cost || 0), 0),
};
}
function renderResults(run, live) {
const models = Object.keys(run.models);
if (!models.length) {
mount(resultsHost);
return;
}
const routing = run.mode === "routing";
const detail = h("div", {});
let summary;
if (routing) {
const rec = run.models[models[0]];
const st = stats(rec);
const judged = rec.results.filter((r) => r.taskOk != null);
const okCount = judged.filter((r) => r.taskOk).length;
// какие модели реально выбирал роутер
const dist = {};
for (const r of rec.results) {
if (!r.chosen) continue;
dist[r.chosen] = dist[r.chosen] || { n: 0, cost: 0 };
dist[r.chosen].n++;
dist[r.chosen].cost += r.cost || 0;
}
const pct = Math.round(st.score * 100);
summary = h("div", {},
h("div", { class: "kpis", style: "grid-template-columns: repeat(auto-fit, minmax(150px, 1fr))" },
h("div", { class: "kpi" }, h("div", { class: "big" }, pct + "%"), h("div", { class: "lbl" }, t("bm.answersOk"))),
h("div", { class: "kpi" }, h("div", { class: "big" }, judged.length ? `${okCount}/${judged.length}` : "—"), h("div", { class: "lbl" }, t("bm.classAcc"))),
h("div", { class: "kpi" }, h("div", { class: "big" }, st.ttft != null ? ms(Math.round(st.ttft)) : "—"), h("div", { class: "lbl" }, t("pg.ttft"))),
h("div", { class: "kpi" }, h("div", { class: "big" }, st.tokS != null ? st.tokS.toFixed(1) : "—"), h("div", { class: "lbl" }, "tok/s")),
h("div", { class: "kpi" }, h("div", { class: "big" }, money(st.cost)), h("div", { class: "lbl" }, t("bm.spent")))),
h("div", { class: "flex wrap", style: "gap:6px; margin-top:4px" },
h("span", { class: "muted small" }, t("bm.modelDist") + ":"),
...Object.entries(dist).map(([m, d]) =>
h("span", { class: "badge" }, `${m} × ${d.n}` + (d.cost > 0 ? ` · ${money(d.cost)}` : "")))));
} else {
const sorted = models.slice().sort((a, b) => stats(run.models[b]).score - stats(run.models[a]).score);
summary = h("div", {},
...sorted.map((m) => {
const st = stats(run.models[m]);
const pct = Math.round(st.score * 100);
return h("div", { class: "bm-sum" },
h("div", { class: "flex" },
h("b", { class: "mono grow" }, m),
h("span", { class: "badge " + (pct >= 80 ? "ok" : pct >= 50 ? "" : "bad") }, pct + "%"),
st.tokS != null ? h("span", { class: "muted small", style: "margin-left:8px" }, st.tokS.toFixed(1) + " tok/s") : null,
st.ttft != null ? h("span", { class: "muted small", style: "margin-left:8px" }, "TTFT " + ms(Math.round(st.ttft))) : null,
st.cost > 0 ? h("span", { class: "muted small", style: "margin-left:8px" }, t("bm.spent") + " " + money(st.cost)) : null),
h("div", { class: "sysbar" }, h("i", { class: pct < 50 ? "bad" : "", style: `width:${pct}%` })));
}));
}
function answerCell(r, m) {
const cell = h("td", { class: "bm-cell", title: r.answer || r.note },
statusIcon(r.status) + " ",
h("span", { class: "muted small" }, r.note + (r.latency ? ` · ${(r.latency / 1000).toFixed(1)}s` : "")));
cell.addEventListener("click", () => {
mount(detail, h("div", { class: "card" },
h("div", { class: "card-title" }, `${r.id} · ${m} · ${statusIcon(r.status)}`),
h("pre", { class: "conv-log", style: "max-height:200px" }, r.answer || "—")));
});
return cell;
}
let header, rows;
if (routing) {
const rec = run.models[models[0]];
header = h("tr", {},
h("th", {}, t("bm.test")),
h("th", {}, t("pg.task")),
h("th", {}, t("pg.tier")),
h("th", {}, t("pg.model")),
h("th", {}, t("bm.answer")));
rows = rec.results.map((r) =>
h("tr", {},
h("td", {}, h("div", {}, r.id), h("div", { class: "muted small" }, r.cat)),
h("td", {},
r.taskOk == null
? h("span", { class: "muted" }, r.task || "—")
: h("span", { class: r.taskOk ? "" : "err" },
(r.taskOk ? "✓ " : "✗ ") + r.task,
r.taskOk ? "" : h("div", { class: "muted small" }, t("bm.expectedLbl") + " " + (r.expected || []).join("/")))),
h("td", {}, r.tier ? h("span", { class: "badge tier-" + r.tier }, r.tier) : "—"),
h("td", {},
h("span", { class: "mono small" }, r.chosen || "—"),
r.cost > 0 ? h("div", { class: "muted small" }, money(r.cost)) : null,
r.source && r.source.includes("task-rule") ? h("div", { class: "badge", style: "margin-top:2px" }, "task-rule") : null),
answerCell(r, r.chosen || "cortiq-auto")));
} else {
header = h("tr", {}, h("th", {}, t("bm.test")), ...models.map((m) => h("th", { class: "mono" }, m)));
rows = (run.models[models[0]].results || []).map((_, i) =>
h("tr", {},
h("td", {},
h("div", {}, run.models[models[0]].results[i].id),
h("div", { class: "muted small" }, run.models[models[0]].results[i].cat)),
...models.map((m) => {
const r = run.models[m].results[i];
return r ? answerCell(r, m) : h("td", {}, "—");
})));
}
mount(resultsHost,
h("div", { class: "card" },
h("div", { class: "card-head" },
h("h3", {}, (routing ? "🧭 " : "🎯 ") + t("bm.results")),
h("span", { class: "card-head sub" },
(live ? t("bm.liveRunning") + " · " : "") + new Date(run.ts).toLocaleString() + (run.quick ? " · " + t("bm.quick") : ""))),
summary,
h("div", { class: "table-wrap", style: "margin-top:14px" }, h("table", {}, h("thead", {}, header), h("tbody", {}, ...rows))),
h("div", { class: "muted small", style: "margin-top:6px" }, t("bm.cellHint"))),
detail);
}
function renderHistory() {
const l = history();
if (!l.length) {
mount(historyHost);
return;
}
mount(historyHost, h("div", { class: "card" },
h("div", { class: "card-title" }, t("bm.history")),
...l.map((run) => {
const models = Object.keys(run.models);
const scoreOf = (m) => {
const rs = run.models[m].results || [];
return rs.length ? rs.reduce((a, r) => a + r.score, 0) / rs.length : 0;
};
const best = models.slice().sort((a, b) => scoreOf(b) - scoreOf(a))[0];
const row = h("div", { class: "flex bench-row", style: "cursor:pointer" },
h("span", {}, run.mode === "routing" ? "🧭" : "🎯"),
h("span", { class: "muted small" }, new Date(run.ts).toLocaleString()),
h("span", { class: "grow mono small" }, run.mode === "routing" ? t("bm.mode.routing") : models.join(", ")),
run.quick ? h("span", { class: "badge" }, t("bm.quick")) : null,
best ? h("b", {}, Math.round(scoreOf(best) * 100) + "%") : null);
row.addEventListener("click", () => renderResults(run));
return row;
})));
}
// ── custom tests (stored in the browser) ──
function customCard() {
const host = h("div", { class: "card" });
function paint() {
const list = customTests();
const promptIn = h("input", { placeholder: t("bm.promptPh") });
const checkIn = h("input", { placeholder: t("bm.checkPh") });
const addBtn = h("button", { class: "btn ghost sm" }, "+ " + t("bm.addTest"));
addBtn.addEventListener("click", () => {
const prompt = promptIn.value.trim();
const check = checkIn.value.split(",").map((s) => s.trim()).filter(Boolean);
if (!prompt || !check.length) return;
list.push({ id: "custom-" + (list.length + 1), cat: "custom", custom: true, max: 128, prompt, check });
saveCustom(list);
paint();
});
mount(host,
h("div", { class: "card-title" }, t("bm.custom")),
h("div", { class: "muted small", style: "margin-bottom:8px" }, t("bm.customHint")),
...list.map((tst, i) =>
h("div", { class: "flex bench-row" },
h("span", { class: "grow small" }, tst.prompt.slice(0, 90)),
h("span", { class: "muted small mono" }, tst.check.join(", ").slice(0, 40)),
h("button", { class: "btn sm ghost danger", onClick: () => { list.splice(i, 1); saveCustom(list); paint(); } }, "×"))),
h("div", { class: "flex", style: "gap:8px; margin-top:8px" }, promptIn, checkIn, addBtn));
}
paint();
return host;
}
mount(root,
h("div", { class: "page-head" },
h("h2", {}, t("bm.title")),
h("p", {}, t("bm.subtitle"))),
ids.length === 0 ? h("div", { class: "callout" }, t("routing.empty")) : null,
h("div", { class: "card" },
modeSeg,
modeDesc,
modelsPick,
h("div", { class: "flex", style: "justify-content:space-between; margin-top:14px" },
h("label", { class: "check", style: "margin:0", title: t("bm.quickHint") }, quickCb, "⚡ " + t("bm.quickMode")),
h("div", { class: "flex", style: "gap:8px" }, stopBtn, runBtn)),
h("div", { class: "muted small", style: "margin-top:6px" },
t("bm.suiteInfo", { q: SUITE.filter((x) => x.quick).length, f: SUITE.length }))),
progressHost,
resultsHost,
customCard(),
historyHost);
renderHistory();
return root;
}