feat(benchmark): the built-in benchmark, redesigned into the console
The old standalone benchmark.html (Russian-only, Tailwind+Chart.js from CDNs,
never shipped in the crate) becomes a proper sidebar view. Same scoring ideas,
reimplemented: keyword checks with |-synonyms, think-tag stripping, tail-loop
detection, pass/partial/fail — plus what the old page never had: runs go
through the LIVE pipeline (pinned per model, streamed), so every test also
measures tok/s and time-to-first-token with the gateway's real token counts.
A built-in suite of 12 language-neutral probes (math, logic, code, QA,
instruction-following, extraction, translation, JSON formatting); the ⚡ quick
mode runs the 6 short ones — about a minute on a local model. Results: score
bars per model sorted best-first, a per-test grid (click a cell for the full
answer), previous runs and custom tests kept in the browser.
22 new strings in all 7 languages.
This commit is contained in:
infosave2007
Vendored
+305
@@ -0,0 +1,305 @@
|
||||
// Benchmark — quality + speed probes for pool models, run through the live
|
||||
// pipeline (pinned per model, streaming → TTFT and tok/s are real).
|
||||
// A redesign of the old standalone benchmark.html: same scoring ideas
|
||||
// (keyword checks with | synonyms, think-tag stripping, anti-loop detection,
|
||||
// pass/partial/fail), now embedded, localized and CDN-free.
|
||||
import { h, mount, toast, ms } from "../ui.js";
|
||||
import { t } from "../i18n.js";
|
||||
import { api, testStream } from "../api.js";
|
||||
|
||||
// ── built-in suite ──────────────────────────────────────────────────────────
|
||||
// Checks are language-neutral where possible (digits, identifiers, emails);
|
||||
// `check` entries may hold |-separated synonyms — one hit per entry counts.
|
||||
// `quick: true` marks the fast subset.
|
||||
const SUITE = [
|
||||
{ id: "math-mult", cat: "math", quick: true, max: 96, prompt: "What is 17 * 23? Answer with the number only.", check: ["391"] },
|
||||
{ id: "math-eq", cat: "math", max: 128, prompt: "Solve 2x + 6 = 14. Answer with the value of x only.", check: ["4"] },
|
||||
{ id: "logic-syll", cat: "logic", max: 96, prompt: "If all bloops are razzies and all razzies are lazzies, are all bloops lazzies? Answer yes or no.", check: ["yes|да|oui|ja|sí|si|evet|是"] },
|
||||
{ id: "count-letters", cat: "logic", quick: true, max: 64, prompt: "How many letters are in the word 'cat'? Answer with a digit.", check: ["3"] },
|
||||
{ id: "apples", cat: "logic", quick: true, max: 96, prompt: "John has 3 apples, buys 2 more, then eats 1. How many apples does he have? Answer with a digit.", check: ["4"] },
|
||||
{ id: "code-fact", cat: "code", max: 256, prompt: "Write a Python function factorial(n) that returns n!.", check: ["def", "factorial", "return"] },
|
||||
{ id: "qa-capital", cat: "qa", quick: true, max: 48, prompt: "What is the capital of France? One word.", check: ["paris|париж"] },
|
||||
{ id: "primes", cat: "qa", max: 96, prompt: "List the first five prime numbers, comma-separated.", check: ["2", "3", "5", "7", "11"] },
|
||||
{ id: "instr-ok", cat: "instruction", quick: true, max: 32, prompt: "Reply with exactly the word OK and nothing else.", check: ["ok"] },
|
||||
{ id: "extract-email", cat: "extraction", quick: true, max: 64, prompt: "Extract the email address from: 'Contact us at info@example.com today.' Reply with the address only.", check: ["info@example.com"] },
|
||||
{ id: "translate", cat: "translation", max: 64, prompt: "Translate to English: «Доброе утро». Reply with the translation only.", check: ["good morning"] },
|
||||
{ id: "format-json", cat: "instruction", max: 96, prompt: 'Return a JSON object with a single key "n" whose value is 7. JSON only.', check: ['"n"', "7"] },
|
||||
];
|
||||
|
||||
const LS_CUSTOM = "allaigate_bench_custom";
|
||||
const LS_HISTORY = "allaigate_bench_history";
|
||||
|
||||
function customTests() {
|
||||
try {
|
||||
return JSON.parse(localStorage.getItem(LS_CUSTOM) || "[]");
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
function saveCustom(list) {
|
||||
localStorage.setItem(LS_CUSTOM, JSON.stringify(list));
|
||||
}
|
||||
function history() {
|
||||
try {
|
||||
return JSON.parse(localStorage.getItem(LS_HISTORY) || "[]");
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
function pushHistory(run) {
|
||||
const l = history();
|
||||
l.unshift(run);
|
||||
localStorage.setItem(LS_HISTORY, JSON.stringify(l.slice(0, 12)));
|
||||
}
|
||||
|
||||
// ── evaluation (ported from the old benchmark, trimmed) ─────────────────────
|
||||
function stripThink(text) {
|
||||
return text.replace(/<(?:\|)?(?:think|thought)(?:\|)?>[\s\S]*?(?:<\/(?:\|)?(?:think|thought)(?:\|)?>|$)/gi, "").trim();
|
||||
}
|
||||
// tail loop: the same 12+ char chunk repeated 3+ times back to back
|
||||
function looksLooped(text) {
|
||||
const tail = text.slice(-360);
|
||||
for (let len = 12; len <= 60; len += 8) {
|
||||
const probe = tail.slice(-len);
|
||||
if (probe && tail.split(probe).length - 1 >= 3) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
function evaluate(raw, test) {
|
||||
let text = stripThink(raw);
|
||||
if (!text && raw) text = raw; // max_tokens cut inside <think> — judge the raw
|
||||
const resp = text.toLowerCase();
|
||||
if (!resp.trim()) return { status: "fail", score: 0, note: t("bm.emptyAnswer") };
|
||||
if (looksLooped(resp)) return { status: "fail", score: 0, note: t("bm.loopDetected") };
|
||||
let hit = 0;
|
||||
for (const cond of test.check) {
|
||||
const opts = cond.toLowerCase().split("|").map((s) => s.trim());
|
||||
if (opts.some((o) => o && resp.includes(o))) hit++;
|
||||
}
|
||||
const frac = hit / test.check.length;
|
||||
const status = frac >= 1 ? "pass" : frac >= 0.5 ? "partial" : "fail";
|
||||
return { status, score: frac, note: `${hit}/${test.check.length}` };
|
||||
}
|
||||
|
||||
export async function renderBenchmark() {
|
||||
const modelsData = await api.listModels();
|
||||
const ids = (modelsData.models || []).map((m) => m.id);
|
||||
const root = h("div");
|
||||
let running = false;
|
||||
let stopFlag = false;
|
||||
|
||||
const modelCbs = ids.map((id) => ({ id, cb: h("input", { type: "checkbox", checked: ids.length === 1 ? true : null }) }));
|
||||
const quickCb = h("input", { type: "checkbox", checked: true });
|
||||
const resultsHost = h("div", {});
|
||||
const progressHost = h("div", {});
|
||||
const historyHost = h("div", {});
|
||||
|
||||
function suite() {
|
||||
const base = SUITE.concat(customTests());
|
||||
return quickCb.checked ? base.filter((x) => x.quick || x.custom) : base;
|
||||
}
|
||||
|
||||
const runBtn = h("button", { class: "btn primary" }, "▶ " + t("bm.run"));
|
||||
const stopBtn = h("button", { class: "btn ghost", style: "display:none" }, "✕ " + t("import.cancel"));
|
||||
stopBtn.addEventListener("click", () => {
|
||||
stopFlag = true;
|
||||
stopBtn.disabled = true;
|
||||
});
|
||||
|
||||
runBtn.addEventListener("click", async () => {
|
||||
const models = modelCbs.filter((m) => m.cb.checked).map((m) => m.id);
|
||||
if (!models.length) {
|
||||
toast(t("bm.pickModel"), "warn");
|
||||
return;
|
||||
}
|
||||
running = true;
|
||||
stopFlag = false;
|
||||
runBtn.disabled = true;
|
||||
stopBtn.style.display = "";
|
||||
stopBtn.disabled = false;
|
||||
const tests = suite();
|
||||
const run = { ts: Date.now(), quick: quickCb.checked, models: {}, nTests: tests.length };
|
||||
|
||||
for (const model of models) {
|
||||
const rec = { results: [], score: 0, tokS: [], ttft: [], failed: false };
|
||||
run.models[model] = rec;
|
||||
for (let i = 0; i < tests.length; i++) {
|
||||
if (stopFlag) break;
|
||||
const test = tests[i];
|
||||
mount(
|
||||
progressHost,
|
||||
h("div", { class: "card" },
|
||||
h("div", { class: "flex" },
|
||||
h("span", { class: "spinner" }), " ",
|
||||
h("b", { class: "mono" }, model), " · ",
|
||||
t("bm.testOf", { i: i + 1, n: tests.length }), " · ",
|
||||
h("span", { class: "muted" }, test.id)),
|
||||
h("div", { class: "pbar", style: "margin-top:8px" },
|
||||
h("div", { class: "pbar-fill det", style: `width:${Math.round((i / tests.length) * 100)}%` })))
|
||||
);
|
||||
const started = performance.now();
|
||||
let text = "";
|
||||
try {
|
||||
const info = await testStream(
|
||||
{ model, messages: [{ role: "user", content: test.prompt }], temperature: 0, max_tokens: test.max || 128 },
|
||||
(d) => (text += d)
|
||||
);
|
||||
const latency = performance.now() - started;
|
||||
const ev = evaluate(text, test);
|
||||
const compl = info.usage?.completion_tokens;
|
||||
if (compl && latency > 0) {
|
||||
const gen = info.ttft_ms != null ? Math.max(1, latency - info.ttft_ms) : latency;
|
||||
rec.tokS.push(compl / (gen / 1000));
|
||||
}
|
||||
if (info.ttft_ms != null) rec.ttft.push(info.ttft_ms);
|
||||
rec.results.push({ id: test.id, cat: test.cat, ...ev, latency: Math.round(latency), answer: stripThink(text).slice(0, 300) });
|
||||
} catch (e) {
|
||||
rec.results.push({ id: test.id, cat: test.cat, status: "fail", score: 0, note: String(e.message || e).slice(0, 80), latency: 0, answer: "" });
|
||||
rec.failed = true;
|
||||
}
|
||||
}
|
||||
const n = rec.results.length || 1;
|
||||
rec.score = rec.results.reduce((a, r) => a + r.score, 0) / n;
|
||||
rec.avgTokS = rec.tokS.length ? rec.tokS.reduce((a, b) => a + b, 0) / rec.tokS.length : null;
|
||||
rec.avgTtft = rec.ttft.length ? rec.ttft.reduce((a, b) => a + b, 0) / rec.ttft.length : null;
|
||||
if (stopFlag) break;
|
||||
}
|
||||
|
||||
mount(progressHost);
|
||||
running = false;
|
||||
runBtn.disabled = false;
|
||||
stopBtn.style.display = "none";
|
||||
if (!stopFlag) pushHistory(run);
|
||||
renderResults(run);
|
||||
renderHistory();
|
||||
});
|
||||
|
||||
function statusIcon(s) {
|
||||
return s === "pass" ? "✅" : s === "partial" ? "◐" : "❌";
|
||||
}
|
||||
|
||||
function renderResults(run) {
|
||||
const models = Object.keys(run.models);
|
||||
if (!models.length) {
|
||||
mount(resultsHost);
|
||||
return;
|
||||
}
|
||||
// summary bars, best score first
|
||||
const sorted = models.slice().sort((a, b) => run.models[b].score - run.models[a].score);
|
||||
const summary = h("div", {},
|
||||
...sorted.map((m) => {
|
||||
const r = run.models[m];
|
||||
const pct = Math.round(r.score * 100);
|
||||
return h("div", { class: "bm-sum" },
|
||||
h("div", { class: "flex" },
|
||||
h("b", { class: "mono grow" }, m),
|
||||
h("span", { class: "badge " + (pct >= 80 ? "ok" : pct >= 50 ? "" : "bad") }, pct + "%"),
|
||||
r.avgTokS != null ? h("span", { class: "muted small", style: "margin-left:8px" }, r.avgTokS.toFixed(1) + " tok/s") : null,
|
||||
r.avgTtft != null ? h("span", { class: "muted small", style: "margin-left:8px" }, "TTFT " + ms(Math.round(r.avgTtft))) : null),
|
||||
h("div", { class: "sysbar" }, h("i", { class: pct < 50 ? "bad" : "", style: `width:${pct}%` })));
|
||||
}));
|
||||
|
||||
// detail table: rows = tests, one column per model
|
||||
const header = h("tr", {}, h("th", {}, t("bm.test")), ...models.map((m) => h("th", { class: "mono" }, m)));
|
||||
const rows = (run.models[models[0]].results || []).map((_, i) =>
|
||||
h("tr", {},
|
||||
h("td", {},
|
||||
h("div", {}, run.models[models[0]].results[i].id),
|
||||
h("div", { class: "muted small" }, run.models[models[0]].results[i].cat)),
|
||||
...models.map((m) => {
|
||||
const r = run.models[m].results[i];
|
||||
if (!r) return h("td", {}, "—");
|
||||
const cell = h("td", { class: "bm-cell", title: r.answer || r.note },
|
||||
statusIcon(r.status) + " ",
|
||||
h("span", { class: "muted small" }, r.note + (r.latency ? ` · ${(r.latency / 1000).toFixed(1)}s` : "")));
|
||||
cell.addEventListener("click", () => {
|
||||
mount(detail, h("div", { class: "card" },
|
||||
h("div", { class: "card-title" }, `${r.id} · ${m} · ${statusIcon(r.status)}`),
|
||||
h("pre", { class: "conv-log", style: "max-height:200px" }, r.answer || "—")));
|
||||
});
|
||||
return cell;
|
||||
})));
|
||||
const detail = h("div", {});
|
||||
mount(resultsHost,
|
||||
h("div", { class: "card" },
|
||||
h("div", { class: "card-head" },
|
||||
h("h3", {}, t("bm.results")),
|
||||
h("span", { class: "card-head sub" }, new Date(run.ts).toLocaleString() + (run.quick ? " · " + t("bm.quick") : ""))),
|
||||
summary,
|
||||
h("div", { class: "table-wrap", style: "margin-top:14px" }, h("table", {}, h("thead", {}, header), h("tbody", {}, ...rows))),
|
||||
h("div", { class: "muted small", style: "margin-top:6px" }, t("bm.cellHint"))),
|
||||
detail);
|
||||
}
|
||||
|
||||
function renderHistory() {
|
||||
const l = history();
|
||||
if (!l.length) {
|
||||
mount(historyHost);
|
||||
return;
|
||||
}
|
||||
mount(historyHost, h("div", { class: "card" },
|
||||
h("div", { class: "card-title" }, t("bm.history")),
|
||||
...l.map((run) => {
|
||||
const models = Object.keys(run.models);
|
||||
const best = models.slice().sort((a, b) => run.models[b].score - run.models[a].score)[0];
|
||||
const row = h("div", { class: "flex bench-row", style: "cursor:pointer" },
|
||||
h("span", { class: "muted small" }, new Date(run.ts).toLocaleString()),
|
||||
h("span", { class: "grow mono small" }, models.join(", ")),
|
||||
run.quick ? h("span", { class: "badge" }, t("bm.quick")) : null,
|
||||
best ? h("b", {}, Math.round(run.models[best].score * 100) + "%") : null);
|
||||
row.addEventListener("click", () => renderResults(run));
|
||||
return row;
|
||||
})));
|
||||
}
|
||||
|
||||
// ── custom tests (stored in the browser) ──
|
||||
function customCard() {
|
||||
const host = h("div", { class: "card" });
|
||||
function paint() {
|
||||
const list = customTests();
|
||||
const promptIn = h("input", { placeholder: t("bm.promptPh") });
|
||||
const checkIn = h("input", { placeholder: t("bm.checkPh") });
|
||||
const addBtn = h("button", { class: "btn ghost sm" }, "+ " + t("bm.addTest"));
|
||||
addBtn.addEventListener("click", () => {
|
||||
const prompt = promptIn.value.trim();
|
||||
const check = checkIn.value.split(",").map((s) => s.trim()).filter(Boolean);
|
||||
if (!prompt || !check.length) return;
|
||||
list.push({ id: "custom-" + (list.length + 1), cat: "custom", custom: true, max: 128, prompt, check });
|
||||
saveCustom(list);
|
||||
paint();
|
||||
});
|
||||
mount(host,
|
||||
h("div", { class: "card-title" }, t("bm.custom")),
|
||||
h("div", { class: "muted small", style: "margin-bottom:8px" }, t("bm.customHint")),
|
||||
...list.map((tst, i) =>
|
||||
h("div", { class: "flex bench-row" },
|
||||
h("span", { class: "grow small" }, tst.prompt.slice(0, 90)),
|
||||
h("span", { class: "muted small mono" }, tst.check.join(", ").slice(0, 40)),
|
||||
h("button", { class: "btn sm ghost danger", onClick: () => { list.splice(i, 1); saveCustom(list); paint(); } }, "×"))),
|
||||
h("div", { class: "flex", style: "gap:8px; margin-top:8px" }, promptIn, checkIn, addBtn));
|
||||
}
|
||||
paint();
|
||||
return host;
|
||||
}
|
||||
|
||||
mount(root,
|
||||
h("div", { class: "page-head" },
|
||||
h("h2", {}, t("bm.title")),
|
||||
h("p", {}, t("bm.subtitle"))),
|
||||
ids.length === 0 ? h("div", { class: "callout" }, t("routing.empty")) : null,
|
||||
h("div", { class: "card" },
|
||||
h("div", { class: "card-title" }, t("bm.models")),
|
||||
h("div", { class: "flex wrap", style: "gap:10px" },
|
||||
...modelCbs.map(({ id, cb }) => h("label", { class: "check" }, cb, h("span", { class: "mono" }, id)))),
|
||||
h("div", { class: "flex", style: "justify-content:space-between; margin-top:14px" },
|
||||
h("label", { class: "check", style: "margin:0", title: t("bm.quickHint") }, quickCb, "⚡ " + t("bm.quickMode")),
|
||||
h("div", { class: "flex", style: "gap:8px" }, stopBtn, runBtn)),
|
||||
h("div", { class: "muted small", style: "margin-top:6px" },
|
||||
t("bm.suiteInfo", { q: SUITE.filter((x) => x.quick).length, f: SUITE.length }))),
|
||||
progressHost,
|
||||
resultsHost,
|
||||
customCard(),
|
||||
historyHost);
|
||||
renderHistory();
|
||||
return root;
|
||||
}
|
||||
Reference in New Issue
Block a user