From ec92bdddb76bbf42fc1bd1063c445fa7d50bfe9b Mon Sep 17 00:00:00 2001 From: infosave2007 Date: Fri, 7 Aug 2026 19:04:41 +0300 Subject: [PATCH] =?UTF-8?q?feat(benchmark):=20smart-routing=20mode=20is=20?= =?UTF-8?q?the=20point=20=E2=80=94=20live=20results,=20per-model=20cost?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The benchmark now leads with the product's actual feature. Two explicit modes: 🧭 Smart routing (default) — every probe goes through cortiq-auto; each row shows the router's task label judged against the expected one (✓/✗ with "expected: …"), the complexity tier, the chosen model with what that request cost, and a task-rule badge when a routing rule fired. KPIs: answer quality, classification accuracy, TTFT, tok/s and money spent, plus a "requests went to" distribution with per-model cost. 🎯 Individual models — the old head-to-head comparison, now as the clearly separate option (model checkboxes appear only here). The suite covers all 8 taxonomy labels (added summarization, creative-writing and chitchat probes; expected labels annotated per test). Results are LIVE: the table grows a row after every test (with a small fade-in), the progress card shows the prompt being asked, and a 🏁 verdict toast lands at the end with the score and the total spend. 12 new strings in all 7 languages. --- web/css/styles.css | 2 + web/js/i18n/de.js | 14 +++ web/js/i18n/en.js | 14 +++ web/js/i18n/es.js | 14 +++ web/js/i18n/fr.js | 14 +++ web/js/i18n/ru.js | 14 +++ web/js/i18n/tr.js | 14 +++ web/js/i18n/zh.js | 14 +++ web/js/views/benchmark.js | 254 ++++++++++++++++++++++++++++---------- 9 files changed, 286 insertions(+), 68 deletions(-) diff --git a/web/css/styles.css b/web/css/styles.css index 93a74c6..c452001 100644 --- a/web/css/styles.css +++ b/web/css/styles.css @@ -586,6 +586,8 @@ body.modal-open { overflow: hidden; } room than the search grid before it starts scrolling. */ .featured-section .hf-grid { max-height: min(62vh, 760px); } /* Benchmark view */ +@keyframes bm-in { from { opacity: 0; transform: translateY(-3px); } to { opacity: 1; transform: none; } } +tbody tr:last-child { animation: bm-in .25s ease; } .bm-sum { margin-bottom: 10px; } .bm-cell { cursor: pointer; white-space: nowrap; } .bm-cell:hover { background: var(--hover); } diff --git a/web/js/i18n/de.js b/web/js/i18n/de.js index be9e935..5d7ba4b 100644 --- a/web/js/i18n/de.js +++ b/web/js/i18n/de.js @@ -587,4 +587,18 @@ export default { "bm.addTest": "Test hinzufügen", "bm.emptyAnswer": "leere Antwort", "bm.loopDetected": "Schleife erkannt", + + // benchmark: routing mode, cost, live + "bm.mode.routing": "Smartes Routing", + "bm.mode.models": "Einzelne Modelle", + "bm.mode.routingDesc": "Das Kernfeature des Produkts im Test: jede Probe läuft über cortiq-auto — Sie sehen, welche Aufgabe der Router erkannt hat, welches Modell er wählte, ob das den Erwartungen entspricht und was es gekostet hat.", + "bm.mode.modelsDesc": "Dieselbe Suite, an gewählte Modelle gepinnt — Qualität, Tempo und Kosten im direkten Vergleich, am Router vorbei.", + "bm.answersOk": "Antworten korrekt", + "bm.classAcc": "Aufgaben-Klassifikation", + "bm.modelDist": "Anfragen gingen an", + "bm.expectedLbl": "erwartet:", + "bm.answer": "Antwort", + "bm.liveRunning": "läuft…", + "bm.spent": "ausgegeben", + "bm.verdict": "🏁 Fertig: Antworten {p}, ausgegeben {s}", }; diff --git a/web/js/i18n/en.js b/web/js/i18n/en.js index 9d85153..a306bee 100644 --- a/web/js/i18n/en.js +++ b/web/js/i18n/en.js @@ -580,4 +580,18 @@ export default { "bm.addTest": "Add test", "bm.emptyAnswer": "empty answer", "bm.loopDetected": "loop detected", + + // benchmark: routing mode, cost, live + "bm.mode.routing": "Smart routing", + "bm.mode.models": "Individual models", + "bm.mode.routingDesc": "The product's core feature under test: every probe goes through cortiq-auto — you see which task the router detected, which model it picked, whether that matches expectations, and what it cost.", + "bm.mode.modelsDesc": "The same suite pinned to selected models — compare their quality, speed and cost head-to-head, bypassing the router.", + "bm.answersOk": "answers correct", + "bm.classAcc": "task classification", + "bm.modelDist": "Requests went to", + "bm.expectedLbl": "expected:", + "bm.answer": "Answer", + "bm.liveRunning": "running…", + "bm.spent": "spent", + "bm.verdict": "🏁 Done: answers {p}, spent {s}", }; diff --git a/web/js/i18n/es.js b/web/js/i18n/es.js index 86d4024..a97db37 100644 --- a/web/js/i18n/es.js +++ b/web/js/i18n/es.js @@ -587,4 +587,18 @@ export default { "bm.addTest": "Añadir prueba", "bm.emptyAnswer": "respuesta vacía", "bm.loopDetected": "bucle detectado", + + // benchmark: routing mode, cost, live + "bm.mode.routing": "Enrutado inteligente", + "bm.mode.models": "Modelos individuales", + "bm.mode.routingDesc": "La función clave del producto a prueba: cada sonda pasa por cortiq-auto — ves qué tarea detectó el router, qué modelo eligió, si coincide con lo esperado y cuánto costó.", + "bm.mode.modelsDesc": "La misma suite fijada a los modelos elegidos — compara su calidad, velocidad y coste directamente, sin pasar por el router.", + "bm.answersOk": "respuestas correctas", + "bm.classAcc": "clasificación de tareas", + "bm.modelDist": "Las peticiones fueron a", + "bm.expectedLbl": "esperado:", + "bm.answer": "Respuesta", + "bm.liveRunning": "en curso…", + "bm.spent": "gastado", + "bm.verdict": "🏁 Listo: respuestas {p}, gastado {s}", }; diff --git a/web/js/i18n/fr.js b/web/js/i18n/fr.js index 70405b2..b3fa03e 100644 --- a/web/js/i18n/fr.js +++ b/web/js/i18n/fr.js @@ -587,4 +587,18 @@ export default { "bm.addTest": "Ajouter un test", "bm.emptyAnswer": "réponse vide", "bm.loopDetected": "boucle détectée", + + // benchmark: routing mode, cost, live + "bm.mode.routing": "Routage intelligent", + "bm.mode.models": "Modèles individuels", + "bm.mode.routingDesc": "La fonctionnalité clé du produit à l'épreuve : chaque sonde passe par cortiq-auto — vous voyez quelle tâche le routeur a détectée, quel modèle il a choisi, si cela correspond aux attentes et combien cela a coûté.", + "bm.mode.modelsDesc": "La même suite épinglée aux modèles choisis — comparez leur qualité, vitesse et coût en direct, sans passer par le routeur.", + "bm.answersOk": "réponses correctes", + "bm.classAcc": "classification des tâches", + "bm.modelDist": "Requêtes envoyées vers", + "bm.expectedLbl": "attendu :", + "bm.answer": "Réponse", + "bm.liveRunning": "en cours…", + "bm.spent": "dépensé", + "bm.verdict": "🏁 Terminé : réponses {p}, dépensé {s}", }; diff --git a/web/js/i18n/ru.js b/web/js/i18n/ru.js index a3184fe..867be0e 100644 --- a/web/js/i18n/ru.js +++ b/web/js/i18n/ru.js @@ -579,4 +579,18 @@ export default { "bm.addTest": "Добавить тест", "bm.emptyAnswer": "пустой ответ", "bm.loopDetected": "зацикливание", + + // benchmark: routing mode, cost, live + "bm.mode.routing": "Умный роутинг", + "bm.mode.models": "Отдельные модели", + "bm.mode.routingDesc": "Проверка главной фишки продукта: каждый тест идёт через cortiq-auto — видно, какую задачу распознал роутер, какую модель выбрал, совпало ли это с ожиданием и сколько это стоило.", + "bm.mode.modelsDesc": "Тот же набор тестов, но закреплённый за выбранными моделями — сравнение их качества, скорости и стоимости напрямую, мимо роутера.", + "bm.answersOk": "ответы верны", + "bm.classAcc": "классификация задач", + "bm.modelDist": "Запросы ушли в", + "bm.expectedLbl": "ожидалось:", + "bm.answer": "Ответ", + "bm.liveRunning": "идёт…", + "bm.spent": "потрачено", + "bm.verdict": "🏁 Готово: ответы {p}, потрачено {s}", }; diff --git a/web/js/i18n/tr.js b/web/js/i18n/tr.js index ab1db12..91061e6 100644 --- a/web/js/i18n/tr.js +++ b/web/js/i18n/tr.js @@ -587,4 +587,18 @@ export default { "bm.addTest": "Test ekle", "bm.emptyAnswer": "boş yanıt", "bm.loopDetected": "döngü tespit edildi", + + // benchmark: routing mode, cost, live + "bm.mode.routing": "Akıllı yönlendirme", + "bm.mode.models": "Tek tek modeller", + "bm.mode.routingDesc": "Ürünün temel özelliği testte: her prob cortiq-auto üzerinden gider — yönlendiricinin hangi görevi algıladığını, hangi modeli seçtiğini, bunun beklentiyle örtüşüp örtüşmediğini ve maliyetini görürsünüz.", + "bm.mode.modelsDesc": "Aynı küme seçilen modellere sabitlenir — yönlendiriciyi atlayarak kalite, hız ve maliyeti doğrudan karşılaştırın.", + "bm.answersOk": "yanıtlar doğru", + "bm.classAcc": "görev sınıflandırması", + "bm.modelDist": "İstekler şuraya gitti", + "bm.expectedLbl": "beklenen:", + "bm.answer": "Yanıt", + "bm.liveRunning": "sürüyor…", + "bm.spent": "harcanan", + "bm.verdict": "🏁 Bitti: yanıtlar {p}, harcanan {s}", }; diff --git a/web/js/i18n/zh.js b/web/js/i18n/zh.js index 9082b91..2b266a0 100644 --- a/web/js/i18n/zh.js +++ b/web/js/i18n/zh.js @@ -587,4 +587,18 @@ export default { "bm.addTest": "添加测试", "bm.emptyAnswer": "回答为空", "bm.loopDetected": "检测到循环", + + // benchmark: routing mode, cost, live + "bm.mode.routing": "智能路由", + "bm.mode.models": "单独模型", + "bm.mode.routingDesc": "测试产品核心能力：每个探测都经过 cortiq-auto — 可以看到路由器识别的任务、选中的模型、是否符合预期，以及花费了多少。", + "bm.mode.modelsDesc": "同一套测试固定在所选模型上 — 绕过路由器，直接对比它们的质量、速度与成本。", + "bm.answersOk": "回答正确", + "bm.classAcc": "任务分类", + "bm.modelDist": "请求流向", + "bm.expectedLbl": "预期：", + "bm.answer": "回答", + "bm.liveRunning": "进行中……", + "bm.spent": "花费", + "bm.verdict": "🏁 完成：回答 {p}，花费 {s}", }; diff --git a/web/js/views/benchmark.js b/web/js/views/benchmark.js index 156dfab..01bada9 100644 --- a/web/js/views/benchmark.js +++ b/web/js/views/benchmark.js @@ -1,9 +1,13 @@ -// Benchmark — quality + speed probes for pool models, run through the live -// pipeline (pinned per model, streaming → TTFT and tok/s are real). +// Benchmark — two modes over the live pipeline: +// 🧭 Smart routing (default): the product's core promise under test — every +// probe goes through cortiq-auto and the router's task/tier decision and +// chosen model are judged alongside the answer itself. +// 🎯 Individual models: the same suite pinned to selected models — compare +// quality and speed head-to-head. // A redesign of the old standalone benchmark.html: same scoring ideas // (keyword checks with | synonyms, think-tag stripping, anti-loop detection, // pass/partial/fail), now embedded, localized and CDN-free. -import { h, mount, toast, ms } from "../ui.js"; +import { h, mount, toast, ms, money } from "../ui.js"; import { t } from "../i18n.js"; import { api, testStream } from "../api.js"; @@ -11,19 +15,24 @@ import { api, testStream } from "../api.js"; // Checks are language-neutral where possible (digits, identifiers, emails); // `check` entries may hold |-separated synonyms — one hit per entry counts. // `quick: true` marks the fast subset. +// `label`: router task labels accepted as a correct classification (null = not +// judged — the prompt has no natural label in the taxonomy). const SUITE = [ - { id: "math-mult", cat: "math", quick: true, max: 96, prompt: "What is 17 * 23? Answer with the number only.", check: ["391"] }, - { id: "math-eq", cat: "math", max: 128, prompt: "Solve 2x + 6 = 14. Answer with the value of x only.", check: ["4"] }, - { id: "logic-syll", cat: "logic", max: 96, prompt: "If all bloops are razzies and all razzies are lazzies, are all bloops lazzies? Answer yes or no.", check: ["yes|да|oui|ja|sí|si|evet|是"] }, - { id: "count-letters", cat: "logic", quick: true, max: 64, prompt: "How many letters are in the word 'cat'? Answer with a digit.", check: ["3"] }, - { id: "apples", cat: "logic", quick: true, max: 96, prompt: "John has 3 apples, buys 2 more, then eats 1. How many apples does he have? Answer with a digit.", check: ["4"] }, - { id: "code-fact", cat: "code", max: 256, prompt: "Write a Python function factorial(n) that returns n!.", check: ["def", "factorial", "return"] }, - { id: "qa-capital", cat: "qa", quick: true, max: 48, prompt: "What is the capital of France? One word.", check: ["paris|париж"] }, - { id: "primes", cat: "qa", max: 96, prompt: "List the first five prime numbers, comma-separated.", check: ["2", "3", "5", "7", "11"] }, - { id: "instr-ok", cat: "instruction", quick: true, max: 32, prompt: "Reply with exactly the word OK and nothing else.", check: ["ok"] }, - { id: "extract-email", cat: "extraction", quick: true, max: 64, prompt: "Extract the email address from: 'Contact us at info@example.com today.' Reply with the address only.", check: ["info@example.com"] }, - { id: "translate", cat: "translation", max: 64, prompt: "Translate to English: «Доброе утро». Reply with the translation only.", check: ["good morning"] }, - { id: "format-json", cat: "instruction", max: 96, prompt: 'Return a JSON object with a single key "n" whose value is 7. JSON only.', check: ['"n"', "7"] }, + { id: "math-mult", cat: "math", label: ["math"], quick: true, max: 96, prompt: "What is 17 * 23? Answer with the number only.", check: ["391"] }, + { id: "math-eq", cat: "math", label: ["math"], max: 128, prompt: "Solve 2x + 6 = 14. Answer with the value of x only.", check: ["4"] }, + { id: "logic-syll", cat: "logic", label: ["qa", "math"], max: 96, prompt: "If all bloops are razzies and all razzies are lazzies, are all bloops lazzies? Answer yes or no.", check: ["yes|да|oui|ja|sí|si|evet|是"] }, + { id: "count-letters", cat: "logic", label: ["qa", "math"], quick: true, max: 64, prompt: "How many letters are in the word 'cat'? Answer with a digit.", check: ["3"] }, + { id: "apples", cat: "logic", label: ["math", "qa"], quick: true, max: 96, prompt: "John has 3 apples, buys 2 more, then eats 1. How many apples does he have? Answer with a digit.", check: ["4"] }, + { id: "code-fact", cat: "code", label: ["code"], max: 256, prompt: "Write a Python function factorial(n) that returns n!.", check: ["def", "factorial", "return"] }, + { id: "qa-capital", cat: "qa", label: ["qa"], quick: true, max: 48, prompt: "What is the capital of France? One word.", check: ["paris|париж"] }, + { id: "primes", cat: "qa", label: ["math", "qa"], max: 96, prompt: "List the first five prime numbers, comma-separated.", check: ["2", "3", "5", "7", "11"] }, + { id: "instr-ok", cat: "instruction", label: null, quick: true, max: 32, prompt: "Reply with exactly the word OK and nothing else.", check: ["ok"] }, + { id: "extract-email", cat: "extraction", label: ["extraction"], quick: true, max: 64, prompt: "Extract the email address from: 'Contact us at info@example.com today.' Reply with the address only.", check: ["info@example.com"] }, + { id: "translate", cat: "translation", label: ["translation"], max: 64, prompt: "Translate to English: «Доброе утро». Reply with the translation only.", check: ["good morning"] }, + { id: "format-json", cat: "instruction", label: null, max: 96, prompt: 'Return a JSON object with a single key "n" whose value is 7. JSON only.', check: ['"n"', "7"] }, + { id: "summarize", cat: "summarization", label: ["summarization"], max: 96, prompt: "Summarize in one short sentence: 'The cat sat on the warm mat by the window. After a while it curled up and fell asleep in the sun.'", check: ["cat|кот|кошк"] }, + { id: "creative", cat: "creative-writing", label: ["creative-writing"], max: 96, prompt: "Write a two-line poem about the sea.", check: ["sea|море|ocean"] }, + { id: "chitchat", cat: "chitchat", label: ["chitchat"], quick: true, max: 48, prompt: "Hi! How are you today?", check: [] }, ]; const LS_CUSTOM = "allaigate_bench_custom"; @@ -71,6 +80,7 @@ function evaluate(raw, test) { const resp = text.toLowerCase(); if (!resp.trim()) return { status: "fail", score: 0, note: t("bm.emptyAnswer") }; if (looksLooped(resp)) return { status: "fail", score: 0, note: t("bm.loopDetected") }; + if (!test.check.length) return { status: "pass", score: 1, note: "✓" }; let hit = 0; for (const cond of test.check) { const opts = cond.toLowerCase().split("|").map((s) => s.trim()); @@ -88,12 +98,27 @@ export async function renderBenchmark() { let running = false; let stopFlag = false; + let mode = "routing"; // routing (cortiq-auto, the product's feature) | models const modelCbs = ids.map((id) => ({ id, cb: h("input", { type: "checkbox", checked: ids.length === 1 ? true : null }) })); const quickCb = h("input", { type: "checkbox", checked: true }); const resultsHost = h("div", {}); const progressHost = h("div", {}); const historyHost = h("div", {}); + const modeDesc = h("p", { class: "muted", style: "margin:8px 0 0" }, t("bm.mode.routingDesc")); + const modelsPick = h("div", { class: "flex wrap", style: "gap:10px; margin-top:12px; display:none" }, + ...modelCbs.map(({ id, cb }) => h("label", { class: "check" }, cb, h("span", { class: "mono" }, id)))); + const modeSeg = h("div", { class: "seg" }, + h("button", { class: "active" }, "🧭 " + t("bm.mode.routing")), + h("button", {}, "🎯 " + t("bm.mode.models"))); + modeSeg.querySelectorAll("button").forEach((b, i) => + b.addEventListener("click", () => { + mode = i === 0 ? "routing" : "models"; + modeSeg.querySelectorAll("button").forEach((x, j) => x.classList.toggle("active", j === i)); + modelsPick.style.display = mode === "models" ? "" : "none"; + modeDesc.textContent = t(mode === "routing" ? "bm.mode.routingDesc" : "bm.mode.modelsDesc"); + })); + function suite() { const base = SUITE.concat(customTests()); return quickCb.checked ? base.filter((x) => x.quick || x.custom) : base; @@ -107,7 +132,7 @@ export async function renderBenchmark() { }); runBtn.addEventListener("click", async () => { - const models = modelCbs.filter((m) => m.cb.checked).map((m) => m.id); + const models = mode === "routing" ? ["cortiq-auto"] : modelCbs.filter((m) => m.cb.checked).map((m) => m.id); if (!models.length) { toast(t("bm.pickModel"), "warn"); return; @@ -118,7 +143,7 @@ export async function renderBenchmark() { stopBtn.style.display = ""; stopBtn.disabled = false; const tests = suite(); - const run = { ts: Date.now(), quick: quickCb.checked, models: {}, nTests: tests.length }; + const run = { ts: Date.now(), mode, quick: quickCb.checked, models: {}, nTests: tests.length }; for (const model of models) { const rec = { results: [], score: 0, tokS: [], ttft: [], failed: false }; @@ -131,9 +156,10 @@ export async function renderBenchmark() { h("div", { class: "card" }, h("div", { class: "flex" }, h("span", { class: "spinner" }), " ", - h("b", { class: "mono" }, model), " · ", + h("b", { class: "mono" }, mode === "routing" ? t("bm.mode.routing") : model), " · ", t("bm.testOf", { i: i + 1, n: tests.length }), " · ", h("span", { class: "muted" }, test.id)), + h("div", { class: "muted small", style: "margin-top:6px; font-style:italic" }, "«" + test.prompt.slice(0, 110) + (test.prompt.length > 110 ? "…»" : "»")), h("div", { class: "pbar", style: "margin-top:8px" }, h("div", { class: "pbar-fill det", style: `width:${Math.round((i / tests.length) * 100)}%` }))) ); @@ -152,16 +178,22 @@ export async function renderBenchmark() { rec.tokS.push(compl / (gen / 1000)); } if (info.ttft_ms != null) rec.ttft.push(info.ttft_ms); - rec.results.push({ id: test.id, cat: test.cat, ...ev, latency: Math.round(latency), answer: stripThink(text).slice(0, 300) }); + const row = { id: test.id, cat: test.cat, ...ev, latency: Math.round(latency), cost: info.cost_usd || 0, answer: stripThink(text).slice(0, 300) }; + if (mode === "routing") { + row.task = info.task_label || "—"; + row.tier = info.tier || "—"; + row.chosen = info.selected_model || "—"; + row.source = info.route_source || "—"; + row.expected = test.label; // null = classification not judged + row.taskOk = test.label ? test.label.includes(info.task_label) : null; + } + rec.results.push(row); } catch (e) { rec.results.push({ id: test.id, cat: test.cat, status: "fail", score: 0, note: String(e.message || e).slice(0, 80), latency: 0, answer: "" }); rec.failed = true; } + renderResults(run, true); // live: строка появляется сразу после теста } - const n = rec.results.length || 1; - rec.score = rec.results.reduce((a, r) => a + r.score, 0) / n; - rec.avgTokS = rec.tokS.length ? rec.tokS.reduce((a, b) => a + b, 0) / rec.tokS.length : null; - rec.avgTtft = rec.ttft.length ? rec.ttft.reduce((a, b) => a + b, 0) / rec.ttft.length : null; if (stopFlag) break; } @@ -169,7 +201,18 @@ export async function renderBenchmark() { running = false; runBtn.disabled = false; stopBtn.style.display = "none"; - if (!stopFlag) pushHistory(run); + if (!stopFlag) { + pushHistory(run); + // финальный аккорд: короткий вердикт с процентами и деньгами + const first = run.models[Object.keys(run.models)[0]]; + if (first) { + const st = stats(first); + const judged = first.results.filter((r) => r.taskOk != null); + const cls = judged.length ? ` · ${judged.filter((r) => r.taskOk).length}/${judged.length}` : ""; + const total = Object.values(run.models).reduce((a, rec) => a + stats(rec).cost, 0); + toast(t("bm.verdict", { p: Math.round(st.score * 100) + "%" + cls, s: money(total) }), "ok", 6000); + } + } renderResults(run); renderHistory(); }); @@ -178,53 +221,123 @@ export async function renderBenchmark() { return s === "pass" ? "✅" : s === "partial" ? "◐" : "❌"; } - function renderResults(run) { + function stats(rec) { + const n = rec.results.length || 1; + return { + score: rec.results.reduce((a, r) => a + r.score, 0) / n, + tokS: rec.tokS.length ? rec.tokS.reduce((a, b) => a + b, 0) / rec.tokS.length : null, + ttft: rec.ttft.length ? rec.ttft.reduce((a, b) => a + b, 0) / rec.ttft.length : null, + cost: rec.results.reduce((a, r) => a + (r.cost || 0), 0), + }; + } + + function renderResults(run, live) { const models = Object.keys(run.models); if (!models.length) { mount(resultsHost); return; } - // summary bars, best score first - const sorted = models.slice().sort((a, b) => run.models[b].score - run.models[a].score); - const summary = h("div", {}, - ...sorted.map((m) => { - const r = run.models[m]; - const pct = Math.round(r.score * 100); - return h("div", { class: "bm-sum" }, - h("div", { class: "flex" }, - h("b", { class: "mono grow" }, m), - h("span", { class: "badge " + (pct >= 80 ? "ok" : pct >= 50 ? "" : "bad") }, pct + "%"), - r.avgTokS != null ? h("span", { class: "muted small", style: "margin-left:8px" }, r.avgTokS.toFixed(1) + " tok/s") : null, - r.avgTtft != null ? h("span", { class: "muted small", style: "margin-left:8px" }, "TTFT " + ms(Math.round(r.avgTtft))) : null), - h("div", { class: "sysbar" }, h("i", { class: pct < 50 ? "bad" : "", style: `width:${pct}%` }))); - })); - - // detail table: rows = tests, one column per model - const header = h("tr", {}, h("th", {}, t("bm.test")), ...models.map((m) => h("th", { class: "mono" }, m))); - const rows = (run.models[models[0]].results || []).map((_, i) => - h("tr", {}, - h("td", {}, - h("div", {}, run.models[models[0]].results[i].id), - h("div", { class: "muted small" }, run.models[models[0]].results[i].cat)), - ...models.map((m) => { - const r = run.models[m].results[i]; - if (!r) return h("td", {}, "—"); - const cell = h("td", { class: "bm-cell", title: r.answer || r.note }, - statusIcon(r.status) + " ", - h("span", { class: "muted small" }, r.note + (r.latency ? ` · ${(r.latency / 1000).toFixed(1)}s` : ""))); - cell.addEventListener("click", () => { - mount(detail, h("div", { class: "card" }, - h("div", { class: "card-title" }, `${r.id} · ${m} · ${statusIcon(r.status)}`), - h("pre", { class: "conv-log", style: "max-height:200px" }, r.answer || "—"))); - }); - return cell; - }))); + const routing = run.mode === "routing"; const detail = h("div", {}); + + let summary; + if (routing) { + const rec = run.models[models[0]]; + const st = stats(rec); + const judged = rec.results.filter((r) => r.taskOk != null); + const okCount = judged.filter((r) => r.taskOk).length; + // какие модели реально выбирал роутер + const dist = {}; + for (const r of rec.results) { + if (!r.chosen) continue; + dist[r.chosen] = dist[r.chosen] || { n: 0, cost: 0 }; + dist[r.chosen].n++; + dist[r.chosen].cost += r.cost || 0; + } + const pct = Math.round(st.score * 100); + summary = h("div", {}, + h("div", { class: "kpis", style: "grid-template-columns: repeat(auto-fit, minmax(150px, 1fr))" }, + h("div", { class: "kpi" }, h("div", { class: "big" }, pct + "%"), h("div", { class: "lbl" }, t("bm.answersOk"))), + h("div", { class: "kpi" }, h("div", { class: "big" }, judged.length ? `${okCount}/${judged.length}` : "—"), h("div", { class: "lbl" }, t("bm.classAcc"))), + h("div", { class: "kpi" }, h("div", { class: "big" }, st.ttft != null ? ms(Math.round(st.ttft)) : "—"), h("div", { class: "lbl" }, t("pg.ttft"))), + h("div", { class: "kpi" }, h("div", { class: "big" }, st.tokS != null ? st.tokS.toFixed(1) : "—"), h("div", { class: "lbl" }, "tok/s")), + h("div", { class: "kpi" }, h("div", { class: "big" }, money(st.cost)), h("div", { class: "lbl" }, t("bm.spent")))), + h("div", { class: "flex wrap", style: "gap:6px; margin-top:4px" }, + h("span", { class: "muted small" }, t("bm.modelDist") + ":"), + ...Object.entries(dist).map(([m, d]) => + h("span", { class: "badge" }, `${m} × ${d.n}` + (d.cost > 0 ? ` · ${money(d.cost)}` : ""))))); + } else { + const sorted = models.slice().sort((a, b) => stats(run.models[b]).score - stats(run.models[a]).score); + summary = h("div", {}, + ...sorted.map((m) => { + const st = stats(run.models[m]); + const pct = Math.round(st.score * 100); + return h("div", { class: "bm-sum" }, + h("div", { class: "flex" }, + h("b", { class: "mono grow" }, m), + h("span", { class: "badge " + (pct >= 80 ? "ok" : pct >= 50 ? "" : "bad") }, pct + "%"), + st.tokS != null ? h("span", { class: "muted small", style: "margin-left:8px" }, st.tokS.toFixed(1) + " tok/s") : null, + st.ttft != null ? h("span", { class: "muted small", style: "margin-left:8px" }, "TTFT " + ms(Math.round(st.ttft))) : null, + st.cost > 0 ? h("span", { class: "muted small", style: "margin-left:8px" }, t("bm.spent") + " " + money(st.cost)) : null), + h("div", { class: "sysbar" }, h("i", { class: pct < 50 ? "bad" : "", style: `width:${pct}%` }))); + })); + } + + function answerCell(r, m) { + const cell = h("td", { class: "bm-cell", title: r.answer || r.note }, + statusIcon(r.status) + " ", + h("span", { class: "muted small" }, r.note + (r.latency ? ` · ${(r.latency / 1000).toFixed(1)}s` : ""))); + cell.addEventListener("click", () => { + mount(detail, h("div", { class: "card" }, + h("div", { class: "card-title" }, `${r.id} · ${m} · ${statusIcon(r.status)}`), + h("pre", { class: "conv-log", style: "max-height:200px" }, r.answer || "—"))); + }); + return cell; + } + + let header, rows; + if (routing) { + const rec = run.models[models[0]]; + header = h("tr", {}, + h("th", {}, t("bm.test")), + h("th", {}, t("pg.task")), + h("th", {}, t("pg.tier")), + h("th", {}, t("pg.model")), + h("th", {}, t("bm.answer"))); + rows = rec.results.map((r) => + h("tr", {}, + h("td", {}, h("div", {}, r.id), h("div", { class: "muted small" }, r.cat)), + h("td", {}, + r.taskOk == null + ? h("span", { class: "muted" }, r.task || "—") + : h("span", { class: r.taskOk ? "" : "err" }, + (r.taskOk ? "✓ " : "✗ ") + r.task, + r.taskOk ? "" : h("div", { class: "muted small" }, t("bm.expectedLbl") + " " + (r.expected || []).join("/")))), + h("td", {}, r.tier ? h("span", { class: "badge tier-" + r.tier }, r.tier) : "—"), + h("td", {}, + h("span", { class: "mono small" }, r.chosen || "—"), + r.cost > 0 ? h("div", { class: "muted small" }, money(r.cost)) : null, + r.source && r.source.includes("task-rule") ? h("div", { class: "badge", style: "margin-top:2px" }, "task-rule") : null), + answerCell(r, r.chosen || "cortiq-auto"))); + } else { + header = h("tr", {}, h("th", {}, t("bm.test")), ...models.map((m) => h("th", { class: "mono" }, m))); + rows = (run.models[models[0]].results || []).map((_, i) => + h("tr", {}, + h("td", {}, + h("div", {}, run.models[models[0]].results[i].id), + h("div", { class: "muted small" }, run.models[models[0]].results[i].cat)), + ...models.map((m) => { + const r = run.models[m].results[i]; + return r ? answerCell(r, m) : h("td", {}, "—"); + }))); + } + mount(resultsHost, h("div", { class: "card" }, h("div", { class: "card-head" }, - h("h3", {}, t("bm.results")), - h("span", { class: "card-head sub" }, new Date(run.ts).toLocaleString() + (run.quick ? " · " + t("bm.quick") : ""))), + h("h3", {}, (routing ? "🧭 " : "🎯 ") + t("bm.results")), + h("span", { class: "card-head sub" }, + (live ? t("bm.liveRunning") + " · " : "") + new Date(run.ts).toLocaleString() + (run.quick ? " · " + t("bm.quick") : ""))), summary, h("div", { class: "table-wrap", style: "margin-top:14px" }, h("table", {}, h("thead", {}, header), h("tbody", {}, ...rows))), h("div", { class: "muted small", style: "margin-top:6px" }, t("bm.cellHint"))), @@ -241,12 +354,17 @@ export async function renderBenchmark() { h("div", { class: "card-title" }, t("bm.history")), ...l.map((run) => { const models = Object.keys(run.models); - const best = models.slice().sort((a, b) => run.models[b].score - run.models[a].score)[0]; + const scoreOf = (m) => { + const rs = run.models[m].results || []; + return rs.length ? rs.reduce((a, r) => a + r.score, 0) / rs.length : 0; + }; + const best = models.slice().sort((a, b) => scoreOf(b) - scoreOf(a))[0]; const row = h("div", { class: "flex bench-row", style: "cursor:pointer" }, + h("span", {}, run.mode === "routing" ? "🧭" : "🎯"), h("span", { class: "muted small" }, new Date(run.ts).toLocaleString()), - h("span", { class: "grow mono small" }, models.join(", ")), + h("span", { class: "grow mono small" }, run.mode === "routing" ? t("bm.mode.routing") : models.join(", ")), run.quick ? h("span", { class: "badge" }, t("bm.quick")) : null, - best ? h("b", {}, Math.round(run.models[best].score * 100) + "%") : null); + best ? h("b", {}, Math.round(scoreOf(best) * 100) + "%") : null); row.addEventListener("click", () => renderResults(run)); return row; }))); @@ -288,9 +406,9 @@ export async function renderBenchmark() { h("p", {}, t("bm.subtitle"))), ids.length === 0 ? h("div", { class: "callout" }, t("routing.empty")) : null, h("div", { class: "card" }, - h("div", { class: "card-title" }, t("bm.models")), - h("div", { class: "flex wrap", style: "gap:10px" }, - ...modelCbs.map(({ id, cb }) => h("label", { class: "check" }, cb, h("span", { class: "mono" }, id)))), + modeSeg, + modeDesc, + modelsPick, h("div", { class: "flex", style: "justify-content:space-between; margin-top:14px" }, h("label", { class: "check", style: "margin:0", title: t("bm.quickHint") }, quickCb, "⚡ " + t("bm.quickMode")), h("div", { class: "flex", style: "gap:8px" }, stopBtn, runBtn)),