From 7479cc51c2f3b95a9c1944fad4438bb2c999a2e8 Mon Sep 17 00:00:00 2001 From: infosave2007 Date: Fri, 7 Aug 2026 20:15:46 +0300 Subject: [PATCH] =?UTF-8?q?feat(benchmark):=20hard=20tests=20that=20light?= =?UTF-8?q?=20up=20the=20high=20tier=20=E2=80=94=20judged=20on=20routing,?= =?UTF-8?q?=20not=20essays?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two "hard" probes (full suite only): a compiler-backend design and a security protocol design, each dense enough for the router's complexity model to rate them high. They are judged on the ROUTER's decision (judge: "route") — pass means tier=high was reached and the request went to the big model; the answer itself is not scored (open-ended essays aren't keyword-judgeable, and reasoning models happily burn 3000+ thinking tokens on them — measured). With max_tokens kept tiny the probes cost cents even on the priciest model, while the cost column tells the whole story: $0.0148 for the high-tier request next to $0.00005 for the easy ones. Each run picks a random variant of the prompt: the router deliberately scores familiar text as less novel, so a fixed wording would decay to medium over repeated runs. In pinned-models mode route-judged tests are skipped. Also: creative-writing budget 96→512 (deepseek was spending the whole cap thinking and returning an empty poem), and when a model does burn its entire budget inside the table now says so ("budget spent on reasoning") instead of the misleading "empty answer". Server-side (cortiq-router prod, both replicas rolling-restarted): the high band started at 0.66 while the complexity model tops out ~0.65 on the hardest single prompts — high was unreachable. Band boundary moved to 0.50 based on measured distribution (easy 0.25-0.36, medium 0.37-0.49, hard 0.50-0.65); config backed up as config.prod.toml.bak-tiers. --- web/js/i18n/de.js | 1 + web/js/i18n/en.js | 1 + web/js/i18n/es.js | 1 + web/js/i18n/fr.js | 1 + web/js/i18n/ru.js | 1 + web/js/i18n/tr.js | 1 + web/js/i18n/zh.js | 1 + web/js/views/benchmark.js | 36 +++++++++++++++++++++++++++++++----- 8 files changed, 38 insertions(+), 5 deletions(-) diff --git a/web/js/i18n/de.js b/web/js/i18n/de.js index 0112cd4..32edf8c 100644 --- a/web/js/i18n/de.js +++ b/web/js/i18n/de.js @@ -618,4 +618,5 @@ export default { "keys.h.st.notFound": "Auf der Gateway-Maschine nicht gefunden", "bm.degraded": "Router degradiert", "bm.routerDegraded": "Der Router hat nicht rechtzeitig geantwortet — die Anfrage nutzte die Fallback-Kette; die Klassifikation wird nicht gewertet.", + "bm.cutByCap": "Budget ging fürs Denken drauf — keine Antwort mehr", }; diff --git a/web/js/i18n/en.js b/web/js/i18n/en.js index 0ef2998..64476e6 100644 --- a/web/js/i18n/en.js +++ b/web/js/i18n/en.js @@ -611,4 +611,5 @@ export default { "keys.h.st.notFound": "Not found on the gateway machine", "bm.degraded": "router degraded", "bm.routerDegraded": "The router did not answer in time — the request used the fallback chain; classification is not judged.", + "bm.cutByCap": "budget spent on reasoning — answer never came", }; diff --git a/web/js/i18n/es.js b/web/js/i18n/es.js index 969a1bd..df380d5 100644 --- a/web/js/i18n/es.js +++ b/web/js/i18n/es.js @@ -618,4 +618,5 @@ export default { "keys.h.st.notFound": "No encontrado en la máquina de la pasarela", "bm.degraded": "router degradado", "bm.routerDegraded": "El router no respondió a tiempo — la petición usó la cadena de reserva; la clasificación no se cuenta.", + "bm.cutByCap": "presupuesto gastado en razonar — sin respuesta final", }; diff --git a/web/js/i18n/fr.js b/web/js/i18n/fr.js index ca5b61f..2b09e49 100644 --- a/web/js/i18n/fr.js +++ b/web/js/i18n/fr.js @@ -618,4 +618,5 @@ export default { "keys.h.st.notFound": "Introuvable sur la machine de la passerelle", "bm.degraded": "routeur dégradé", "bm.routerDegraded": "Le routeur n'a pas répondu à temps — la requête a suivi la chaîne de secours ; la classification n'est pas comptée.", + "bm.cutByCap": "budget épuisé en raisonnement — pas de réponse", }; diff --git a/web/js/i18n/ru.js b/web/js/i18n/ru.js index 2930096..df0de9e 100644 --- a/web/js/i18n/ru.js +++ b/web/js/i18n/ru.js @@ -610,4 +610,5 @@ export default { "keys.h.st.notFound": "Не найден на машине шлюза", "bm.degraded": "роутер деградировал", "bm.routerDegraded": "Роутер не успел ответить — запрос ушёл по резервной цепочке; классификация не засчитывается.", + "bm.cutByCap": "бюджет ушёл на размышления — ответ не успел", }; diff --git a/web/js/i18n/tr.js b/web/js/i18n/tr.js index 4aeedfc..65cd0f3 100644 --- a/web/js/i18n/tr.js +++ b/web/js/i18n/tr.js @@ -618,4 +618,5 @@ export default { "keys.h.st.notFound": "Ağ geçidi makinesinde bulunamadı", "bm.degraded": "yönlendirici yedeğe düştü", "bm.routerDegraded": "Yönlendirici zamanında yanıt vermedi — istek yedek zincire gitti; sınıflandırma sayılmaz.", + "bm.cutByCap": "bütçe düşünmeye gitti — yanıt gelemedi", }; diff --git a/web/js/i18n/zh.js b/web/js/i18n/zh.js index 1a90a9e..60afc1d 100644 --- a/web/js/i18n/zh.js +++ b/web/js/i18n/zh.js @@ -618,4 +618,5 @@ export default { "keys.h.st.notFound": "网关所在机器上未找到", "bm.degraded": "路由器降级", "bm.routerDegraded": "路由器未及时响应 — 请求走了备用链路；分类不计入统计。", + "bm.cutByCap": "预算全用于思考——未能给出答案", }; diff --git a/web/js/views/benchmark.js b/web/js/views/benchmark.js index 13cce3f..d96db88 100644 --- a/web/js/views/benchmark.js +++ b/web/js/views/benchmark.js @@ -31,8 +31,22 @@ const SUITE = [ { id: "translate", cat: "translation", label: ["translation"], max: 64, prompt: "Translate to English: «Доброе утро». Reply with the translation only.", check: ["good morning"] }, { id: "format-json", cat: "instruction", label: null, max: 96, prompt: 'Return a JSON object with a single key "n" whose value is 7. JSON only.', check: ['"n"', "7"] }, { id: "summarize", cat: "summarization", label: ["summarization"], max: 96, prompt: "Summarize in one short sentence: 'The cat sat on the warm mat by the window. After a while it curled up and fell asleep in the sun.'", check: ["cat|кот|кошк"] }, - { id: "creative", cat: "creative-writing", label: ["creative-writing"], max: 96, prompt: "Write a two-line poem about the sea.", check: ["sea|море|ocean"] }, + { id: "creative", cat: "creative-writing", label: ["creative-writing"], max: 512, prompt: "Write a two-line poem about the sea.", check: ["sea|море|ocean"] }, { id: "chitchat", cat: "chitchat", label: ["chitchat"], quick: true, max: 48, prompt: "Hi! How are you today?", check: [] }, + // hard probes: dense enough that the router's complexity model rates them + // high tier — this is where tier-based routing actually earns its keep. + // Each run picks a random variant: the router treats familiar prompts as + // less novel (by design), so a fixed wording would decay to medium/low. + { id: "hard-compiler", cat: "hard", label: ["code", "qa"], judge: "route", max: 8, check: [], variants: [ + "Design an SSA-based optimizing compiler backend for an exposed-pipeline VLIW media processor: formulate instruction scheduling as an integer linear program with resource and hazard constraints, apply modulo scheduling to software-pipelined inner loops using rotating register files, do spill-aware register allocation via graph coloring with live-range splitting, add a translation-validated peephole pass, and explain how to prove the scheduler can never violate a structural hazard.", + "Build the code-generation backend of a JIT for GPU warp-specialized kernels: cost-model-driven instruction selection across divergent branches, software pipelining of memory latency with double-buffered shared memory, register allocation under occupancy pressure with rematerialization, a verified peephole pass over PTX, and a proof sketch that no schedule can deadlock the warp scheduler.", + "Construct an optimizing backend for a RISC-V vector coprocessor with configurable lane widths: vector-length-agnostic instruction selection as cost-model search, an integer-linear-program schedule around a chained load-store unit's structural hazards, VL-dependent spill-cost register allocation with live-range splitting, and a translation-validated peephole pass, plus the argument that the scheduler never issues into an occupied pipeline slot.", + ] }, + { id: "hard-protocol", cat: "hard", label: ["qa", "code"], judge: "route", max: 8, check: [], variants: [ + "Design and security-review an end-to-end encrypted coordination protocol for a federated fleet of 100000 IoT devices: lightweight key agreement under 32KB RAM budgets, per-group sender-key trees with post-compromise security, secure firmware-update attestation chained to a transparency log, metadata minimization against a network-level adversary, and revocation of physically captured devices without a round-trip to every peer. Analyze the trade-offs precisely, then specify the exact message flow for rotating a compromised region's keys.", + "Design an end-to-end encrypted messaging system for a disaster-relief mesh network that partitions for days: store-and-forward double ratchet with bounded state, group membership changes that commute across partitions, post-compromise security once connectivity returns, spam and sybil resistance without a central server, and forensic-resistant local storage. Weigh the trade-offs, then give the exact key schedule for merging two long-partitioned group histories.", + "Architect the security layer of a multi-tenant confidential-computing platform: remote attestation chained through TEE quotes to a transparency log, per-tenant key hierarchies with crypto-agile rotation, sealed storage that survives enclave upgrades, side-channel hardening policy for co-resident tenants, and revocation that cannot be rolled back by a malicious host. Analyze each trade-off, then specify the exact handshake for migrating a sealed workload between hosts.", + ] }, ]; const LS_CUSTOM = "allaigate_bench_custom"; @@ -145,7 +159,7 @@ export async function renderBenchmark() { runBtn.disabled = true; stopBtn.style.display = ""; stopBtn.disabled = false; - const tests = suite(); + const tests = suite().filter((x) => mode === "routing" || x.judge !== "route"); const run = { ts: Date.now(), mode, quick: quickCb.checked, models: {}, nTests: tests.length }; for (const model of models) { @@ -154,6 +168,7 @@ export async function renderBenchmark() { for (let i = 0; i < tests.length; i++) { if (stopFlag) break; const test = tests[i]; + const prompt = test.variants ? test.variants[(Math.random() * test.variants.length) | 0] : test.prompt; mount( progressHost, h("div", { class: "card" }, @@ -162,7 +177,7 @@ export async function renderBenchmark() { h("b", { class: "mono" }, mode === "routing" ? t("bm.mode.routing") : model), " · ", t("bm.testOf", { i: i + 1, n: tests.length }), " · ", h("span", { class: "muted" }, test.id)), - h("div", { class: "muted small", style: "margin-top:6px; font-style:italic" }, "«" + test.prompt.slice(0, 110) + (test.prompt.length > 110 ? "…»" : "»")), + h("div", { class: "muted small", style: "margin-top:6px; font-style:italic" }, "«" + prompt.slice(0, 110) + (prompt.length > 110 ? "…»" : "»")), h("div", { class: "pbar", style: "margin-top:8px" }, h("div", { class: "pbar-fill det", style: `width:${Math.round((i / tests.length) * 100)}%` }))) ); @@ -172,12 +187,23 @@ export async function renderBenchmark() { const info = await testStream( // +256: headroom for reasoning models — a tight cap cuts them // mid- and turns a correct model into a false ❌ - { model, messages: [{ role: "user", content: test.prompt }], temperature: 0, max_tokens: (test.max || 128) + 256 }, + { model, messages: [{ role: "user", content: prompt }], temperature: 0, max_tokens: (test.max || 128) + 256 }, (d) => (text += d) ); const latency = performance.now() - started; - const ev = evaluate(text, test); + let ev; + if (test.judge === "route") { + // the test asserts the ROUTER's decision: did it rate this high? + ev = info.tier === "high" + ? { status: "pass", score: 1, note: "high ✓" } + : { status: "fail", score: 0, note: (info.tier || "—") + " ≠ high" }; + } else { + ev = evaluate(text, test); + } const compl = info.usage?.completion_tokens; + if (test.judge !== "route" && ev.status === "fail" && !stripThink(text).trim() && compl >= (test.max || 128) + 250) { + ev.note = t("bm.cutByCap"); // the whole budget went into + } if (compl && latency > 0) { const gen = info.ttft_ms != null ? Math.max(1, latency - info.ttft_ms) : latency; rec.tokS.push(compl / (gen / 1000));