feat(benchmark): hard tests that light up the high tier — judged on routing, not essays
Two "hard" probes (full suite only): a compiler-backend design and a security
protocol design, each dense enough for the router's complexity model to rate
them high. They are judged on the ROUTER's decision (judge: "route") — pass
means tier=high was reached and the request went to the big model; the answer
itself is not scored (open-ended essays aren't keyword-judgeable, and
reasoning models happily burn 3000+ thinking tokens on them — measured). With
max_tokens kept tiny the probes cost cents even on the priciest model, while
the cost column tells the whole story: $0.0148 for the high-tier request next
to $0.00005 for the easy ones.
Each run picks a random variant of the prompt: the router deliberately scores
familiar text as less novel, so a fixed wording would decay to medium over
repeated runs. In pinned-models mode route-judged tests are skipped.
Also: creative-writing budget 96→512 (deepseek was spending the whole cap
thinking and returning an empty poem), and when a model does burn its entire
budget inside <think> the table now says so ("budget spent on reasoning")
instead of the misleading "empty answer".
Server-side (cortiq-router prod, both replicas rolling-restarted): the high
band started at 0.66 while the complexity model tops out ~0.65 on the hardest
single prompts — high was unreachable. Band boundary moved to 0.50 based on
measured distribution (easy 0.25-0.36, medium 0.37-0.49, hard 0.50-0.65);
config backed up as config.prod.toml.bak-tiers.
This commit is contained in:
infosave2007
Vendored
+1
@@ -618,4 +618,5 @@ export default {
|
||||
"keys.h.st.notFound": "Auf der Gateway-Maschine nicht gefunden",
|
||||
"bm.degraded": "Router degradiert",
|
||||
"bm.routerDegraded": "Der Router hat nicht rechtzeitig geantwortet — die Anfrage nutzte die Fallback-Kette; die Klassifikation wird nicht gewertet.",
|
||||
"bm.cutByCap": "Budget ging fürs Denken drauf — keine Antwort mehr",
|
||||
};
|
||||
|
||||
Vendored
+1
@@ -611,4 +611,5 @@ export default {
|
||||
"keys.h.st.notFound": "Not found on the gateway machine",
|
||||
"bm.degraded": "router degraded",
|
||||
"bm.routerDegraded": "The router did not answer in time — the request used the fallback chain; classification is not judged.",
|
||||
"bm.cutByCap": "budget spent on reasoning — answer never came",
|
||||
};
|
||||
|
||||
Vendored
+1
@@ -618,4 +618,5 @@ export default {
|
||||
"keys.h.st.notFound": "No encontrado en la máquina de la pasarela",
|
||||
"bm.degraded": "router degradado",
|
||||
"bm.routerDegraded": "El router no respondió a tiempo — la petición usó la cadena de reserva; la clasificación no se cuenta.",
|
||||
"bm.cutByCap": "presupuesto gastado en razonar — sin respuesta final",
|
||||
};
|
||||
|
||||
Vendored
+1
@@ -618,4 +618,5 @@ export default {
|
||||
"keys.h.st.notFound": "Introuvable sur la machine de la passerelle",
|
||||
"bm.degraded": "routeur dégradé",
|
||||
"bm.routerDegraded": "Le routeur n'a pas répondu à temps — la requête a suivi la chaîne de secours ; la classification n'est pas comptée.",
|
||||
"bm.cutByCap": "budget épuisé en raisonnement — pas de réponse",
|
||||
};
|
||||
|
||||
Vendored
+1
@@ -610,4 +610,5 @@ export default {
|
||||
"keys.h.st.notFound": "Не найден на машине шлюза",
|
||||
"bm.degraded": "роутер деградировал",
|
||||
"bm.routerDegraded": "Роутер не успел ответить — запрос ушёл по резервной цепочке; классификация не засчитывается.",
|
||||
"bm.cutByCap": "бюджет ушёл на размышления — ответ не успел",
|
||||
};
|
||||
|
||||
Vendored
+1
@@ -618,4 +618,5 @@ export default {
|
||||
"keys.h.st.notFound": "Ağ geçidi makinesinde bulunamadı",
|
||||
"bm.degraded": "yönlendirici yedeğe düştü",
|
||||
"bm.routerDegraded": "Yönlendirici zamanında yanıt vermedi — istek yedek zincire gitti; sınıflandırma sayılmaz.",
|
||||
"bm.cutByCap": "bütçe düşünmeye gitti — yanıt gelemedi",
|
||||
};
|
||||
|
||||
Vendored
+1
@@ -618,4 +618,5 @@ export default {
|
||||
"keys.h.st.notFound": "网关所在机器上未找到",
|
||||
"bm.degraded": "路由器降级",
|
||||
"bm.routerDegraded": "路由器未及时响应 — 请求走了备用链路；分类不计入统计。",
|
||||
"bm.cutByCap": "预算全用于思考——未能给出答案",
|
||||
};
|
||||
|
||||
Vendored
+31
-5
@@ -31,8 +31,22 @@ const SUITE = [
|
||||
{ id: "translate", cat: "translation", label: ["translation"], max: 64, prompt: "Translate to English: «Доброе утро». Reply with the translation only.", check: ["good morning"] },
|
||||
{ id: "format-json", cat: "instruction", label: null, max: 96, prompt: 'Return a JSON object with a single key "n" whose value is 7. JSON only.', check: ['"n"', "7"] },
|
||||
{ id: "summarize", cat: "summarization", label: ["summarization"], max: 96, prompt: "Summarize in one short sentence: 'The cat sat on the warm mat by the window. After a while it curled up and fell asleep in the sun.'", check: ["cat|кот|кошк"] },
|
||||
{ id: "creative", cat: "creative-writing", label: ["creative-writing"], max: 96, prompt: "Write a two-line poem about the sea.", check: ["sea|море|ocean"] },
|
||||
{ id: "creative", cat: "creative-writing", label: ["creative-writing"], max: 512, prompt: "Write a two-line poem about the sea.", check: ["sea|море|ocean"] },
|
||||
{ id: "chitchat", cat: "chitchat", label: ["chitchat"], quick: true, max: 48, prompt: "Hi! How are you today?", check: [] },
|
||||
// hard probes: dense enough that the router's complexity model rates them
|
||||
// high tier — this is where tier-based routing actually earns its keep.
|
||||
// Each run picks a random variant: the router treats familiar prompts as
|
||||
// less novel (by design), so a fixed wording would decay to medium/low.
|
||||
{ id: "hard-compiler", cat: "hard", label: ["code", "qa"], judge: "route", max: 8, check: [], variants: [
|
||||
"Design an SSA-based optimizing compiler backend for an exposed-pipeline VLIW media processor: formulate instruction scheduling as an integer linear program with resource and hazard constraints, apply modulo scheduling to software-pipelined inner loops using rotating register files, do spill-aware register allocation via graph coloring with live-range splitting, add a translation-validated peephole pass, and explain how to prove the scheduler can never violate a structural hazard.",
|
||||
"Build the code-generation backend of a JIT for GPU warp-specialized kernels: cost-model-driven instruction selection across divergent branches, software pipelining of memory latency with double-buffered shared memory, register allocation under occupancy pressure with rematerialization, a verified peephole pass over PTX, and a proof sketch that no schedule can deadlock the warp scheduler.",
|
||||
"Construct an optimizing backend for a RISC-V vector coprocessor with configurable lane widths: vector-length-agnostic instruction selection as cost-model search, an integer-linear-program schedule around a chained load-store unit's structural hazards, VL-dependent spill-cost register allocation with live-range splitting, and a translation-validated peephole pass, plus the argument that the scheduler never issues into an occupied pipeline slot.",
|
||||
] },
|
||||
{ id: "hard-protocol", cat: "hard", label: ["qa", "code"], judge: "route", max: 8, check: [], variants: [
|
||||
"Design and security-review an end-to-end encrypted coordination protocol for a federated fleet of 100000 IoT devices: lightweight key agreement under 32KB RAM budgets, per-group sender-key trees with post-compromise security, secure firmware-update attestation chained to a transparency log, metadata minimization against a network-level adversary, and revocation of physically captured devices without a round-trip to every peer. Analyze the trade-offs precisely, then specify the exact message flow for rotating a compromised region's keys.",
|
||||
"Design an end-to-end encrypted messaging system for a disaster-relief mesh network that partitions for days: store-and-forward double ratchet with bounded state, group membership changes that commute across partitions, post-compromise security once connectivity returns, spam and sybil resistance without a central server, and forensic-resistant local storage. Weigh the trade-offs, then give the exact key schedule for merging two long-partitioned group histories.",
|
||||
"Architect the security layer of a multi-tenant confidential-computing platform: remote attestation chained through TEE quotes to a transparency log, per-tenant key hierarchies with crypto-agile rotation, sealed storage that survives enclave upgrades, side-channel hardening policy for co-resident tenants, and revocation that cannot be rolled back by a malicious host. Analyze each trade-off, then specify the exact handshake for migrating a sealed workload between hosts.",
|
||||
] },
|
||||
];
|
||||
|
||||
const LS_CUSTOM = "allaigate_bench_custom";
|
||||
@@ -145,7 +159,7 @@ export async function renderBenchmark() {
|
||||
runBtn.disabled = true;
|
||||
stopBtn.style.display = "";
|
||||
stopBtn.disabled = false;
|
||||
const tests = suite();
|
||||
const tests = suite().filter((x) => mode === "routing" || x.judge !== "route");
|
||||
const run = { ts: Date.now(), mode, quick: quickCb.checked, models: {}, nTests: tests.length };
|
||||
|
||||
for (const model of models) {
|
||||
@@ -154,6 +168,7 @@ export async function renderBenchmark() {
|
||||
for (let i = 0; i < tests.length; i++) {
|
||||
if (stopFlag) break;
|
||||
const test = tests[i];
|
||||
const prompt = test.variants ? test.variants[(Math.random() * test.variants.length) | 0] : test.prompt;
|
||||
mount(
|
||||
progressHost,
|
||||
h("div", { class: "card" },
|
||||
@@ -162,7 +177,7 @@ export async function renderBenchmark() {
|
||||
h("b", { class: "mono" }, mode === "routing" ? t("bm.mode.routing") : model), " · ",
|
||||
t("bm.testOf", { i: i + 1, n: tests.length }), " · ",
|
||||
h("span", { class: "muted" }, test.id)),
|
||||
h("div", { class: "muted small", style: "margin-top:6px; font-style:italic" }, "«" + test.prompt.slice(0, 110) + (test.prompt.length > 110 ? "…»" : "»")),
|
||||
h("div", { class: "muted small", style: "margin-top:6px; font-style:italic" }, "«" + prompt.slice(0, 110) + (prompt.length > 110 ? "…»" : "»")),
|
||||
h("div", { class: "pbar", style: "margin-top:8px" },
|
||||
h("div", { class: "pbar-fill det", style: `width:${Math.round((i / tests.length) * 100)}%` })))
|
||||
);
|
||||
@@ -172,12 +187,23 @@ export async function renderBenchmark() {
|
||||
const info = await testStream(
|
||||
// +256: headroom for reasoning models — a tight cap cuts them
|
||||
// mid-<think> and turns a correct model into a false ❌
|
||||
{ model, messages: [{ role: "user", content: test.prompt }], temperature: 0, max_tokens: (test.max || 128) + 256 },
|
||||
{ model, messages: [{ role: "user", content: prompt }], temperature: 0, max_tokens: (test.max || 128) + 256 },
|
||||
(d) => (text += d)
|
||||
);
|
||||
const latency = performance.now() - started;
|
||||
const ev = evaluate(text, test);
|
||||
let ev;
|
||||
if (test.judge === "route") {
|
||||
// the test asserts the ROUTER's decision: did it rate this high?
|
||||
ev = info.tier === "high"
|
||||
? { status: "pass", score: 1, note: "high ✓" }
|
||||
: { status: "fail", score: 0, note: (info.tier || "—") + " ≠ high" };
|
||||
} else {
|
||||
ev = evaluate(text, test);
|
||||
}
|
||||
const compl = info.usage?.completion_tokens;
|
||||
if (test.judge !== "route" && ev.status === "fail" && !stripThink(text).trim() && compl >= (test.max || 128) + 250) {
|
||||
ev.note = t("bm.cutByCap"); // the whole budget went into <think>
|
||||
}
|
||||
if (compl && latency > 0) {
|
||||
const gen = info.ttft_ms != null ? Math.max(1, latency - info.ttft_ms) : latency;
|
||||
rec.tokS.push(compl / (gen / 1000));
|
||||
|
||||
Reference in New Issue
Block a user