github/cortiq-gateway
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(benchmark): retry the SAME hard prompt after a pause before reaching for a fresh one

Browse Source 
Difficulty is judged in the background: the first-ever occurrence of a hard
prompt routes by prior (often low/medium), the oracle's judgement lands in
the cache a couple of seconds later. The route-judged retry fired instantly
with a FRESH template — which has the same first-encounter fate — so the
test could go red exactly when the system was working as designed. The retry
now re-asks the same prompt after 4s (steady-state answer from the cache),
and only then tries a fresh template.
This commit is contained in:
infosave2007
2026-08-08 09:23:53 +03:00
parent 115b04f20e
commit 179c414a0e
1 changed files with 27 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/js/views/benchmark.js
Vendored
+27 -17
View File
@@ -216,24 +216,34 @@ export async function renderBenchmark() {
let ev; let ev;
if (test.judge === "route") { if (test.judge === "route") {
// the test asserts the ROUTER's decision: did it rate this high? // the test asserts the ROUTER's decision: did it rate this high?
// One retry with a FRESH template: the router deliberately scores // Two honest retries. (1) The SAME prompt after a pause: the
// familiar text lower (novelty), so a single burnt wording must // router judges difficulty in the background — the first-ever
// not fail the whole assertion. // occurrence routes by prior, the judgement lands in the cache a
// couple of seconds later, so the second occurrence is the real
// steady-state answer. (2) A fresh template, in case this one's
// wording clustered into a burnt region.
if (info.tier !== "high" && test.gen) { if (info.tier !== "high" && test.gen) {
const p2 = hardPrompt(test.gen); const attempts = [
try { { prompt, delayMs: 4000 },
const second = await testStream( { prompt: hardPrompt(test.gen), delayMs: 0 },
{ model, messages: [{ role: "user", content: p2 }], temperature: 0, max_tokens: (test.max || 128) + 256 }, ];
() => {} for (const a of attempts) {
); if (info.tier === "high") break;
if (second.tier === "high") { if (a.delayMs) await new Promise((r) => setTimeout(r, a.delayMs));
info.tier = second.tier; try {
info.selected_model = second.selected_model; const again = await testStream(
info.task_label = second.task_label; { model, messages: [{ role: "user", content: a.prompt }], temperature: 0, max_tokens: (test.max || 128) + 256 },
info.route_source = second.route_source; () => {}
info.cost_usd = (info.cost_usd || 0) + (second.cost_usd || 0); );
} info.cost_usd = (info.cost_usd || 0) + (again.cost_usd || 0);
} catch { /* keep first attempt */ } if (again.tier === "high") {
info.tier = again.tier;
info.selected_model = again.selected_model;
info.task_label = again.task_label;
info.route_source = again.route_source;
}
} catch { /* keep previous attempt */ }
}
} }
ev = info.tier === "high" ev = info.tier === "high"
? { status: "pass", score: 1, note: "high ✓" } ? { status: "pass", score: 1, note: "high ✓" }