From 179c414a0e9dcbe480d6dbc5dd9e0d1f80af824b Mon Sep 17 00:00:00 2001
From: infosave2007 <urevich55@gmail.com>
Date: Sat, 8 Aug 2026 09:23:53 +0300
Subject: [PATCH] fix(benchmark): retry the SAME hard prompt after a pause
 before reaching for a fresh one
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Difficulty is judged in the background: the first-ever occurrence of a hard
prompt routes by prior (often low/medium), the oracle's judgement lands in
the cache a couple of seconds later. The route-judged retry fired instantly
with a FRESH template — which has the same first-encounter fate — so the
test could go red exactly when the system was working as designed. The retry
now re-asks the same prompt after 4s (steady-state answer from the cache),
and only then tries a fresh template.
---
 web/js/views/benchmark.js | 44 ++++++++++++++++++++++++---------------
 1 file changed, 27 insertions(+), 17 deletions(-)

diff --git a/web/js/views/benchmark.js b/web/js/views/benchmark.js
index cdd3769..edeae8e 100644
--- a/web/js/views/benchmark.js
+++ b/web/js/views/benchmark.js
@@ -216,24 +216,34 @@ export async function renderBenchmark() {
           let ev;
           if (test.judge === "route") {
             // the test asserts the ROUTER's decision: did it rate this high?
-            // One retry with a FRESH template: the router deliberately scores
-            // familiar text lower (novelty), so a single burnt wording must
-            // not fail the whole assertion.
+            // Two honest retries. (1) The SAME prompt after a pause: the
+            // router judges difficulty in the background — the first-ever
+            // occurrence routes by prior, the judgement lands in the cache a
+            // couple of seconds later, so the second occurrence is the real
+            // steady-state answer. (2) A fresh template, in case this one's
+            // wording clustered into a burnt region.
             if (info.tier !== "high" && test.gen) {
-              const p2 = hardPrompt(test.gen);
-              try {
-                const second = await testStream(
-                  { model, messages: [{ role: "user", content: p2 }], temperature: 0, max_tokens: (test.max || 128) + 256 },
-                  () => {}
-                );
-                if (second.tier === "high") {
-                  info.tier = second.tier;
-                  info.selected_model = second.selected_model;
-                  info.task_label = second.task_label;
-                  info.route_source = second.route_source;
-                  info.cost_usd = (info.cost_usd || 0) + (second.cost_usd || 0);
-                }
-              } catch { /* keep first attempt */ }
+              const attempts = [
+                { prompt, delayMs: 4000 },
+                { prompt: hardPrompt(test.gen), delayMs: 0 },
+              ];
+              for (const a of attempts) {
+                if (info.tier === "high") break;
+                if (a.delayMs) await new Promise((r) => setTimeout(r, a.delayMs));
+                try {
+                  const again = await testStream(
+                    { model, messages: [{ role: "user", content: a.prompt }], temperature: 0, max_tokens: (test.max || 128) + 256 },
+                    () => {}
+                  );
+                  info.cost_usd = (info.cost_usd || 0) + (again.cost_usd || 0);
+                  if (again.tier === "high") {
+                    info.tier = again.tier;
+                    info.selected_model = again.selected_model;
+                    info.task_label = again.task_label;
+                    info.route_source = again.route_source;
+                  }
+                } catch { /* keep previous attempt */ }
+              }
             }
             ev = info.tier === "high"
               ? { status: "pass", score: 1, note: "high ✓" }