From ff8415d7d49aae17d81fdb81a4408be615155d99 Mon Sep 17 00:00:00 2001 From: infosave2007 Date: Fri, 7 Aug 2026 21:34:24 +0300 Subject: [PATCH] feat(routing): high tier escalates over task rules; benchmark generates hard prompts that stay hard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Routing semantics fix observed live: a task rule ("code → cheap model") was pinning even high-complexity prompts to the cheap model — the tier system never got a say. At tier=high the tier chain now leads and the task rule demotes to fallback; low/medium keep task-rule-first. Unit-tested both ways. The benchmark's hard probes are now GENERATED per run from two pools (science: protein folding / telescope scheduling / materials discovery; ops: epidemic logistics / grid markets / climate risk) with rotating domains and numbers. Two findings against the live router drove this: ambiguity carries the biggest weight in the complexity score, so interdisciplinary prompts rate high reliably (0.52-0.66) while single-domain "hard" prompts classify too confidently and sink to medium; and any fixed wording loses novelty on every repeat. Their labels are null — straddling task labels is precisely what makes them high, so classification is not judged. Verified end-to-end: full run 100% answers, 13/13 classification, both hard probes high → routed to the expensive model (one of them over a code task rule), at $0.0018 each thanks to the tiny route-judge token budget. --- src/routing.rs | 56 ++++++++++++++++++++++++++++++--------- web/js/views/benchmark.js | 42 ++++++++++++++++++++--------- 2 files changed, 72 insertions(+), 26 deletions(-) diff --git a/src/routing.rs b/src/routing.rs index 6d006ad..a3fdefa 100644 --- a/src/routing.rs +++ b/src/routing.rs @@ -43,20 +43,35 @@ impl RoutingTable { /// Returns `(candidates, task_rule_matched)`. pub fn candidates_for(&self, task: &str, tier: &str) -> (Vec, bool) { let mut out: Vec = Vec::new(); - let matched = if let Some(rule) = self.tasks.get(task) { - out.extend(rule.iter().cloned()); - true - } else { - false - }; let tier_chain = self .tiers .get(tier) .cloned() .unwrap_or_else(|| vec![self.default.clone()]); - for id in tier_chain { - if !out.contains(&id) { - out.push(id); + // Task rules steer everyday traffic — but a high-complexity verdict + // must escalate ABOVE them, or a "code → cheap-model" rule would pin + // the hardest prompts to the small model forever. At tier=high the + // tier chain leads and the task rule becomes the fallback; otherwise + // the task rule leads as before. + let escalate = tier == "high" && self.tiers.contains_key("high"); + if escalate { + out.extend(tier_chain.iter().cloned()); + } + let matched = if let Some(rule) = self.tasks.get(task) { + for id in rule { + if !out.contains(id) { + out.push(id.clone()); + } + } + true + } else { + false + }; + if !escalate { + for id in tier_chain { + if !out.contains(&id) { + out.push(id); + } } } out.retain(|id| !id.is_empty()); @@ -104,14 +119,29 @@ mod tests { &[("low", &["cheap", "local"]), ("high", &["big"])], "cheap", ); - let (c, matched) = t.candidates_for("translation", "high"); - assert!(matched); - assert_eq!(c, vec!["local", "big"]); // dedup: rule model already in the tier chain appears once, first - let (c, _) = t.candidates_for("translation", "low"); + let (c, matched) = t.candidates_for("translation", "low"); + assert!(matched); assert_eq!(c, vec!["local", "cheap"]); } + #[test] + fn high_tier_escalates_over_task_rule() { + let t = table( + &[("code", &["cheap-coder"])], + &[("low", &["cheap"]), ("high", &["big"])], + "cheap", + ); + // medium (no chain defined → default): task rule still leads + let (c, m) = t.candidates_for("code", "medium"); + assert!(m); + assert_eq!(c[0], "cheap-coder"); + // high: tier chain leads, task rule follows as fallback + let (c, m) = t.candidates_for("code", "high"); + assert!(m); + assert_eq!(c, vec!["big", "cheap-coder"]); + } + #[test] fn no_rule_falls_back_to_tier_then_default() { let t = table(&[], &[("low", &["a"])], "d"); diff --git a/web/js/views/benchmark.js b/web/js/views/benchmark.js index d96db88..6406467 100644 --- a/web/js/views/benchmark.js +++ b/web/js/views/benchmark.js @@ -35,20 +35,36 @@ const SUITE = [ { id: "chitchat", cat: "chitchat", label: ["chitchat"], quick: true, max: 48, prompt: "Hi! How are you today?", check: [] }, // hard probes: dense enough that the router's complexity model rates them // high tier — this is where tier-based routing actually earns its keep. - // Each run picks a random variant: the router treats familiar prompts as - // less novel (by design), so a fixed wording would decay to medium/low. - { id: "hard-compiler", cat: "hard", label: ["code", "qa"], judge: "route", max: 8, check: [], variants: [ - "Design an SSA-based optimizing compiler backend for an exposed-pipeline VLIW media processor: formulate instruction scheduling as an integer linear program with resource and hazard constraints, apply modulo scheduling to software-pipelined inner loops using rotating register files, do spill-aware register allocation via graph coloring with live-range splitting, add a translation-validated peephole pass, and explain how to prove the scheduler can never violate a structural hazard.", - "Build the code-generation backend of a JIT for GPU warp-specialized kernels: cost-model-driven instruction selection across divergent branches, software pipelining of memory latency with double-buffered shared memory, register allocation under occupancy pressure with rematerialization, a verified peephole pass over PTX, and a proof sketch that no schedule can deadlock the warp scheduler.", - "Construct an optimizing backend for a RISC-V vector coprocessor with configurable lane widths: vector-length-agnostic instruction selection as cost-model search, an integer-linear-program schedule around a chained load-store unit's structural hazards, VL-dependent spill-cost register allocation with live-range splitting, and a translation-validated peephole pass, plus the argument that the scheduler never issues into an occupied pipeline slot.", - ] }, - { id: "hard-protocol", cat: "hard", label: ["qa", "code"], judge: "route", max: 8, check: [], variants: [ - "Design and security-review an end-to-end encrypted coordination protocol for a federated fleet of 100000 IoT devices: lightweight key agreement under 32KB RAM budgets, per-group sender-key trees with post-compromise security, secure firmware-update attestation chained to a transparency log, metadata minimization against a network-level adversary, and revocation of physically captured devices without a round-trip to every peer. Analyze the trade-offs precisely, then specify the exact message flow for rotating a compromised region's keys.", - "Design an end-to-end encrypted messaging system for a disaster-relief mesh network that partitions for days: store-and-forward double ratchet with bounded state, group membership changes that commute across partitions, post-compromise security once connectivity returns, spam and sybil resistance without a central server, and forensic-resistant local storage. Weigh the trade-offs, then give the exact key schedule for merging two long-partitioned group histories.", - "Architect the security layer of a multi-tenant confidential-computing platform: remote attestation chained through TEE quotes to a transparency log, per-tenant key hierarchies with crypto-agile rotation, sealed storage that survives enclave upgrades, side-channel hardening policy for co-resident tenants, and revocation that cannot be rolled back by a malicious host. Analyze each trade-off, then specify the exact handshake for migrating a sealed workload between hosts.", - ] }, + // Prompts are GENERATED fresh each run (domain + numbers rotate): the + // router deliberately scores familiar text as less novel, so any fixed + // wording would decay to medium/low over repeated runs. + // label: null — ambiguity between task labels is exactly what makes these high + { id: "hard-science", cat: "hard", label: null, judge: "route", max: 8, check: [], gen: "science" }, + { id: "hard-ops", cat: "hard", label: null, judge: "route", max: 8, check: [], gen: "ops" }, ]; +// Fresh hard prompts per run. Two findings drive the design (measured against +// the live router): ambiguity carries the biggest weight in the complexity +// score, so INTERDISCIPLINARY prompts (science + engineering + planning) rate +// high reliably, while single-domain ones classify too confidently and drop +// to medium; and familiar text loses novelty on every repeat, so domains and +// numbers rotate to keep each run lexically fresh. +function hardPrompt(kind) { + const pick = (a) => a[(Math.random() * a.length) | 0]; + const n = (lo, hi) => lo + ((Math.random() * (hi - lo)) | 0); + const science = [ + () => `Architect a protein-folding inference pipeline for a wet lab: ensemble MSA generation with taxonomy-aware subsampling, template selection under a ${n(30, 60)} percent identity ceiling, distributed attention across ${n(4, 16)} GPUs with activation checkpointing, confidence calibration against experimental B-factors, and an active-learning loop that decides which of ${n(300, 2000)} candidate mutants to synthesize next week.`, + () => `Schedule observations for a network of ${n(8, 30)} robotic telescopes chasing gravitational-wave counterparts: probability skymaps updated every ${n(5, 20)} minutes, slew-time-aware target assignment with decaying rewards, cloud-cover forecasts folded in as stochastic availability, spectroscopy triggered only above a ${n(40, 85)} percent kilonova likelihood, and a plan to validate the scheduler against the one historical multi-messenger event.`, + () => `Run a closed-loop materials-discovery campaign for solid-state electrolytes: DFT screening of ${n(2000, 9000)} candidates, Bayesian optimization driving ${n(3, 12)} parallel synthesis robots, CNN-based XRD phase identification, an ionic-conductivity target of ${n(2, 25)} mS/cm, and an explicit decision policy for when simulation is trusted over synthesis.`, + ]; + const ops = [ + () => `Model a dengue outbreak across ${n(20, 90)} districts by coupling an SEIR compartment model to vaccine cold-chain optimization: transmission parameters estimated from noisy case counts with a particle filter, uncertainty propagated into weekly demand forecasts, refrigerated routing under a ${n(120, 400)}-dose-per-crate constraint, and a defended ethical weighting for which ${n(3, 12)} districts get priority when supply covers ${n(20, 60)} percent of demand.`, + () => `Co-optimize a regional power grid's day-ahead market with a fleet of ${n(200, 2000)} grid-scale batteries: bid curves learned from ${n(2, 6)} years of price data, unit commitment as mixed-integer programming with ramp constraints, degradation amortized into marginal costs, N-1 contingency screening after every dispatch, and a mechanism-design argument that the bidding rules cannot be gamed by a coalition controlling ${n(10, 30)} percent of storage.`, + () => `Couple regional climate downscaling at ${n(2, 12)} km resolution to catastrophe-bond pricing: bias-correct ${n(10, 40)} CMIP ensemble members, generate stochastic hurricane tracks, propagate landfall distributions into a portfolio of ${n(30, 200)} bonds, price the attachment probabilities, and defend the discount rate applied to century-scale tail risk.`, + ]; + return pick(kind === "science" ? science : ops)(); +} + const LS_CUSTOM = "allaigate_bench_custom"; const LS_HISTORY = "allaigate_bench_history"; @@ -168,7 +184,7 @@ export async function renderBenchmark() { for (let i = 0; i < tests.length; i++) { if (stopFlag) break; const test = tests[i]; - const prompt = test.variants ? test.variants[(Math.random() * test.variants.length) | 0] : test.prompt; + const prompt = test.gen ? hardPrompt(test.gen) : test.prompt; mount( progressHost, h("div", { class: "card" },