feat(routing): high tier escalates over task rules; benchmark generates hard prompts that stay hard
Routing semantics fix observed live: a task rule ("code → cheap model") was
pinning even high-complexity prompts to the cheap model — the tier system
never got a say. At tier=high the tier chain now leads and the task rule
demotes to fallback; low/medium keep task-rule-first. Unit-tested both ways.
The benchmark's hard probes are now GENERATED per run from two pools
(science: protein folding / telescope scheduling / materials discovery;
ops: epidemic logistics / grid markets / climate risk) with rotating domains
and numbers. Two findings against the live router drove this: ambiguity
carries the biggest weight in the complexity score, so interdisciplinary
prompts rate high reliably (0.52-0.66) while single-domain "hard" prompts
classify too confidently and sink to medium; and any fixed wording loses
novelty on every repeat. Their labels are null — straddling task labels is
precisely what makes them high, so classification is not judged.
Verified end-to-end: full run 100% answers, 13/13 classification, both hard
probes high → routed to the expensive model (one of them over a code task
rule), at $0.0018 each thanks to the tiny route-judge token budget.
This commit is contained in:
+43
-13
@@ -43,20 +43,35 @@ impl RoutingTable {
|
||||
/// Returns `(candidates, task_rule_matched)`.
|
||||
pub fn candidates_for(&self, task: &str, tier: &str) -> (Vec<String>, bool) {
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
let matched = if let Some(rule) = self.tasks.get(task) {
|
||||
out.extend(rule.iter().cloned());
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
let tier_chain = self
|
||||
.tiers
|
||||
.get(tier)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| vec![self.default.clone()]);
|
||||
for id in tier_chain {
|
||||
if !out.contains(&id) {
|
||||
out.push(id);
|
||||
// Task rules steer everyday traffic — but a high-complexity verdict
|
||||
// must escalate ABOVE them, or a "code → cheap-model" rule would pin
|
||||
// the hardest prompts to the small model forever. At tier=high the
|
||||
// tier chain leads and the task rule becomes the fallback; otherwise
|
||||
// the task rule leads as before.
|
||||
let escalate = tier == "high" && self.tiers.contains_key("high");
|
||||
if escalate {
|
||||
out.extend(tier_chain.iter().cloned());
|
||||
}
|
||||
let matched = if let Some(rule) = self.tasks.get(task) {
|
||||
for id in rule {
|
||||
if !out.contains(id) {
|
||||
out.push(id.clone());
|
||||
}
|
||||
}
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
if !escalate {
|
||||
for id in tier_chain {
|
||||
if !out.contains(&id) {
|
||||
out.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
out.retain(|id| !id.is_empty());
|
||||
@@ -104,14 +119,29 @@ mod tests {
|
||||
&[("low", &["cheap", "local"]), ("high", &["big"])],
|
||||
"cheap",
|
||||
);
|
||||
let (c, matched) = t.candidates_for("translation", "high");
|
||||
assert!(matched);
|
||||
assert_eq!(c, vec!["local", "big"]);
|
||||
// dedup: rule model already in the tier chain appears once, first
|
||||
let (c, _) = t.candidates_for("translation", "low");
|
||||
let (c, matched) = t.candidates_for("translation", "low");
|
||||
assert!(matched);
|
||||
assert_eq!(c, vec!["local", "cheap"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn high_tier_escalates_over_task_rule() {
|
||||
let t = table(
|
||||
&[("code", &["cheap-coder"])],
|
||||
&[("low", &["cheap"]), ("high", &["big"])],
|
||||
"cheap",
|
||||
);
|
||||
// medium (no chain defined → default): task rule still leads
|
||||
let (c, m) = t.candidates_for("code", "medium");
|
||||
assert!(m);
|
||||
assert_eq!(c[0], "cheap-coder");
|
||||
// high: tier chain leads, task rule follows as fallback
|
||||
let (c, m) = t.candidates_for("code", "high");
|
||||
assert!(m);
|
||||
assert_eq!(c, vec!["big", "cheap-coder"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_rule_falls_back_to_tier_then_default() {
|
||||
let t = table(&[], &[("low", &["a"])], "d");
|
||||
|
||||
Reference in New Issue
Block a user