github/cortiq-gateway
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: per-key spend accounting and per-task complexity pinning

Browse Source 
API keys page: the table now answers "which department spends what" — per-key
requests, tokens in/out and cost for a selectable window (24h/7d/30d) plus a
cumulative all-time cost column, joined from stats by the key's account.

The stats log stays bounded while the accounting survives: once stats.jsonl
exceeds max_file_mb (default 16, 0 disables), records older than the
retention window fold into cumulative per-account totals in a small sidecar
file (tmp+rename on both) and leave the log. alltime = folded + in-file, a
record is never counted twice, verified end-to-end on synthetic data
(1.2 MB → 301 KB, totals identical before/after compaction and restarts).

Routing: a task rule can now PIN the complexity tier —
`[routing.tasks.legal] models=[...] tier="high"` (old plain-array rules parse
unchanged). The pinned tier replaces the router's verdict before the
escalation check and reports itself as route_source "router+task-rule@high".
The Routing page grows a per-task «Complexity» selector (Auto/low/medium/
high); verified live: a code prompt with a pinned high went
router+task-rule@high → the expensive model.

13 new strings in all 7 languages; docs/ROUTING(.ru) updated.
This commit is contained in:
infosave2007
2026-08-08 08:42:11 +03:00
parent ef28a6633d
commit 95e560dc72
18 changed files with 441 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/admin/mod.rs
+8 -15
View File
@@ -116,13 +116,7 @@ fn tiers_to_map(routing: &RoutingCfg) -> HashMap<String, Vec<String>> {
routing
.tiers
.iter()
.map(|(k, v)| {
let list = match v {
TierTargets::List(l) => l.clone(),
TierTargets::One(s) => vec![s.clone()],
};
(k.clone(), list)
})
.map(|(k, v)| (k.clone(), v.models()))
.collect()
}
@@ -758,6 +752,7 @@ fn strip_id_from_routing(routing: &mut RoutingCfg, id: &str) {
}
for tt in routing.tiers.values_mut().chain(routing.tasks.values_mut()) {
match tt {
crate::config::TierTargets::Rule { models, .. } => models.retain(|x| x != id),
crate::config::TierTargets::List(v) => v.retain(|x| x != id),
crate::config::TierTargets::One(s) if s == id => {
*tt = crate::config::TierTargets::List(Vec::new());
@@ -767,6 +762,7 @@ fn strip_id_from_routing(routing: &mut RoutingCfg, id: &str) {
}
// an emptied task rule means "no rule", not "no candidates"
routing.tasks.retain(|_, tt| match tt {
crate::config::TierTargets::Rule { models, tier } => !models.is_empty() || tier.is_some(),
crate::config::TierTargets::List(v) => !v.is_empty(),
crate::config::TierTargets::One(s) => !s.is_empty(),
});
@@ -910,7 +906,7 @@ async fn probe_model(State(state): State<SharedState>, Path(id): Path<String>) -
async fn get_routing(State(state): State<SharedState>) -> ApiResult {
let live = state.live();
let tasks: HashMap<String, Vec<String>> = live
let tasks: HashMap<String, serde_json::Value> = live
.cfg
.routing
.tasks
@@ -918,10 +914,7 @@ async fn get_routing(State(state): State<SharedState>) -> ApiResult {
.map(|(k, v)| {
(
k.clone(),
match v {
TierTargets::List(l) => l.clone(),
TierTargets::One(s) => vec![s.clone()],
},
json!({ "models": v.models(), "tier": v.forced_tier() }),
)
})
.collect();
@@ -973,7 +966,7 @@ struct RoutingBody {
#[serde(default)]
tiers: HashMap<String, Vec<String>>,
#[serde(default)]
tasks: HashMap<String, Vec<String>>,
tasks: HashMap<String, TierTargets>,
default: String,
#[serde(default)]
policy: RoutingPolicy,
@@ -990,8 +983,8 @@ async fn put_routing(State(state): State<SharedState>, Json(body): Json<RoutingB
tasks: body
.tasks
.into_iter()
.filter(|(_, v)| !v.is_empty()) // empty rule = "по сложности", not a rule
.map(|(k, v)| (k, TierTargets::List(v)))
// empty rule (no models AND no forced tier) = "по сложности"
.filter(|(_, v)| !v.models().is_empty() || v.forced_tier().is_some())
.collect(),
default: body.default,
policy: body.policy,
src/config.rs
+51 -5
View File
@@ -408,13 +408,42 @@ pub struct RoutingCfg {
/// A routing tier is either a list of model_ids or the string `default`.
/// (`default` is handled by a separate field; only tier lists appear here.)
/// Task rules additionally accept the full form
/// `{ models = [...], tier = "high" }`, where `tier` FORCES the complexity
/// tier for that task (e.g. legal review is always high, no matter what the
/// router scored). Untagged: the map variant is tried first, so the old
/// plain-array form keeps parsing unchanged.
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(untagged)]
pub enum TierTargets {
Rule {
#[serde(default)]
models: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
tier: Option<String>,
},
List(Vec<String>),
One(String),
}
impl TierTargets {
/// The ordered model list, whatever the syntactic form.
pub fn models(&self) -> Vec<String> {
match self {
TierTargets::Rule { models, .. } => models.clone(),
TierTargets::List(v) => v.clone(),
TierTargets::One(s) => vec![s.clone()],
}
}
/// Forced complexity tier (task rules only).
pub fn forced_tier(&self) -> Option<&str> {
match self {
TierTargets::Rule { tier, .. } => tier.as_deref(),
_ => None,
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct RoutingPolicy {
#[serde(default = "default_mode")]
@@ -587,6 +616,12 @@ pub struct StatsCfg {
/// Retention window for aggregates/time-series replayed on startup, e.g. `7d`.
#[serde(default = "default_stats_retention")]
pub retention: String,
/// Compact the JSONL file once it exceeds this size: records older than
/// `retention` fold into cumulative per-account totals (a small sidecar
/// file) and are removed, so per-key accounting survives while the disk
/// stays bounded. 0 disables compaction.
#[serde(default = "default_stats_max_file_mb")]
pub max_file_mb: u64,
}
impl Default for StatsCfg {
fn default() -> Self {
@@ -595,6 +630,7 @@ impl Default for StatsCfg {
file: default_stats_file(),
ring_size: default_stats_ring(),
retention: default_stats_retention(),
max_file_mb: default_stats_max_file_mb(),
}
}
}
@@ -610,6 +646,9 @@ fn default_stats_ring() -> usize {
fn default_stats_retention() -> String {
"7d".into()
}
fn default_stats_max_file_mb() -> u64 {
16
}
/// Self-warming shadow loop (docs/SELF_WARMING_GATEWAY.md). While a task
/// label is not yet promoted, the client is served the CLOUD answer; in
@@ -863,6 +902,7 @@ impl Config {
}
for (tier, targets) in &self.routing.tiers {
let list = match targets {
TierTargets::Rule { models, .. } => models.clone(),
TierTargets::List(v) => v.clone(),
TierTargets::One(s) => vec![s.clone()],
};
@@ -877,11 +917,7 @@ impl Config {
}
}
for (task, targets) in &self.routing.tasks {
let list = match targets {
TierTargets::List(v) => v.clone(),
TierTargets::One(s) => vec![s.clone()],
};
for id in &list {
for id in &targets.models() {
if !ids.contains(id.as_str()) {
anyhow::bail!(
"config: routing.tasks.{} references unknown model id '{}'",
@@ -890,6 +926,15 @@ impl Config {
);
}
}
if let Some(tier) = targets.forced_tier() {
if !self.routing.tiers.contains_key(tier) {
anyhow::bail!(
"config: routing.tasks.{}.tier '{}' is not a defined routing tier",
task,
tier
);
}
}
}
Ok(())
}
@@ -981,6 +1026,7 @@ min_class = { low = "cheap", medium = "mid" }
let low = back.routing.tiers.get("low").expect("low tier present");
match low {
TierTargets::Rule { models, .. } => assert_eq!(models, &vec!["local-qwen".to_string()]),
TierTargets::List(v) => assert_eq!(v, &vec!["local-qwen".to_string()]),
TierTargets::One(s) => assert_eq!(s, "local-qwen"),
}
src/pipeline.rs
+13 -6
View File
@@ -232,15 +232,22 @@ impl Pipeline {
match live.router.route(&text, prof).await {
Ok(Some(dec)) => {
let src = dec.source.clone();
let (cands, task_rule) = live
let mut dec = dec;
let (cands, task_rule, forced) = live
.routing
.candidates_for(&dec.task_label, &dec.complexity_tier);
// surface which rule chose the head of the chain
let src = if task_rule {
format!("{src}+task-rule")
} else {
src
// surface which rule chose the head of the chain,
// and when the rule pinned the complexity —
// "…+task-rule@high" — reflect it in the
// decision so headers/stats tell the truth
let src = match (task_rule, &forced) {
(true, Some(tier)) => format!("{src}+task-rule@{tier}"),
(true, None) => format!("{src}+task-rule"),
_ => src,
};
if let Some(tier) = forced {
dec.complexity_tier = tier;
}
(dec, src, cands)
}
Ok(None) | Err(_) => {
src/routing.rs
+77 -18
View File
@@ -4,16 +4,21 @@
use crate::config::{Config, TierTargets};
fn to_list(t: &TierTargets) -> Vec<String> {
match t {
TierTargets::List(v) => v.clone(),
TierTargets::One(s) => vec![s.clone()],
}
t.models()
}
/// A task rule: models that lead the chain, plus an optional FORCED tier —
/// "this task is always this complexity", overriding the router's verdict.
#[derive(Clone)]
pub struct TaskRule {
pub models: Vec<String>,
pub tier: Option<String>,
}
pub struct RoutingTable {
tiers: std::collections::HashMap<String, Vec<String>>,
/// Per-task rules — a matching rule's models lead the candidate chain.
tasks: std::collections::HashMap<String, Vec<String>>,
tasks: std::collections::HashMap<String, TaskRule>,
default: String,
}
@@ -30,8 +35,16 @@ impl RoutingTable {
.routing
.tasks
.iter()
.map(|(task, targets)| (task.clone(), to_list(targets)))
.filter(|(_, v)| !v.is_empty())
.map(|(task, targets)| {
(
task.clone(),
TaskRule {
models: targets.models(),
tier: targets.forced_tier().map(String::from),
},
)
})
.filter(|(_, r)| !r.models.is_empty() || r.tier.is_some())
.collect(),
default: cfg.routing.default.clone(),
}
@@ -41,11 +54,18 @@ impl RoutingTable {
/// the task rule's models first (when one exists), then the tier chain —
/// so a task pin adds preference without ever weakening failover.
/// Returns `(candidates, task_rule_matched)`.
pub fn candidates_for(&self, task: &str, tier: &str) -> (Vec<String>, bool) {
/// Returns `(candidates, rule_matched, forced_tier)`. A rule with a
/// forced tier replaces the router's verdict for this task BEFORE the
/// escalation check — "legal is always high" both escalates and reports
/// itself in route_source.
pub fn candidates_for(&self, task: &str, tier: &str) -> (Vec<String>, bool, Option<String>) {
let mut out: Vec<String> = Vec::new();
let rule = self.tasks.get(task);
let forced = rule.and_then(|r| r.tier.clone());
let eff_tier = forced.as_deref().unwrap_or(tier);
let tier_chain = self
.tiers
.get(tier)
.get(eff_tier)
.cloned()
.unwrap_or_else(|| vec![self.default.clone()]);
// Task rules steer everyday traffic — but a high-complexity verdict
@@ -53,12 +73,12 @@ impl RoutingTable {
// the hardest prompts to the small model forever. At tier=high the
// tier chain leads and the task rule becomes the fallback; otherwise
// the task rule leads as before.
let escalate = tier == "high" && self.tiers.contains_key("high");
let escalate = eff_tier == "high" && self.tiers.contains_key("high");
if escalate {
out.extend(tier_chain.iter().cloned());
}
let matched = if let Some(rule) = self.tasks.get(task) {
for id in rule {
let matched = if let Some(rule) = rule {
for id in &rule.models {
if !out.contains(id) {
out.push(id.clone());
}
@@ -75,7 +95,7 @@ impl RoutingTable {
}
}
out.retain(|id| !id.is_empty());
(out, matched)
(out, matched, forced)
}
/// Candidates for the given complexity tier, in preference order.
@@ -106,12 +126,31 @@ mod tests {
.collect(),
tasks: tasks
.iter()
.map(|(k, v)| (k.to_string(), v.iter().map(|s| s.to_string()).collect()))
.map(|(k, v)| {
(
k.to_string(),
TaskRule {
models: v.iter().map(|s| s.to_string()).collect(),
tier: None,
},
)
})
.collect(),
default: default.to_string(),
}
}
fn with_task_tier(mut t: RoutingTable, task: &str, tier: &str) -> RoutingTable {
t.tasks
.entry(task.to_string())
.or_insert(TaskRule {
models: Vec::new(),
tier: None,
})
.tier = Some(tier.to_string());
t
}
#[test]
fn task_rule_leads_tier_chain_follows() {
let t = table(
@@ -120,7 +159,7 @@ mod tests {
"cheap",
);
// dedup: rule model already in the tier chain appears once, first
let (c, matched) = t.candidates_for("translation", "low");
let (c, matched, _) = t.candidates_for("translation", "low");
assert!(matched);
assert_eq!(c, vec!["local", "cheap"]);
}
@@ -133,20 +172,40 @@ mod tests {
"cheap",
);
// medium (no chain defined → default): task rule still leads
let (c, m) = t.candidates_for("code", "medium");
let (c, m, _) = t.candidates_for("code", "medium");
assert!(m);
assert_eq!(c[0], "cheap-coder");
// high: tier chain leads, task rule follows as fallback
let (c, m) = t.candidates_for("code", "high");
let (c, m, _) = t.candidates_for("code", "high");
assert!(m);
assert_eq!(c, vec!["big", "cheap-coder"]);
}
#[test]
fn forced_tier_overrides_router_verdict() {
let base = table(
&[("legal", &["cheap-para"])],
&[("low", &["cheap"]), ("high", &["big"])],
"cheap",
);
let t = with_task_tier(base, "legal", "high");
// router said low — the rule forces high: escalation chain leads,
// rule models follow, and the forced tier is reported
let (c, m, forced) = t.candidates_for("legal", "low");
assert!(m);
assert_eq!(forced.as_deref(), Some("high"));
assert_eq!(c, vec!["big", "cheap-para"]);
// a task without a rule is untouched
let (_, m, forced) = t.candidates_for("qa", "low");
assert!(!m);
assert!(forced.is_none());
}
#[test]
fn no_rule_falls_back_to_tier_then_default() {
let t = table(&[], &[("low", &["a"])], "d");
assert_eq!(t.candidates_for("qa", "low").0, vec!["a"]);
let (c, matched) = t.candidates_for("qa", "unknown-tier");
let (c, matched, _) = t.candidates_for("qa", "unknown-tier");
assert!(!matched);
assert_eq!(c, vec!["d"]);
}
src/stats.rs
+154 -1
View File
@@ -55,7 +55,7 @@ pub struct RequestRecord {
pub error: Option<String>,
}
#[derive(Clone, Debug, Default, Serialize)]
#[derive(Clone, Debug, Default, Serialize, serde::Deserialize)]
pub struct Totals {
pub requests: u64,
pub ok: u64,
@@ -130,6 +130,15 @@ struct Inner {
/// Per-minute buckets — the single source for the windowed admin snapshot.
buckets: VecDeque<Bucket>,
recent: VecDeque<RequestRecord>,
/// Per-account totals over EVERY record still in the JSONL file (no
/// retention gate) — per-key accounting for the keys page.
by_account_alltime: HashMap<String, Totals>,
/// Per-account totals folded out of the file by compaction. alltime(key)
/// = folded + by_account_alltime, with no double counting: a record is
/// either in the file or folded, never both.
folded: HashMap<String, Totals>,
/// Appends since the last compaction size check.
writes_since_check: u32,
}
pub struct Stats {
@@ -137,6 +146,7 @@ pub struct Stats {
file: Option<String>,
ring_size: usize,
retention_secs: u64,
max_file_bytes: u64,
inner: Mutex<Inner>,
}
@@ -152,9 +162,12 @@ impl Stats {
file,
ring_size: cfg.ring_size.max(1),
retention_secs: parse_duration_secs(&cfg.retention).unwrap_or(7 * 86_400),
max_file_bytes: cfg.max_file_mb.saturating_mul(1024 * 1024),
inner: Mutex::new(Inner::default()),
};
s.load_folded();
s.replay();
s.maybe_compact(true);
std::sync::Arc::new(s)
}
@@ -175,6 +188,11 @@ impl Stats {
// push everything to the "recent" ring (it self-limits by size),
// but apply to aggregates/series only entries within the window.
Self::push_recent(&mut inner, &rec, self.ring_size);
inner
.by_account_alltime
.entry(rec.account.clone())
.or_default()
.apply(&rec);
if rec.ts >= cutoff {
Self::apply_aggregates(&mut inner, &rec, self.retention_secs);
}
@@ -233,6 +251,12 @@ impl Stats {
let mut inner = self.inner.lock().unwrap();
Self::apply_aggregates(&mut inner, &rec, self.retention_secs);
Self::push_recent(&mut inner, &rec, self.ring_size);
inner
.by_account_alltime
.entry(rec.account.clone())
.or_default()
.apply(&rec);
inner.writes_since_check += 1;
}
if let Some(path) = &self.file {
if let Ok(line) = serde_json::to_string(&rec) {
@@ -245,6 +269,112 @@ impl Stats {
}
}
}
self.maybe_compact(false);
}
fn totals_path(&self) -> Option<String> {
self.file.as_ref().map(|f| format!("{f}.totals.json"))
}
fn load_folded(&self) {
let Some(path) = self.totals_path() else {
return;
};
let Ok(content) = std::fs::read_to_string(&path) else {
return;
};
if let Ok(m) = serde_json::from_str::<HashMap<String, Totals>>(&content) {
self.inner.lock().unwrap().folded = m;
}
}
/// Keep the JSONL bounded: once it outgrows `max_file_bytes`, records
/// older than the retention window fold into the cumulative per-account
/// sidecar and are dropped from the file (tmp + rename on both files, so
/// a crash mid-compaction loses nothing). `force` checks size regardless
/// of the write counter (startup).
fn maybe_compact(&self, force: bool) {
if self.max_file_bytes == 0 {
return;
}
let Some(path) = &self.file else { return };
{
let mut inner = self.inner.lock().unwrap();
if !force {
if inner.writes_since_check < 256 {
return;
}
inner.writes_since_check = 0;
}
}
let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
if size <= self.max_file_bytes {
return;
}
let Ok(content) = std::fs::read_to_string(path) else {
return;
};
let cutoff = now_secs().saturating_sub(self.retention_secs);
let mut kept = String::with_capacity(content.len() / 2);
let mut inner = self.inner.lock().unwrap();
for line in content.lines() {
let line = line.trim();
if line.is_empty() {
continue;
}
match serde_json::from_str::<RequestRecord>(line) {
Ok(rec) if rec.ts < cutoff => {
inner
.folded
.entry(rec.account.clone())
.or_default()
.apply(&rec);
// fold = leave by_account_alltime (record leaves the file,
// its share moves from "alltime" into "folded")
if let Some(t) = inner.by_account_alltime.get_mut(&rec.account) {
// subtract by rebuilding is costly; instead track the
// invariant additively: remove the record's share
t.requests = t.requests.saturating_sub(1);
if rec.outcome == "ok" {
t.ok = t.ok.saturating_sub(1);
} else {
t.errors = t.errors.saturating_sub(1);
}
if rec.failover {
t.failovers = t.failovers.saturating_sub(1);
}
t.prompt_tokens = t.prompt_tokens.saturating_sub(rec.prompt_tokens as u64);
t.completion_tokens = t
.completion_tokens
.saturating_sub(rec.completion_tokens as u64);
t.cost_usd = (t.cost_usd - rec.cost_usd).max(0.0);
t.latency_ms_sum = t.latency_ms_sum.saturating_sub(rec.latency_ms);
}
}
Ok(_) => {
kept.push_str(line);
kept.push('\n');
}
Err(_) => { /* unparseable line — drop */ }
}
}
// persist the folded totals FIRST, then swap the trimmed file in
if let Some(tpath) = self.totals_path() {
if let Ok(json) = serde_json::to_string(&inner.folded) {
let tmp = format!("{tpath}.tmp");
if std::fs::write(&tmp, json).is_ok() {
let _ = std::fs::rename(&tmp, &tpath);
}
}
}
let tmp = format!("{path}.tmp");
if std::fs::write(&tmp, kept).is_ok() {
let _ = std::fs::rename(&tmp, path);
}
tracing::info!(
new_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0),
"stats log compacted (older records folded into cumulative totals)"
);
}
/// Snapshot for the admin API: aggregates + time series for `range_secs` + breakdown by `groupby`.
@@ -303,7 +433,27 @@ impl Stats {
.cmp(&a["requests"].as_u64().unwrap_or(0))
});
let mut alltime: HashMap<String, Totals> = inner.folded.clone();
for (k, t) in &inner.by_account_alltime {
alltime.entry(k.clone()).or_default().merge(t);
}
let alltime_by_account: serde_json::Map<String, serde_json::Value> = alltime
.iter()
.map(|(k, t)| {
(
k.clone(),
serde_json::json!({
"requests": t.requests,
"prompt_tokens": t.prompt_tokens,
"completion_tokens": t.completion_tokens,
"cost_usd": t.cost_usd,
}),
)
})
.collect();
serde_json::json!({
"alltime_by_account": alltime_by_account,
"totals": {
"requests": win.requests,
"ok": win.ok,
@@ -327,6 +477,9 @@ impl Stats {
/// Reset all in-memory stats and truncate the JSONL log — the "clear logs"
/// admin action.
pub fn clear(&self) {
if let Some(tpath) = self.totals_path() {
let _ = std::fs::remove_file(tpath);
}
{
let mut inner = self.inner.lock().unwrap();
*inner = Inner::default();