feat: ops & UX batch — jobs survive restarts, logs/system in the console, media queue, gallery, chat, exact stream tokens
Fifteen changes aimed at the first day of real use: - Import and media jobs persist to <data>/jobs-*.json and reload on start (running ones settle as errors); the Register button no longer vanishes because the gateway restarted. - Docker ships the cortiq binary (built in the builder stage) — local .cmf chat models and image/video generation now work in containers, where auto-install never could (no cargo in the runtime image). libvulkan1 added for optional GPU offload. - Open admin console on 0.0.0.0 without a token now warns in the log and shows a dismissible banner — the VPS footgun is at least labeled. - "Restart gateway" button in Settings: stops managed model servers, spawns a fresh copy (bind retried while the port frees), Docker relies on the container restart policy instead. - Dashboard "System" panel: CPU sparkline + load, RAM (with the gateway's own RSS), free disk at the models dir — sampled by sysinfo every 2 s. - Featured model cards warn "not enough disk" / "may not fit in RAM" before the download, from the same live snapshot. - Log viewer in the sidebar: a 500-line in-memory ring fed by a second ANSI-free tracing layer; filter, level, pause, copy. - Update check: crates.io polled twice a day, the console shows an "update available" notice. - Media generations queue up (with position shown) instead of being rejected; progress gained an honest ETA from measured throughput. - Generation gallery under the image/video tabs: lazy PPM thumbnails, click to reopen with seed/steps, delete. - Playground chat is a real dialogue now: history is resent, bubbles, Clear, Cmd/Ctrl+Enter. - "Connect your app" snippets (curl / Python / JS) on the keys page, with the real key inlined right after creation. - One-click benchmark on the Models page via `cortiq bench --json`. - Config export now downloads the actual gateway.toml (comments intact); import accepts TOML (validated, hot-reloaded) and the legacy JSON. - Streaming token counts are exact when the model server provides usage (cortiq-server patch, shipped separately); the gateway fallback now counts content chunks (1/token on cortiq) instead of chars/4. Wizard's Finish lands on the Playground. 37 new strings in all 7 languages.
This commit is contained in:
+166
@@ -0,0 +1,166 @@
|
||||
//! One-click local model benchmark: run `cortiq bench <model> --json` and keep
|
||||
//! the parsed result per model, so the Models page can show real tok/s numbers
|
||||
//! measured on this machine.
|
||||
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use std::process::Stdio;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
#[derive(Clone, Serialize)]
|
||||
pub struct BenchRun {
|
||||
pub model: String, // file stem under models_dir
|
||||
pub state: String, // running | done | error
|
||||
pub log: Vec<String>,
|
||||
pub tok_s: Option<f64>,
|
||||
/// The full JSON object the benchmark printed, passed through untouched.
|
||||
pub raw: Option<serde_json::Value>,
|
||||
pub started: u64,
|
||||
pub finished: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct BenchStore {
|
||||
runs: Mutex<HashMap<String, BenchRun>>,
|
||||
busy: Mutex<bool>,
|
||||
}
|
||||
|
||||
fn now() -> u64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
/// tok/s out of the benchmark's JSON: first numeric field whose key mentions
|
||||
/// tokens-per-second (the exact name has varied across cortiq releases).
|
||||
fn extract_tok_s(v: &serde_json::Value) -> Option<f64> {
|
||||
let obj = v.as_object()?;
|
||||
for want in ["tok_s", "tok_per_s", "tokens_per_sec", "decode_tok_s"] {
|
||||
if let Some(n) = obj.get(want).and_then(|x| x.as_f64()) {
|
||||
return Some(n);
|
||||
}
|
||||
}
|
||||
obj.iter()
|
||||
.find(|(k, val)| {
|
||||
let k = k.to_ascii_lowercase();
|
||||
(k.contains("tok") && (k.contains("/s") || k.contains("_s") || k.contains("per")))
|
||||
&& val.as_f64().is_some()
|
||||
})
|
||||
.and_then(|(_, val)| val.as_f64())
|
||||
}
|
||||
|
||||
impl BenchStore {
|
||||
pub fn new() -> Arc<Self> {
|
||||
Arc::new(Self::default())
|
||||
}
|
||||
pub fn get(&self, model: &str) -> Option<BenchRun> {
|
||||
self.runs.lock().unwrap().get(model).cloned()
|
||||
}
|
||||
pub fn list(&self) -> Vec<BenchRun> {
|
||||
self.runs.lock().unwrap().values().cloned().collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Kick off a benchmark for `<models_dir>/<model>.cmf`. One at a time — a
|
||||
/// benchmark loads the whole model and owns the machine while it runs.
|
||||
pub fn start(
|
||||
store: Arc<BenchStore>,
|
||||
cmf: &crate::config::CmfCfg,
|
||||
model: String,
|
||||
) -> Result<(), String> {
|
||||
let path = std::path::Path::new(&cmf.models_dir).join(format!("{model}.cmf"));
|
||||
if !path.exists() {
|
||||
return Err(format!("model file not found: {}", path.display()));
|
||||
}
|
||||
{
|
||||
let mut busy = store.busy.lock().unwrap();
|
||||
if *busy {
|
||||
return Err("a benchmark is already running — wait for it to finish".into());
|
||||
}
|
||||
*busy = true;
|
||||
}
|
||||
store.runs.lock().unwrap().insert(
|
||||
model.clone(),
|
||||
BenchRun {
|
||||
model: model.clone(),
|
||||
state: "running".into(),
|
||||
log: Vec::new(),
|
||||
tok_s: None,
|
||||
raw: None,
|
||||
started: now(),
|
||||
finished: None,
|
||||
},
|
||||
);
|
||||
let bin = cmf.cortiq_bin.clone();
|
||||
let gpu = cmf.gpu;
|
||||
tokio::spawn(async move {
|
||||
let out = tokio::process::Command::new(&bin)
|
||||
.arg("bench")
|
||||
.arg(&path)
|
||||
.arg("--json")
|
||||
.env("CMF_GPU", if gpu { "1" } else { "0" })
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.output()
|
||||
.await;
|
||||
let mut g = store.runs.lock().unwrap();
|
||||
let run = g.get_mut(&model).unwrap();
|
||||
run.finished = Some(now());
|
||||
match out {
|
||||
Ok(o) => {
|
||||
let text = format!(
|
||||
"{}\n{}",
|
||||
String::from_utf8_lossy(&o.stdout),
|
||||
String::from_utf8_lossy(&o.stderr)
|
||||
);
|
||||
// keep a readable tail; the JSON line is parsed separately
|
||||
run.log = text
|
||||
.lines()
|
||||
.map(|l| l.trim())
|
||||
.filter(|l| !l.is_empty() && !l.contains("Metal GPU path"))
|
||||
.map(String::from)
|
||||
.collect::<Vec<_>>();
|
||||
let n = run.log.len();
|
||||
if n > 30 {
|
||||
run.log.drain(0..n - 30);
|
||||
}
|
||||
// the --json contract: one JSON object on its own line
|
||||
let parsed = text
|
||||
.lines()
|
||||
.rev()
|
||||
.map(str::trim)
|
||||
.find(|l| l.starts_with('{') && l.ends_with('}'))
|
||||
.and_then(|l| serde_json::from_str::<serde_json::Value>(l).ok());
|
||||
match parsed {
|
||||
Some(v) if o.status.success() => {
|
||||
run.tok_s = extract_tok_s(&v);
|
||||
run.raw = Some(v);
|
||||
run.state = "done".into();
|
||||
}
|
||||
_ => run.state = "error".into(),
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
run.log.push(format!("✗ failed to start cortiq bench: {e}"));
|
||||
run.state = "error".into();
|
||||
}
|
||||
}
|
||||
drop(g);
|
||||
*store.busy.lock().unwrap() = false;
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn tok_s_from_common_keys() {
|
||||
let v = serde_json::json!({"tok_s": 42.5, "allocs_per_token": 0});
|
||||
assert_eq!(extract_tok_s(&v), Some(42.5));
|
||||
let v = serde_json::json!({"decode_tokens_per_sec": 31.0});
|
||||
assert_eq!(extract_tok_s(&v), Some(31.0));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user