Compare commits

...
4 Commits
Author SHA1 Message Date
infosave2007 9d2181e82b chore(release): v0.2.7 — honest pins, visible managed model, physical delete
Docker / publish (push) Canceled after 0s
Release / crates-io (push) Canceled after 0s
Release / release (macos-14, aarch64-apple-darwin) (push) Canceled after 0s
Release / release (ubuntu-latest, x86_64-unknown-linux-gnu) (push) Canceled after 0s
Release / release (windows-latest, x86_64-pc-windows-msvc) (push) Canceled after 0s
- A pinned playground model that is unavailable now errors clearly instead of
  silently answering from the managed local model (route_source no longer lies).
- The managed CMF model is surfaced in GET /admin/api/models (marked "managed",
  shown read-only in the UI) so the real running model is visible and pinnable
  without a hand-maintained [[models]] entry; a fresh install still lists nothing.
- Deleting a model now removes its local .cmf physically (file or tensor-dir);
  deleting the managed model also stops the server and clears the [cmf] config.
  Routing is stripped of the deleted id so the config still validates.
2026-07-08 21:14:03 +03:00
infosave2007 30378f7b6a chore: fix clippy lints to green the lint CI
- import.rs / promotion.rs: sort_by(|a,b| b.k.cmp(&a.k)) -> sort_by_key(Reverse(k))
- pipeline.rs: skip_while(!p).next() -> find(p)
Behaviour-preserving; completes the CI fix started by the cargo fmt commit.
2026-07-08 20:09:54 +03:00
infosave2007 b4f78bd192 chore: cargo fmt — fix lint 'Format' CI job (red since v0.2.3)
Whole-repo rustfmt drift; behaviour unchanged (whitespace/layout only).
All test platforms already passed; this greens the lint job.
2026-07-08 19:32:35 +03:00
infosave2007 3c90dad9b9 chore(release): v0.2.6 — Settings UX, empty pool by default, CMF port check
Docker / publish (push) Canceled after 0s
Release / crates-io (push) Canceled after 0s
Release / release (macos-14, aarch64-apple-darwin) (push) Canceled after 0s
Release / release (ubuntu-latest, x86_64-unknown-linux-gnu) (push) Canceled after 0s
Release / release (windows-latest, x86_64-pc-windows-msvc) (push) Canceled after 0s
- Settings: "?" help tooltips on every option; router key auto-probes the
  connection right after saving (no more hanging "unavailable")
- Local models card reworked into 3 clear steps (install cortiq → add model →
  run locally); removed the hand-entered .cmf path / pool-id fields — models
  are added via the Import tab and the managed server registers them
- Fresh install ships an EMPTY model pool; the local .cmf is auto-registered by
  the managed CMF server, so it needs no static [[models]] entry
- New: port-availability check — GET /admin/api/cmf/port + "Check port" button
  with next-free-port suggestion. bind()+connect() probe so Docker-proxied /
  dual-stack ports (e.g. 8081) are correctly detected as occupied
- /admin/api/cmf now reports the serving model + port; card shows "▶ serving …"
- On-demand runtime install: POST /admin/api/cmf/install
2026-07-08 19:07:24 +03:00
16 changed files with 763 additions and 157 deletions
Generated
+1 -1
View File
@@ -285,7 +285,7 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
[[package]]
name = "cortiq-gateway"
version = "0.2.5"
version = "0.2.7"
dependencies = [
"anyhow",
"arc-swap",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "cortiq-gateway"
version = "0.2.5"
version = "0.2.7"
edition = "2021"
rust-version = "1.75"
license = "Apache-2.0"
+1 -1
View File
@@ -142,7 +142,7 @@ max_entries = 1000
manage_server = false # true → gateway spawns `cortiq serve` at startup
local_model = "models/model.cmf" # path to the .cmf to serve
local_host = "127.0.0.1"
local_port = 8081 # must match the cmf-local base_url above
local_port = 8081 # confirm it's free in Settings → "Check port" (8081 often clashes with Docker/Apache)
model_id = "cmf-local" # id it registers under (matches [[models]])
threads = 8 # decode worker threads (CMF_THREADS); ~3x vs serial
gpu = false # offload matvecs to GPU (Metal/wgpu); mainly a prefill win
+215 -19
View File
@@ -26,6 +26,7 @@ use axum::{
use serde::Deserialize;
use serde_json::{json, Value};
use std::collections::HashMap;
use std::net::ToSocketAddrs;
use std::time::Instant;
// ───────────────────────────── common types / helpers ───────────────────────
@@ -168,9 +169,15 @@ pub fn api_routes(admin_token: String) -> Router<SharedState> {
.route("/admin/api/shadow", get(get_shadow))
.route("/admin/api/hf/search", get(hf_search))
.route("/admin/api/import", get(list_imports).post(start_import))
.route("/admin/api/import/:job", get(import_status).delete(delete_import))
.route(
"/admin/api/import/:job",
get(import_status).delete(delete_import),
)
.route("/admin/api/import/:job/cancel", post(cancel_import))
.route("/admin/api/import/:job/register", post(register_import))
.route("/admin/api/cmf", get(cmf_status))
.route("/admin/api/cmf/install", post(cmf_install))
.route("/admin/api/cmf/port", get(cmf_port_check))
.route("/admin/api/requests", get(get_requests))
.route("/admin/api/test", post(run_test))
.route("/admin/api/test/stream", post(run_test_stream))
@@ -419,7 +426,7 @@ async fn put_config(State(state): State<SharedState>, Json(cfg): Json<Config>) -
async fn list_models(State(state): State<SharedState>) -> ApiResult {
let live = state.live();
let models: Vec<Value> = live
let mut models: Vec<Value> = live
.cfg
.models
.iter()
@@ -433,6 +440,31 @@ async fn list_models(State(state): State<SharedState>) -> ApiResult {
v
})
.collect();
// The managed local CMF server is registered in the routing registry from
// the [cmf] flags, not from [[models]]. Surface it here (marked managed, so
// the UI shows it read-only) so the real running model is visible and
// pinnable without needing a hand-maintained static pool entry. A fresh
// install with no managed model still returns an empty list.
let cmf = &live.cfg.cmf;
if cmf.manage_server
&& !cmf.local_model.trim().is_empty()
&& !models.iter().any(|m| m["id"] == json!(cmf.model_id))
{
models.push(json!({
"id": cmf.model_id,
"provider": "openai",
"base_url": format!("http://{}:{}/v1", cmf.local_host, cmf.local_port),
"model": "cortiq",
"cost_tier": "local",
"price_in": 0.0,
"price_out": 0.0,
"kind": "chat",
"caps": [],
"key_source": "none",
"managed": true,
}));
}
ok(json!({ "models": models }))
}
@@ -466,15 +498,86 @@ async fn update_model(
ok(json!({ "ok": true }))
}
/// Remove a `.cmf` path — it may be a plain file or a tensor-dir. Returns whether
/// something was actually removed.
fn remove_cmf_path(path: &std::path::Path) -> bool {
if path.is_dir() {
std::fs::remove_dir_all(path).is_ok()
} else if path.is_file() {
std::fs::remove_file(path).is_ok()
} else {
false
}
}
/// The local `.cmf` backing a pool model, if it is a local CMF model whose file
/// lives under `models_dir`. Cloud models (real remote base_url) return None.
/// Registered imports store `model` = the file stem under models_dir.
fn cmf_file_for_model(
cmf: &crate::config::CmfCfg,
m: &crate::config::ModelCfg,
) -> Option<std::path::PathBuf> {
let local = m.base_url == cmf.cortiq_server_url
|| m.base_url
.contains(&format!("{}:{}", cmf.local_host, cmf.local_port));
let stem = m.model.trim();
if !local || stem.is_empty() || stem.contains('/') || stem.contains("..") {
return None; // not local, or an unsafe/underivable path
}
Some(std::path::Path::new(&cmf.models_dir).join(format!("{stem}.cmf")))
}
/// Drop a model id from routing (default + every tier) so `validate()` still
/// passes after the model is gone.
fn strip_id_from_routing(routing: &mut RoutingCfg, id: &str) {
if routing.default == id {
routing.default = String::new();
}
for tt in routing.tiers.values_mut() {
match tt {
crate::config::TierTargets::List(v) => v.retain(|x| x != id),
crate::config::TierTargets::One(s) if s == id => {
*tt = crate::config::TierTargets::List(Vec::new());
}
crate::config::TierTargets::One(_) => {}
}
}
}
async fn delete_model(State(state): State<SharedState>, Path(id): Path<String>) -> ApiResult {
let mut cfg = current_cfg(&state);
// The managed local model lives in [cmf], not [[models]]. Deleting it stops
// the running server, clears the managed config, and removes its file.
if id == cfg.cmf.model_id
&& cfg.cmf.manage_server
&& !cfg.cmf.local_model.trim().is_empty()
&& !cfg.models.iter().any(|x| x.id == id)
{
let file = cfg.cmf.local_model.clone();
cfg.cmf.local_model = String::new();
cfg.cmf.manage_server = false;
strip_id_from_routing(&mut cfg.routing, &id);
state.cmf.stop().await;
state.reload(cfg)?;
let file_removed = remove_cmf_path(std::path::Path::new(&file));
return ok(json!({ "ok": true, "file_removed": file_removed }));
}
let victim = cfg.models.iter().find(|x| x.id == id).cloned();
let before = cfg.models.len();
cfg.models.retain(|x| x.id != id);
if cfg.models.len() == before {
return Err(ApiError::not_found(format!("model '{id}' not found")));
}
strip_id_from_routing(&mut cfg.routing, &id);
let file = victim
.as_ref()
.and_then(|m| cmf_file_for_model(&cfg.cmf, m));
state.reload(cfg)?;
ok(json!({ "ok": true }))
// Physically remove the local .cmf after the config is persisted.
let file_removed = file.map(|p| remove_cmf_path(&p)).unwrap_or(false);
ok(json!({ "ok": true, "file_removed": file_removed }))
}
async fn probe_model(State(state): State<SharedState>, Path(id): Path<String>) -> ApiResult {
@@ -671,6 +774,111 @@ async fn put_settings(State(state): State<SharedState>, Json(b): Json<SettingsBo
ok(json!({ "ok": true, "needs_restart": needs_restart }))
}
// ───────────────────────────── cmf runtime ─────────────────────────────────
/// Status of the `cortiq` CLI (the CMF format runtime): is it installed, which
/// version, and whether a managed server is running. Drives the "Install cortiq"
/// button + status in the Settings → Local models card.
async fn cmf_status(State(state): State<SharedState>) -> ApiResult {
let cmf = state.live().cfg.cmf.clone();
let ver = crate::cmf_runtime::installed_version(&cmf.cortiq_bin);
let st = state.cmf.status();
ok(json!({
"installed": ver.is_some(),
"version": ver,
"running": st.running,
"healthy": st.healthy,
"last_error": st.last_error,
"log": st.log,
// serving info so the card can show "serving <model> on :<port>"
"manage_server": cmf.manage_server,
"local_model": cmf.local_model,
"model_id": cmf.model_id,
"local_host": cmf.local_host,
"local_port": cmf.local_port,
}))
}
/// Install/upgrade the `cortiq` CLI from crates.io on demand (spawned in the
/// background; poll `GET /admin/api/cmf` for `installed`/`version`).
async fn cmf_install(State(state): State<SharedState>) -> ApiResult {
let bin = state.live().cfg.cmf.cortiq_bin.clone();
tokio::spawn(crate::cmf_runtime::install_now(state.cmf.clone(), bin));
ok(json!({ "started": true }))
}
#[derive(Deserialize)]
struct PortQuery {
port: u16,
}
/// Check whether a TCP port is free to bind on the CMF local host, so a user can
/// pick an available port before enabling the managed server (avoids clashing
/// with something already on the default 8081). If the port is the one this
/// gateway's own managed server already runs on, that is reported distinctly.
/// When occupied, `suggested` holds the next free port so the UI can offer it.
async fn cmf_port_check(State(state): State<SharedState>, Query(q): Query<PortQuery>) -> ApiResult {
let cmf = state.live().cfg.cmf.clone();
let host = if cmf.local_host.trim().is_empty() {
"127.0.0.1".to_string()
} else {
cmf.local_host.clone()
};
let st = state.cmf.status();
let port = q.port;
let running_here = port == cmf.local_port && st.running;
// The bind+connect probe is blocking std::net; run it off the async runtime.
let (available, detail, suggested) = tokio::task::spawn_blocking(move || {
if running_here {
return (false, "in use by this gateway's managed CMF server", None);
}
if port_is_free(&host, port) {
(true, "free", None)
} else {
(
false,
"occupied by another process",
next_free_port(&host, port),
)
}
})
.await
.unwrap_or((false, "port check failed", None));
ok(json!({
"port": port,
"available": available,
"detail": detail,
"suggested": suggested,
}))
}
/// Whether `port` is truly free on `host`. A bind test alone is not enough: a
/// Docker-published / dual-stack listener can leave an IPv4 bind succeeding even
/// though connections to the port are forwarded to a container. So a port counts
/// as free only if it is BOTH bindable AND nothing answers a connect attempt.
fn port_is_free(host: &str, port: u16) -> bool {
if std::net::TcpListener::bind((host, port)).is_err() {
return false; // something is bound to exactly this address
}
// catch proxied/forwarded ports a bind test misses (e.g. Docker on macOS)
let reachable = (host, port)
.to_socket_addrs()
.ok()
.and_then(|mut it| it.next())
.map(|addr| {
std::net::TcpStream::connect_timeout(&addr, std::time::Duration::from_millis(200))
.is_ok()
})
.unwrap_or(false);
!reachable
}
/// First free port in (from, from+50], or None if all are taken.
fn next_free_port(host: &str, from: u16) -> Option<u16> {
(from.saturating_add(1)..=from.saturating_add(50)).find(|p| port_is_free(host, *p))
}
// ───────────────────────────── keys ────────────────────────────────────────
async fn list_keys(State(state): State<SharedState>) -> ApiResult {
@@ -869,10 +1077,7 @@ async fn start_import(
ok(json!({ "job": id }))
}
async fn import_status(
State(state): State<SharedState>,
Path(job): Path<String>,
) -> ApiResult {
async fn import_status(State(state): State<SharedState>, Path(job): Path<String>) -> ApiResult {
match state.imports.get(&job) {
Some(j) => ok(serde_json::to_value(j).unwrap_or_else(|_| json!({}))),
None => Err(ApiError::not_found(format!("job '{job}' not found"))),
@@ -880,10 +1085,7 @@ async fn import_status(
}
/// Cancel a running conversion: kills the converter and removes partial output.
async fn cancel_import(
State(state): State<SharedState>,
Path(job): Path<String>,
) -> ApiResult {
async fn cancel_import(State(state): State<SharedState>, Path(job): Path<String>) -> ApiResult {
if state.imports.get(&job).is_none() {
return Err(ApiError::not_found(format!("job '{job}' not found")));
}
@@ -892,10 +1094,7 @@ async fn cancel_import(
}
/// Delete a finished conversion and its converted `.cmf` file(s) from disk.
async fn delete_import(
State(state): State<SharedState>,
Path(job): Path<String>,
) -> ApiResult {
async fn delete_import(State(state): State<SharedState>, Path(job): Path<String>) -> ApiResult {
if state.imports.get(&job).is_none() {
return Err(ApiError::not_found(format!("job '{job}' not found")));
}
@@ -904,10 +1103,7 @@ async fn delete_import(
}
/// Register a finished `.cmf` as a local model (OpenAI provider → cortiq-server).
async fn register_import(
State(state): State<SharedState>,
Path(job): Path<String>,
) -> ApiResult {
async fn register_import(State(state): State<SharedState>, Path(job): Path<String>) -> ApiResult {
let j = state
.imports
.get(&job)
+41 -9
View File
@@ -113,14 +113,15 @@ async fn latest_crates_version() -> Option<String> {
.await
.ok()?;
let v: serde_json::Value = serde_json::from_str(&body).ok()?;
v["crate"]["newest_version"]
.as_str()
.map(|s| s.to_string())
v["crate"]["newest_version"].as_str().map(|s| s.to_string())
}
/// `<bin> --version` → the trailing version token (e.g. "cortiq 0.1.2" → "0.1.2").
fn installed_version(bin: &str) -> Option<String> {
let out = std::process::Command::new(bin).arg("--version").output().ok()?;
pub(crate) fn installed_version(bin: &str) -> Option<String> {
let out = std::process::Command::new(bin)
.arg("--version")
.output()
.ok()?;
if !out.status.success() {
return None;
}
@@ -130,7 +131,11 @@ fn installed_version(bin: &str) -> Option<String> {
fn parse_ver(s: &str) -> (u32, u32, u32) {
let mut it = s.split('.').map(|x| x.trim().parse::<u32>().unwrap_or(0));
(it.next().unwrap_or(0), it.next().unwrap_or(0), it.next().unwrap_or(0))
(
it.next().unwrap_or(0),
it.next().unwrap_or(0),
it.next().unwrap_or(0),
)
}
fn version_lt(a: &str, b: &str) -> bool {
@@ -152,6 +157,24 @@ async fn cargo_install(force: bool) -> Result<(), String> {
}
}
/// Install/upgrade `cortiq-cli` from crates.io on demand (the admin "Install
/// cortiq" button). Runs in the background; progress + result land in the CMF
/// status log, and `installed_version` is refreshed on success.
pub async fn install_now(rt: Arc<CmfRuntime>, bin: String) {
rt.log("installing cortiq-cli from crates.io (this compiles — a few minutes)…");
match cargo_install(true).await {
Ok(()) => {
let v = installed_version(&bin);
rt.status.lock().unwrap().installed_version = v.clone();
rt.log(format!(
"installed cortiq-cli {}",
v.as_deref().unwrap_or("?")
));
}
Err(e) => rt.fail(format!("cargo install cortiq-cli failed: {e}")),
}
}
/// Spawn `cortiq serve <model> --host <host> --port <port>`; the child is killed
/// when its handle is dropped (i.e. when the gateway shuts down).
fn spawn_serve(cfg: &CmfCfg) -> Result<Child, String> {
@@ -230,7 +253,9 @@ pub async fn manage(rt: Arc<CmfRuntime>, cfg: CmfCfg) {
rt.log("installed cortiq-cli");
}
} else {
rt.fail("cortiq binary not found (set [cmf].auto_install = true, or install cortiq-cli)");
rt.fail(
"cortiq binary not found (set [cmf].auto_install = true, or install cortiq-cli)",
);
return;
}
}
@@ -240,8 +265,15 @@ pub async fn manage(rt: Arc<CmfRuntime>, cfg: CmfCfg) {
if cfg.auto_update {
if let Some(latest) = latest_crates_version().await {
rt.status.lock().unwrap().latest_version = Some(latest.clone());
if ver.as_deref().map(|c| version_lt(c, &latest)).unwrap_or(false) {
rt.log(format!("updating cortiq-cli {} → {latest}…", ver.as_deref().unwrap_or("?")));
if ver
.as_deref()
.map(|c| version_lt(c, &latest))
.unwrap_or(false)
{
rt.log(format!(
"updating cortiq-cli {} → {latest}…",
ver.as_deref().unwrap_or("?")
));
if let Err(e) = cargo_install(true).await {
rt.log(format!("update failed (keeping current): {e}"));
} else {
+10 -2
View File
@@ -10,10 +10,18 @@ use std::path::PathBuf;
/// `~/.cortiq-gateway` — so defaults work regardless of the process CWD
/// (important once the gateway is installed rather than run from its repo).
pub fn data_dir() -> PathBuf {
for (var, sub) in [("CORTIQ_GATEWAY_HOME", ""), ("XDG_DATA_HOME", "cortiq-gateway"), ("HOME", ".cortiq-gateway")] {
for (var, sub) in [
("CORTIQ_GATEWAY_HOME", ""),
("XDG_DATA_HOME", "cortiq-gateway"),
("HOME", ".cortiq-gateway"),
] {
if let Ok(v) = std::env::var(var) {
if !v.is_empty() {
return if sub.is_empty() { PathBuf::from(v) } else { PathBuf::from(v).join(sub) };
return if sub.is_empty() {
PathBuf::from(v)
} else {
PathBuf::from(v).join(sub)
};
}
}
}
+96 -29
View File
@@ -31,13 +31,19 @@ fn cleanup_output(path: &str) {
}
fn now() -> u64 {
SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs()
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
/// Job id from a hash of (repo, nanos) — unique without a uuid dependency.
fn gen_id(repo: &str) -> String {
use std::hash::{Hash, Hasher};
let nanos = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos();
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_nanos();
let mut h = std::collections::hash_map::DefaultHasher::new();
repo.hash(&mut h);
nanos.hash(&mut h);
@@ -82,8 +88,8 @@ pub struct Job {
pub started: u64,
pub finished: Option<u64>,
pub size_bytes: Option<u64>,
pub progress: Option<f32>, // 0..1, parsed from converter @PROGRESS
pub phase: Option<String>, // current phase label
pub progress: Option<f32>, // 0..1, parsed from converter @PROGRESS
pub phase: Option<String>, // current phase label
}
#[derive(Default)]
@@ -178,16 +184,24 @@ impl JobStore {
}
pub fn list(&self) -> Vec<Job> {
let mut v: Vec<Job> = self.jobs.lock().unwrap().values().cloned().collect();
v.sort_by(|a, b| b.started.cmp(&a.started));
v.sort_by_key(|x| std::cmp::Reverse(x.started));
v.truncate(20);
v
}
}
/// Proxy HuggingFace model search (avoids browser CORS + adds our token).
pub async fn hf_search(query: &str, limit: usize, token: Option<&str>) -> Result<serde_json::Value, String> {
pub async fn hf_search(
query: &str,
limit: usize,
token: Option<&str>,
) -> Result<serde_json::Value, String> {
let q: String = url_escape(query);
let sort = if query.trim().is_empty() { "trendingScore" } else { "downloads" };
let sort = if query.trim().is_empty() {
"trendingScore"
} else {
"downloads"
};
let url = format!(
"https://huggingface.co/api/models?search={q}&sort={sort}&direction=-1&limit={limit}&full=false"
);
@@ -218,7 +232,13 @@ fn url_escape(s: &str) -> String {
fn sanitize(s: &str) -> String {
s.chars()
.map(|c| if c.is_ascii_alphanumeric() || c == '-' || c == '_' { c } else { '-' })
.map(|c| {
if c.is_ascii_alphanumeric() || c == '-' || c == '_' {
c
} else {
'-'
}
})
.collect()
}
@@ -244,7 +264,10 @@ pub fn start_import(store: Arc<JobStore>, cfg: &CmfCfg, p: ImportParams) -> Resu
// doesn't support yet (linear-attention folding, v-bit shaping).
let use_python = p.linear_core.is_some() || p.vbit_shape.is_some() || p.mean_bits.is_some();
if use_python && !std::path::Path::new(&cfg.converter).exists() {
return Err(format!("advanced options need the Python converter, not found: {}", cfg.converter));
return Err(format!(
"advanced options need the Python converter, not found: {}",
cfg.converter
));
}
let _ = std::fs::create_dir_all(&cfg.models_dir);
let base = if p.name.trim().is_empty() {
@@ -288,18 +311,43 @@ pub fn start_import(store: Arc<JobStore>, cfg: &CmfCfg, p: ImportParams) -> Resu
let (program, args, workdir): (String, Vec<String>, std::path::PathBuf) = if use_python {
let conv = std::path::Path::new(&cfg.converter);
let mut a = vec![
std::fs::canonicalize(conv).map(|p| p.to_string_lossy().to_string()).unwrap_or_else(|_| cfg.converter.clone()),
"--model".into(), p.repo.clone(),
"--quant".into(), p.quant.clone(),
"--output".into(), output_abs.clone(),
std::fs::canonicalize(conv)
.map(|p| p.to_string_lossy().to_string())
.unwrap_or_else(|_| cfg.converter.clone()),
"--model".into(),
p.repo.clone(),
"--quant".into(),
p.quant.clone(),
"--output".into(),
output_abs.clone(),
];
if let Some(lc) = &p.linear_core { a.push("--linear-core".into()); a.push(lc.clone()); }
if let Some(n) = p.nphase { a.push("--nphase".into()); a.push(n.to_string()); }
if let Some(vs) = &p.vbit_shape { a.push("--vbit-shape".into()); a.push(vs.clone()); }
if let Some(mb) = p.mean_bits { a.push("--mean-bits".into()); a.push(mb.to_string()); }
if let Some(g) = p.shard_max_gb { a.push("--shard-max-gb".into()); a.push(g.to_string()); }
if p.skip_mtp { a.push("--skip-mtp".into()); }
let wd = conv.parent().map(|d| d.to_path_buf()).unwrap_or_else(|| std::path::PathBuf::from("."));
if let Some(lc) = &p.linear_core {
a.push("--linear-core".into());
a.push(lc.clone());
}
if let Some(n) = p.nphase {
a.push("--nphase".into());
a.push(n.to_string());
}
if let Some(vs) = &p.vbit_shape {
a.push("--vbit-shape".into());
a.push(vs.clone());
}
if let Some(mb) = p.mean_bits {
a.push("--mean-bits".into());
a.push(mb.to_string());
}
if let Some(g) = p.shard_max_gb {
a.push("--shard-max-gb".into());
a.push(g.to_string());
}
if p.skip_mtp {
a.push("--skip-mtp".into());
}
let wd = conv
.parent()
.map(|d| d.to_path_buf())
.unwrap_or_else(|| std::path::PathBuf::from("."));
(cfg.python_bin.clone(), a, wd)
} else if p.repo.to_ascii_lowercase().contains("gguf") {
// GGUF repo → native `cortiq import-gguf` (downloads + dequantizes any
@@ -307,21 +355,40 @@ pub fn start_import(store: Arc<JobStore>, cfg: &CmfCfg, p: ImportParams) -> Resu
let mut a = vec![
"import-gguf".into(),
p.repo.clone(),
"--quant".into(), map_quant(&p.quant).into(),
"--output".into(), output_abs.clone(),
"--quant".into(),
map_quant(&p.quant).into(),
"--output".into(),
output_abs.clone(),
];
if let Some(t) = &hf_token { a.push("--hf-token".into()); a.push(t.clone()); }
(cfg.cortiq_bin.clone(), a, std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from(".")))
if let Some(t) = &hf_token {
a.push("--hf-token".into());
a.push(t.clone());
}
(
cfg.cortiq_bin.clone(),
a,
std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from(".")),
)
} else {
// safetensors → native `cortiq convert` (dense / MoE / GatedDeltaNet).
let mut a = vec![
"convert".into(),
"--model".into(), p.repo.clone(),
"--quant".into(), map_quant(&p.quant).into(),
"--output".into(), output_abs.clone(),
"--model".into(),
p.repo.clone(),
"--quant".into(),
map_quant(&p.quant).into(),
"--output".into(),
output_abs.clone(),
];
if let Some(t) = &hf_token { a.push("--hf-token".into()); a.push(t.clone()); }
(cfg.cortiq_bin.clone(), a, std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from(".")))
if let Some(t) = &hf_token {
a.push("--hf-token".into());
a.push(t.clone());
}
(
cfg.cortiq_bin.clone(),
a,
std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from(".")),
)
};
let ret_id = id.clone();
+44 -12
View File
@@ -208,15 +208,25 @@ impl Pipeline {
};
let (decision, route_source, candidates) = if live.cfg.cmf.local_only {
(mk("local-only", "local"), "local".to_string(), live.local_candidates())
(
mk("local-only", "local"),
"local".to_string(),
live.local_candidates(),
)
} else {
match &req.routing {
RoutingDirective::Pinned { model_id } => {
(mk("pinned", "pinned"), "pinned".to_string(), vec![model_id.clone()])
}
RoutingDirective::Pinned { model_id } => (
mk("pinned", "pinned"),
"pinned".to_string(),
vec![model_id.clone()],
),
RoutingDirective::Auto { profile } => {
if !live.cfg.router.enabled {
(mk("router-disabled", "local"), "local".to_string(), live.local_candidates())
(
mk("router-disabled", "local"),
"local".to_string(),
live.local_candidates(),
)
} else {
let prof = profile.as_deref().unwrap_or(&live.cfg.route.profile);
match live.router.route(&text, prof).await {
@@ -229,7 +239,11 @@ impl Pipeline {
tracing::warn!(
"Router is unavailable. Gracefully degrading to the local/default model."
);
(mk("degraded", "fallback"), "fallback".to_string(), live.local_candidates())
(
mk("degraded", "fallback"),
"fallback".to_string(),
live.local_candidates(),
)
}
}
}
@@ -253,9 +267,21 @@ impl Pipeline {
}
}
// Local-first safety net: if routing produced no usable candidate (e.g.
// an unknown pinned model while the router is off), fall back to the
// managed local model so a configured local backend still answers.
// An explicit pin must never silently answer from a different model: if
// the pinned id is unknown or was filtered out, fail loudly so the caller
// knows their choice was not honoured (instead of getting a stray reply
// from the managed local model via the safety net below).
if final_candidates.is_empty() {
if let RoutingDirective::Pinned { model_id } = &req.routing {
return Err(GatewayError::UpstreamUnavailable(format!(
"pinned model '{model_id}' is not available — it is not in the pool (deleted?) or lacks a required capability such as tools"
)));
}
}
// Local-first safety net: for AUTO/degraded routes only, if routing
// produced no usable candidate, fall back to the managed local model so a
// configured local backend still answers.
if final_candidates.is_empty() && live.cfg.cmf.manage_server {
for id in live.local_candidates() {
if let Some(prov) = live.registry.get(&id) {
@@ -289,7 +315,10 @@ impl Pipeline {
&& !sh.local_model_id.is_empty()
&& matches!(req.routing, RoutingDirective::Auto { .. })
&& state.promotion.serves_local(&decision.task_label)
&& !matches!(decision.complexity_tier.as_str(), "high" | "hard" | "complex")
&& !matches!(
decision.complexity_tier.as_str(),
"high" | "hard" | "complex"
)
&& !final_candidates.iter().any(|m| m == &sh.local_model_id)
{
final_candidates.insert(0, sh.local_model_id.clone());
@@ -713,7 +742,7 @@ fn parse_judge(text: &str) -> (u8, bool) {
let score = text
.split("score")
.nth(1)
.and_then(|t| t.chars().skip_while(|c| !c.is_ascii_digit()).next())
.and_then(|t| t.chars().find(|c| c.is_ascii_digit()))
.and_then(|c| c.to_digit(10))
.map(|d| d.min(4) as u8);
let hard_fail = text
@@ -747,7 +776,10 @@ async fn shadow_capture(
};
// 1. Local answer (free, CPU).
let a_local = match local.chat(shadow_req(&cfg.local_model_id, clip(&prompt, 4000), 512)).await {
let a_local = match local
.chat(shadow_req(&cfg.local_model_id, clip(&prompt, 4000), 512))
.await
{
Ok(r) => r
.choices
.into_iter()
+96 -30
View File
@@ -18,7 +18,10 @@ use std::sync::Mutex;
use std::time::{SystemTime, UNIX_EPOCH};
fn now_secs() -> u64 {
SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs()
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
/// One judgment of a local answer (JSONL line, replayed on startup).
@@ -62,14 +65,14 @@ pub enum PromotionState {
#[derive(Clone, Debug)]
pub struct PromotionCfg {
pub enabled: bool,
pub window: usize, // rolling judgments per label (W)
pub n_min: usize, // min judgments before promotion is even considered
pub promote_lb: f64, // Wilson-95 LB of pass must clear this
pub demote_pass: f64, // point pass-rate below this → demote a served label
pub hardfail_ub_max: f64,// Wilson-95 UPPER bound of hard_fail must stay under this
pub soak: usize, // extra Canary judgments before LocalServed
pub z: f64, // 1.96 = 95%
pub file: Option<String>,// judge.jsonl path (append + replay)
pub window: usize, // rolling judgments per label (W)
pub n_min: usize, // min judgments before promotion is even considered
pub promote_lb: f64, // Wilson-95 LB of pass must clear this
pub demote_pass: f64, // point pass-rate below this → demote a served label
pub hardfail_ub_max: f64, // Wilson-95 UPPER bound of hard_fail must stay under this
pub soak: usize, // extra Canary judgments before LocalServed
pub z: f64, // 1.96 = 95%
pub file: Option<String>, // judge.jsonl path (append + replay)
}
impl Default for PromotionCfg {
@@ -92,7 +95,7 @@ struct LabelState {
passes: VecDeque<bool>, // rolling window of pass/fail
hard_fails: VecDeque<bool>, // rolling window of hard_fail flags
state: PromotionState,
soaked: usize, // judgments accumulated while in Canary
soaked: usize, // judgments accumulated while in Canary
}
impl LabelState {
@@ -124,20 +127,29 @@ pub fn wilson_bound(s: usize, n: usize, z: f64, upper: bool) -> f64 {
let denom = 1.0 + z2 / n;
let centre = p + z2 / (2.0 * n);
let margin = z * ((p * (1.0 - p) / n) + z2 / (4.0 * n * n)).sqrt();
let v = if upper { centre + margin } else { centre - margin };
let v = if upper {
centre + margin
} else {
centre - margin
};
(v / denom).clamp(0.0, 1.0)
}
impl Promotion {
pub fn new(cfg: PromotionCfg) -> std::sync::Arc<Self> {
let p = std::sync::Arc::new(Self { cfg, labels: Mutex::new(HashMap::new()) });
let p = std::sync::Arc::new(Self {
cfg,
labels: Mutex::new(HashMap::new()),
});
p.replay();
p
}
fn replay(&self) {
let Some(path) = &self.cfg.file else { return };
let Ok(content) = std::fs::read_to_string(path) else { return };
let Ok(content) = std::fs::read_to_string(path) else {
return;
};
for line in content.lines() {
if let Ok(rec) = serde_json::from_str::<JudgeRecord>(line) {
self.apply(&rec);
@@ -151,8 +163,10 @@ impl Promotion {
let st = self.apply(&rec);
if let Some(path) = &self.cfg.file {
if let Ok(line) = serde_json::to_string(&rec) {
if let Ok(mut f) =
std::fs::OpenOptions::new().create(true).append(true).open(path)
if let Ok(mut f) = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(path)
{
let _ = writeln!(f, "{line}");
}
@@ -163,7 +177,9 @@ impl Promotion {
fn apply(&self, rec: &JudgeRecord) -> PromotionState {
let mut map = self.labels.lock().unwrap();
let ls = map.entry(rec.task_label.clone()).or_insert_with(LabelState::new);
let ls = map
.entry(rec.task_label.clone())
.or_insert_with(LabelState::new);
push_cap(&mut ls.passes, rec.pass, self.cfg.window);
push_cap(&mut ls.hard_fails, rec.hard_fail, self.cfg.window);
self.advance(ls);
@@ -188,7 +204,12 @@ impl Promotion {
ls.state = match ls.state {
PromotionState::CloudOnly | PromotionState::ShadowJudged => {
if bar { ls.soaked = 0; PromotionState::Canary } else { PromotionState::ShadowJudged }
if bar {
ls.soaked = 0;
PromotionState::Canary
} else {
PromotionState::ShadowJudged
}
}
PromotionState::Canary => {
if regressed {
@@ -203,7 +224,11 @@ impl Promotion {
}
}
PromotionState::LocalServed => {
if regressed { PromotionState::Demoted } else { PromotionState::LocalServed }
if regressed {
PromotionState::Demoted
} else {
PromotionState::LocalServed
}
}
// Demoted goes back to SHADOW to re-earn promotion (skill stays in .cmf).
PromotionState::Demoted => PromotionState::ShadowJudged,
@@ -223,7 +248,10 @@ impl Promotion {
/// Should a request of this label be SERVED by the local model?
/// (Canary/LocalServed; caller still applies the per-request veto.)
pub fn serves_local(&self, label: &str) -> bool {
matches!(self.state(label), PromotionState::Canary | PromotionState::LocalServed)
matches!(
self.state(label),
PromotionState::Canary | PromotionState::LocalServed
)
}
/// Snapshot for /admin: (state, n, pass-rate, Wilson-LB) per label.
@@ -235,10 +263,16 @@ impl Promotion {
let n = ls.passes.len();
let s = ls.passes.iter().filter(|&&p| p).count();
let p = if n > 0 { s as f64 / n as f64 } else { 0.0 };
(k.clone(), ls.state, n, p, wilson_bound(s, n, self.cfg.z, false))
(
k.clone(),
ls.state,
n,
p,
wilson_bound(s, n, self.cfg.z, false),
)
})
.collect();
out.sort_by(|a, b| b.2.cmp(&a.2));
out.sort_by_key(|x| std::cmp::Reverse(x.2));
out
}
@@ -259,12 +293,24 @@ mod tests {
use super::*;
fn cfg() -> PromotionCfg {
PromotionCfg { enabled: true, n_min: 200, soak: 0, ..Default::default() }
PromotionCfg {
enabled: true,
n_min: 200,
soak: 0,
..Default::default()
}
}
fn rec(label: &str, pass: bool, hard_fail: bool) -> JudgeRecord {
JudgeRecord {
ts: 0, task_label: label.into(), pass, hard_fail, score: if pass { 4 } else { 0 },
source: "checker".into(), recon_e: None, born_conf: None, judge_tokens: 0,
ts: 0,
task_label: label.into(),
pass,
hard_fail,
score: if pass { 4 } else { 0 },
source: "checker".into(),
recon_e: None,
born_conf: None,
judge_tokens: 0,
}
}
@@ -316,7 +362,10 @@ mod tests {
match row {
Some((_, st, n, pr, lb)) => println!(
" {label:<12} {:<13} n={n:<4} pass={:.0}% LB={:.3} serves_local={}",
format!("{st:?}"), pr * 100.0, lb, p.serves_local(label)
format!("{st:?}"),
pr * 100.0,
lb,
p.serves_local(label)
),
None => println!(" {label:<12} (cloud-only, no judgments)"),
}
@@ -324,19 +373,29 @@ mod tests {
println!("\n── self-warming loop, live ──");
// A mature skill (e.g. ru-tech): local answers pass the judge ~99%.
println!(" 1) SHADOW→CANARY: судим локаль 'ru_tech' (сильный скилл ~99% pass), 210 судейств:");
println!(
" 1) SHADOW→CANARY: судим локаль 'ru_tech' (сильный скилл ~99% pass), 210 судейств:"
);
for i in 0..210 {
p.record(rec("ru_tech", i % 100 != 0, false));
}
show("ru_tech");
assert_eq!(p.state("ru_tech"), PromotionState::Canary, "n≥200 & LB≥0.95 → CANARY");
assert_eq!(
p.state("ru_tech"),
PromotionState::Canary,
"n≥200 & LB≥0.95 → CANARY"
);
println!(" 2) CANARY soak (ещё 60 судейств держат планку) → LOCAL_SERVED:");
for i in 0..60 {
p.record(rec("ru_tech", i % 100 != 0, false));
}
show("ru_tech");
assert_eq!(p.state("ru_tech"), PromotionState::LocalServed, "soak → LOCAL_SERVED");
assert_eq!(
p.state("ru_tech"),
PromotionState::LocalServed,
"soak → LOCAL_SERVED"
);
assert!(p.serves_local("ru_tech"), "теперь обслуживается локально");
// The code dataset: local fails the checker → never promotes.
@@ -345,7 +404,11 @@ mod tests {
p.record(rec("code", false, false));
}
show("code");
assert_eq!(p.state("code"), PromotionState::ShadowJudged, "остаётся на облаке — безопасно");
assert_eq!(
p.state("code"),
PromotionState::ShadowJudged,
"остаётся на облаке — безопасно"
);
assert!(!p.serves_local("code"));
// Drift on the promoted label → auto-demote back to cloud.
@@ -376,6 +439,9 @@ mod tests {
for _ in 0..300 {
p2.record(rec("t", false, false)); // quality collapses
}
assert!(matches!(p2.state("t"), PromotionState::Demoted | PromotionState::ShadowJudged));
assert!(matches!(
p2.state("t"),
PromotionState::Demoted | PromotionState::ShadowJudged
));
}
}
+5 -1
View File
@@ -90,7 +90,11 @@ fn promotion_cfg(s: &crate::config::ShadowCfg) -> crate::promotion::PromotionCfg
n_min: s.n_min,
promote_lb: s.promote_lb,
soak: s.soak,
file: if s.file.is_empty() { None } else { Some(s.file.clone()) },
file: if s.file.is_empty() {
None
} else {
Some(s.file.clone())
},
..Default::default()
}
}
+12
View File
@@ -177,6 +177,18 @@ textarea { resize: vertical; min-height: 64px; }
.check input { width: auto; }
.hint { color: var(--faint); font-size: 12px; margin-top: 4px; }
/* "?" help badge — hover/focus shows the native title tooltip explaining a setting */
.help {
display: inline-flex; align-items: center; justify-content: center;
width: 15px; height: 15px; border-radius: 50%;
border: 1px solid var(--border); background: var(--panel);
color: var(--muted); font-size: 10px; font-weight: 700; line-height: 1;
cursor: help; vertical-align: middle; user-select: none;
transition: 0.12s;
}
.help:hover, .help:focus { color: var(--accent); border-color: var(--accent); outline: none; }
label.field > span .help { text-transform: none; }
/* ---- buttons ------------------------------------------------------------- */
.btn {
display: inline-flex; align-items: center; justify-content: center; gap: 8px;
+5
View File
@@ -95,6 +95,11 @@ export const api = {
// shadow / self-warming loop
shadow: () => j("GET", "/shadow"),
// CMF runtime (the `cortiq` CLI that runs the .cmf format)
cmfStatus: () => j("GET", "/cmf"),
cmfInstall: () => j("POST", "/cmf/install"),
cmfPortCheck: (port) => j("GET", "/cmf/port?port=" + encodeURIComponent(port)),
// CMF model factory (HuggingFace → local .cmf)
hfSearch: (q, limit = 24) =>
j("GET", `/hf/search?q=${encodeURIComponent(q || "")}&limit=${limit}`),
+34
View File
@@ -129,6 +129,8 @@ export default {
"models.form.secretMissing": "not set",
"models.form.secretPlaceholder": "set or replace key (write-only)",
"models.deleteConfirm": "Delete model \"{id}\"?",
"models.managed": "managed",
"models.managedHint": "Provided by the managed CMF server (Settings → Local models). Configure it there.",
// routing
"routing.title": "Routing",
@@ -242,6 +244,38 @@ export default {
"settings.cmf.threads": "Worker threads",
"settings.cmf.gpu": "Use GPU (Metal / wgpu)",
"settings.cmf.speedNote": "Threads parallelise the decode matvecs (~3x faster; 0 = runtime default). GPU offload mainly helps prefill of long prompts — decode is memory-bandwidth-bound, so on Apple Silicon it is roughly neutral.",
"settings.cmf.checkPort": "Check port",
"settings.cmf.portFree": "port is free",
"settings.cmf.portBusy": "port is in use",
"settings.cmf.useSuggested": "use {port}",
"settings.cmf.serving": "serving {model} on :{port}",
"settings.cmf.intro": "Run models locally with the cortiq engine — works even offline or when the router is down.",
"settings.cmf.runtime": "cortiq runtime",
"settings.cmf.runtimeHelp": "The `cortiq` command-line tool that loads and runs .cmf models. Install it once before starting a local server.",
"settings.cmf.install": "Install cortiq",
"settings.cmf.reinstall": "Reinstall / update",
"settings.cmf.installing": "installing… (compiles, a few minutes)",
"settings.cmf.installedOk": "cortiq installed",
"settings.cmf.notInstalled": "not installed",
"settings.cmf.addModel": "Add a model",
"settings.cmf.addModelHelp": "Models are downloaded and converted in the Import tab (HuggingFace → .cmf). Then point the local server at the .cmf file below.",
"settings.cmf.importNote": "No model yet? Import one:",
"settings.cmf.importLink": "Open Import",
"settings.cmf.modelHelp": "Path to the .cmf file to serve (e.g. models/qwen.cmf), produced by the Import tab. Running several models for different tasks? Import them and switch this path.",
"settings.cmf.portHelp": "TCP port the local model server listens on (default 8081). Change only if it clashes with another service.",
"settings.cmf.threadsHelp": "How many CPU cores to use for generation. More = faster (~3× vs 1 core); 0 lets the runtime decide.",
"settings.cmf.gpuHelp": "Offload matrix math to the GPU (Metal/wgpu). On Apple Silicon this mostly speeds up long prompts — decode is memory-bandwidth-bound.",
"settings.cmf.localOnlyHelp": "Route every request to the local model, ignoring the router and any cloud providers.",
"settings.cmf.manageHelp": "The gateway starts `cortiq serve` for the .cmf below at launch and routes to it. Needs cortiq installed (step 1). Applies on restart.",
"settings.cmf.autoInstallHelp": "If cortiq is missing at startup, install it automatically from crates.io.",
"settings.cmf.autoUpdateHelp": "On startup, upgrade cortiq to the newest release when a newer one exists.",
"settings.router.enabledHelp": "Use the cortiq-router to classify each request (task type + complexity) and pick a model. Off = use the local routing table only.",
"settings.router.verifyTlsHelp": "Verify the router's TLS certificate. Turn off only for a self-signed / IP-bound certificate.",
"settings.telemetry.metricsHelp": "Expose Prometheus metrics at GET /metrics.",
"settings.cortiq.echoHelp": "Add a `cortiq` field (task, model, cost) to each response body. Response headers are always sent.",
"settings.stats.enabledHelp": "Record recent requests to power the dashboard and charts.",
"settings.cache.enabledHelp": "Semantic cache: return a saved answer when a new prompt is very similar to a past one, skipping the model call. Needs an embedding model in the pool.",
"settings.log.bodiesHelp": "Log full request/response message bodies. Off by default for privacy.",
"settings.router.taxonomy": "Taxonomy id",
"settings.router.getKey": "Get or top up a router key at",
"settings.router.secret": "Router key (value)",
+34
View File
@@ -128,6 +128,8 @@ export default {
"models.form.secretMissing": "не задан",
"models.form.secretPlaceholder": "задать или заменить ключ (только запись)",
"models.deleteConfirm": "Удалить модель \"{id}\"?",
"models.managed": "управляемая",
"models.managedHint": "Предоставляется CMF-сервером (Настройки → Локальные модели). Настраивается там.",
// routing
"routing.title": "Роутинг",
@@ -241,6 +243,38 @@ export default {
"settings.cmf.threads": "Рабочие потоки",
"settings.cmf.gpu": "Использовать GPU (Metal / wgpu)",
"settings.cmf.speedNote": "Потоки распараллеливают декод-матвеки (~×3; 0 = по умолчанию рантайма). GPU-оффлоад помогает в основном на prefill длинных промптов — декод упирается в пропускную способность памяти, поэтому на Apple Silicon почти нейтрален.",
"settings.cmf.checkPort": "Проверить порт",
"settings.cmf.portFree": "порт свободен",
"settings.cmf.portBusy": "порт занят",
"settings.cmf.useSuggested": "занять {port}",
"settings.cmf.serving": "работает: {model} на :{port}",
"settings.cmf.intro": "Запускайте модели локально движком cortiq — работает даже офлайн или когда роутер недоступен.",
"settings.cmf.runtime": "Среда cortiq",
"settings.cmf.runtimeHelp": "Инструмент командной строки `cortiq`, который загружает и запускает .cmf-модели. Установите один раз перед запуском локального сервера.",
"settings.cmf.install": "Установить cortiq",
"settings.cmf.reinstall": "Переустановить / обновить",
"settings.cmf.installing": "устанавливается… (компилируется, пара минут)",
"settings.cmf.installedOk": "cortiq установлен",
"settings.cmf.notInstalled": "не установлен",
"settings.cmf.addModel": "Добавить модель",
"settings.cmf.addModelHelp": "Модели скачиваются и конвертируются во вкладке Импорт (HuggingFace → .cmf). Затем укажите путь к .cmf ниже.",
"settings.cmf.importNote": "Ещё нет модели? Импортируйте:",
"settings.cmf.importLink": "Открыть Импорт",
"settings.cmf.modelHelp": "Путь к .cmf-файлу для запуска (напр. models/qwen.cmf), создаётся во вкладке Импорт. Несколько моделей под разные задачи? Импортируйте их и меняйте этот путь.",
"settings.cmf.portHelp": "TCP-порт локального сервера моделей (по умолчанию 8081). Меняйте, только если конфликтует с другим сервисом.",
"settings.cmf.threadsHelp": "Сколько ядер CPU использовать для генерации. Больше = быстрее (~×3 против 1 ядра); 0 — на усмотрение рантайма.",
"settings.cmf.gpuHelp": "Выгрузить матричные операции на GPU (Metal/wgpu). На Apple Silicon ускоряет в основном длинные промпты — декод упирается в пропускную способность памяти.",
"settings.cmf.localOnlyHelp": "Направлять все запросы в локальную модель, игнорируя роутер и облачные провайдеры.",
"settings.cmf.manageHelp": "Шлюз запускает `cortiq serve` для .cmf ниже при старте и маршрутизирует в него. Нужен установленный cortiq (шаг 1). Применяется после перезапуска.",
"settings.cmf.autoInstallHelp": "Если cortiq не найден при старте — установить автоматически с crates.io.",
"settings.cmf.autoUpdateHelp": "При старте обновлять cortiq до новейшей версии, когда есть новее.",
"settings.router.enabledHelp": "Использовать cortiq-router для классификации запроса (тип задачи + сложность) и выбора модели. Выкл = только локальная таблица маршрутизации.",
"settings.router.verifyTlsHelp": "Проверять TLS-сертификат роутера. Выключайте только для самоподписанного / привязанного к IP сертификата.",
"settings.telemetry.metricsHelp": "Отдавать метрики Prometheus по GET /metrics.",
"settings.cortiq.echoHelp": "Добавлять поле `cortiq` (задача, модель, стоимость) в тело каждого ответа. Заголовки ответа шлются всегда.",
"settings.stats.enabledHelp": "Записывать недавние запросы для дашборда и графиков.",
"settings.cache.enabledHelp": "Семантический кэш: возвращать сохранённый ответ, когда новый промпт очень похож на прошлый, пропуская вызов модели. Нужна embedding-модель в пуле.",
"settings.log.bodiesHelp": "Логировать полные тела запросов/ответов. По умолчанию выкл ради приватности.",
"settings.router.taxonomy": "ID таксономии",
"settings.router.getKey": "Получить или пополнить ключ роутера:",
"settings.router.secret": "Ключ роутера (значение)",
+3 -2
View File
@@ -168,7 +168,8 @@ function row(m, meta, reload) {
return h(
"tr",
{},
h("td", { class: "mono" }, m.id),
h("td", { class: "mono" }, m.id,
m.managed ? h("span", { class: "badge", style: "margin-left:6px", title: t("models.managedHint") }, t("models.managed")) : null),
h("td", {}, m.provider),
h("td", { class: "mono" }, m.model),
h("td", {}, h("span", { class: "badge " + (m.cost_tier || "") }, m.cost_tier || "—")),
@@ -182,7 +183,7 @@ function row(m, meta, reload) {
"div",
{ class: "flex", style: "justify-content:flex-end" },
probeBtn,
h("button", { class: "btn sm ghost", onClick: () => openModal(meta, m, reload) }, t("common.edit")),
m.managed ? null : h("button", { class: "btn sm ghost", onClick: () => openModal(meta, m, reload) }, t("common.edit")),
h(
"button",
{
+165 -50
View File
@@ -4,12 +4,22 @@ import { t } from "../i18n.js";
import { api } from "../api.js";
import { appState, SITE_URL } from "../app.js";
function field(label, control, hint) {
return h("label", { class: "field" }, h("span", {}, label), control, hint ? h("div", { class: "hint" }, hint) : null);
// A small "?" badge with a hover/focus tooltip explaining a setting.
function help(text) {
return text ? h("span", { class: "help", title: text, tabindex: "0", role: "img", "aria-label": text }, "?") : null;
}
function check(label, checked) {
function field(label, control, hint, helpText) {
return h(
"label",
{ class: "field" },
h("span", {}, label, helpText ? " " : null, help(helpText)),
control,
hint ? h("div", { class: "hint" }, hint) : null
);
}
function check(label, checked, helpText) {
const cb = h("input", { type: "checkbox", checked: checked ? true : null });
return { node: h("label", { class: "check" }, cb, label), cb };
return { node: h("label", { class: "check" }, cb, label, helpText ? " " : null, help(helpText)), cb };
}
// Flatten router usage (nested account/usage objects) into readable, wrapping
@@ -59,8 +69,8 @@ export async function renderSettings() {
const rKeyEnv = h("input", { value: s.router?.api_key_env || "", placeholder: "CORTIQ_ROUTER_KEY" });
const rTimeout = h("input", { type: "number", value: s.router?.timeout_ms ?? 800 });
const rTax = h("input", { value: s.router?.taxonomy_id || "" });
const rVerify = check(t("settings.router.verifyTls"), s.router?.verify_tls);
const rEnabled = check(t("settings.router.enabled"), s.router?.enabled !== false);
const rVerify = check(t("settings.router.verifyTls"), s.router?.verify_tls, t("settings.router.verifyTlsHelp"));
const rEnabled = check(t("settings.router.enabled"), s.router?.enabled !== false, t("settings.router.enabledHelp"));
// router key value (write-only; stored in the secret store, like model keys)
const rKeySource =
(secretsData.secrets || []).find((x) => x.name === (s.router?.api_key_env || "CORTIQ_ROUTER_KEY"))?.source ||
@@ -108,37 +118,16 @@ export async function renderSettings() {
rKeyBadge.className = "badge store";
return true;
}
const saveKeyBtn = h("button", { class: "btn sm" }, "✓ " + t("settings.router.saveKey"));
saveKeyBtn.addEventListener("click", async () => {
saveKeyBtn.disabled = true;
try {
if (await saveKey()) toast(t("settings.router.keySaved"), "good");
else toast(t("settings.router.keyEmpty"), "warn");
} catch (e) {
toast(String(e.message || e), "bad");
}
saveKeyBtn.disabled = false;
});
// "Test router" — real /v1/route call: tells key problems apart from a down router
const probeBtn = h("button", { class: "btn" }, t("settings.router.test"));
// Shared connection test (real /v1/route call) — tells key problems apart from
// a down router. Rendered into probeOut; also used right after saving the key.
const probeOut = h("div", { class: "hint", style: "margin-top:8px" });
probeBtn.addEventListener("click", async () => {
probeBtn.disabled = true;
async function runProbe() {
mount(probeOut, h("span", { class: "spinner" }));
try {
// If a key was just pasted but not saved, persist it first so the test uses it.
if (rSecret.value.trim()) {
try {
await saveKey();
} catch (_) {
/* fall through — probe will report no_key/auth */
}
}
const r = await api.probeRouter();
const parts = [t(probeMsgKey(r.status), { ms: r.latency_ms ?? "—" })];
if (r.message) parts.push(" — " + r.message);
const line = h("span", {}, ...parts);
const out = [line];
const out = [h("span", {}, ...parts)];
if (PROBE_SITE_STATUSES.includes(r.status)) {
out.push(" · ", h("a", { href: SITE_URL, target: "_blank", rel: "noopener" }, t("dash.health.getKey") + " ↗"));
}
@@ -150,9 +139,37 @@ export async function renderSettings() {
}
mount(probeOut, h("div", {}, h("span", { class: "badge " + (r.ok ? "ok" : "bad") }, r.ok ? "OK" : "×"), " ", ...out));
toast(t(probeMsgKey(r.status), { ms: r.latency_ms ?? "—" }), r.ok ? "good" : "bad");
return r.ok;
} catch (e) {
mount(probeOut, String(e.message || e));
return false;
}
}
const saveKeyBtn = h("button", { class: "btn sm" }, "✓ " + t("settings.router.saveKey"));
saveKeyBtn.addEventListener("click", async () => {
saveKeyBtn.disabled = true;
try {
if (await saveKey()) {
toast(t("settings.router.keySaved"), "good");
// Immediately test the connection so the user sees reachable/error at once,
// instead of a stale "unavailable" until something re-checks.
if (rEnabled.cb.checked) await runProbe();
} else {
toast(t("settings.router.keyEmpty"), "warn");
}
} catch (e) {
toast(String(e.message || e), "bad");
}
saveKeyBtn.disabled = false;
});
const probeBtn = h("button", { class: "btn" }, t("settings.router.test"));
probeBtn.addEventListener("click", async () => {
probeBtn.disabled = true;
// If a key was pasted but not saved, persist it first so the test uses it.
if (rSecret.value.trim()) {
try { await saveKey(); } catch (_) { /* probe will report no_key/auth */ }
}
await runProbe();
probeBtn.disabled = false;
});
// route
@@ -165,18 +182,18 @@ export async function renderSettings() {
const brCool = h("input", { value: s.breaker?.cooldown || "30s" });
// log
const logSel = h("select", {}, ...["error", "warn", "info", "debug", "trace"].map((x) => h("option", { value: x, selected: x === s.log?.level }, x)));
const logBodies = check(t("settings.log.bodies"), s.log?.bodies);
const logBodies = check(t("settings.log.bodies"), s.log?.bodies, t("settings.log.bodiesHelp"));
// telemetry / cortiq
const metricsCb = check(t("settings.telemetry.metrics"), s.telemetry?.metrics);
const echoCb = check(t("settings.cortiq.echo"), s.cortiq?.echo);
const metricsCb = check(t("settings.telemetry.metrics"), s.telemetry?.metrics, t("settings.telemetry.metricsHelp"));
const echoCb = check(t("settings.cortiq.echo"), s.cortiq?.echo, t("settings.cortiq.echoHelp"));
// stats
const statsEnabled = check(t("settings.stats.enabled"), s.stats?.enabled);
const statsEnabled = check(t("settings.stats.enabled"), s.stats?.enabled, t("settings.stats.enabledHelp"));
const statsFile = h("input", { value: s.stats?.file || "" });
const statsRet = h("input", { value: s.stats?.retention || "7d" });
// cache
const modelsData = await api.listModels();
const embedModels = (modelsData.models || []).filter((m) => m.kind === "embedding").map((m) => m.id);
const cacheEnabled = check(t("settings.cache.enabled"), s.cache?.enabled);
const cacheEnabled = check(t("settings.cache.enabled"), s.cache?.enabled, t("settings.cache.enabledHelp"));
const cacheThresh = h("input", { type: "number", step: "0.01", value: s.cache?.threshold ?? 0.92 });
const cacheTtlIn = h("input", { value: s.cache?.ttl || "1h" });
const cacheEmbedSel = h(
@@ -186,15 +203,93 @@ export async function renderSettings() {
...embedModels.map((id) => h("option", { value: id, selected: id === s.cache?.embed_model }, id))
);
// local models (CMF) — run a local .cmf server and/or force local-only routing
const cmfLocalOnly = check(t("settings.cmf.localOnly"), s.cmf?.local_only);
const cmfManage = check(t("settings.cmf.manage"), s.cmf?.manage_server);
const cmfAutoInstall = check(t("settings.cmf.autoInstall"), s.cmf?.auto_install);
const cmfAutoUpdate = check(t("settings.cmf.autoUpdate"), s.cmf?.auto_update);
const cmfModel = h("input", { value: s.cmf?.local_model || "", placeholder: "models/model.cmf" });
const cmfLocalOnly = check(t("settings.cmf.localOnly"), s.cmf?.local_only, t("settings.cmf.localOnlyHelp"));
const cmfManage = check(t("settings.cmf.manage"), s.cmf?.manage_server, t("settings.cmf.manageHelp"));
const cmfAutoInstall = check(t("settings.cmf.autoInstall"), s.cmf?.auto_install, t("settings.cmf.autoInstallHelp"));
const cmfAutoUpdate = check(t("settings.cmf.autoUpdate"), s.cmf?.auto_update, t("settings.cmf.autoUpdateHelp"));
const cmfModel = h("input", { value: s.cmf?.local_model || "", placeholder: "models/my-model.cmf" });
const cmfPort = h("input", { type: "number", value: s.cmf?.local_port ?? 8081 });
const cmfModelId = h("input", { value: s.cmf?.model_id || "cmf-local" });
const cmfThreads = h("input", { type: "number", min: 0, value: s.cmf?.threads ?? 8 });
const cmfGpu = check(t("settings.cmf.gpu"), s.cmf?.gpu);
const cmfGpu = check(t("settings.cmf.gpu"), s.cmf?.gpu, t("settings.cmf.gpuHelp"));
const cmfModelIdVal = s.cmf?.model_id || "cmf-local"; // auto — no manual "model id" field
// "Install cortiq" — the CLI that RUNS the .cmf format. Must be present before
// a local model server can start; this is step 1 for a new user.
const cmfStatusEl = h("span", { class: "muted" }, "…");
const cmfInstallBtn = h("button", { class: "btn" }, t("settings.cmf.install"));
// live "serving <model> on :<port>" indicator for the managed local server
const cmfServingEl = h("div", { class: "hint" });
async function refreshCmfStatus() {
try {
const st = await api.cmfStatus();
cmfInstallBtn.textContent = st.installed ? t("settings.cmf.reinstall") : t("settings.cmf.install");
mount(
cmfStatusEl,
st.installed
? h("span", {}, h("span", { class: "badge ok" }, "✓"), " cortiq " + (st.version || ""))
: h("span", {}, h("span", { class: "badge bad" }, "×"), " " + t("settings.cmf.notInstalled"))
);
// reflect the managed server state under the "run locally" toggle
if (st.manage_server && st.running) {
mount(cmfServingEl, h("span", { style: "color:var(--good)" },
"▶ " + t("settings.cmf.serving", { model: st.model_id || "", port: st.local_port }) + (st.healthy ? "" : " …")));
} else if (st.last_error) {
mount(cmfServingEl, h("span", { style: "color:var(--bad)" }, "✗ " + st.last_error));
} else {
mount(cmfServingEl, "");
}
} catch (_) {
mount(cmfStatusEl, "—");
}
}
cmfInstallBtn.addEventListener("click", async () => {
cmfInstallBtn.disabled = true;
mount(cmfStatusEl, h("span", {}, h("span", { class: "spinner" }), " " + t("settings.cmf.installing")));
try {
await api.cmfInstall();
// install compiles from crates.io — poll status for a few minutes
for (let i = 0; i < 120; i++) {
await new Promise((r) => setTimeout(r, 3000));
const st = await api.cmfStatus();
if (st.installed) { toast(t("settings.cmf.installedOk"), "good"); break; }
if (st.last_error) { toast(st.last_error, "bad"); break; }
}
} catch (e) {
toast(String(e.message || e), "bad");
}
await refreshCmfStatus();
cmfInstallBtn.disabled = false;
});
refreshCmfStatus();
// Port availability check — let the user confirm the chosen port is free
// before enabling the managed server (the default 8081 often clashes).
const cmfPortStatus = h("span", { class: "small", style: "margin-left:8px" });
const cmfPortCheckBtn = h("button", { class: "btn small", type: "button" }, t("settings.cmf.checkPort"));
async function checkPort() {
const p = parseInt(cmfPort.value, 10);
if (!p) { mount(cmfPortStatus, ""); return; }
cmfPortCheckBtn.disabled = true;
mount(cmfPortStatus, t("status.checking"));
try {
const r = await api.cmfPortCheck(p);
if (r.available) {
mount(cmfPortStatus, h("span", { style: "color:var(--good)" }, "✓ " + t("settings.cmf.portFree")));
} else {
const kids = [h("span", { style: "color:var(--bad)" }, "✗ " + t("settings.cmf.portBusy"))];
if (r.suggested) {
const useBtn = h("button", { class: "btn small", type: "button", style: "margin-left:8px" },
t("settings.cmf.useSuggested", { port: r.suggested }));
useBtn.addEventListener("click", () => { cmfPort.value = r.suggested; checkPort(); });
kids.push(useBtn);
}
mount(cmfPortStatus, h("span", {}, ...kids));
}
} catch (e) {
mount(cmfPortStatus, String(e.message || e));
}
cmfPortCheckBtn.disabled = false;
}
cmfPortCheckBtn.addEventListener("click", checkPort);
const saveBtn = h("button", { class: "btn primary" }, t("common.save"));
saveBtn.addEventListener("click", async () => {
@@ -242,7 +337,7 @@ export async function renderSettings() {
auto_update: cmfAutoUpdate.cb.checked,
local_model: cmfModel.value.trim(),
local_port: parseInt(cmfPort.value) || 8081,
model_id: cmfModelId.value.trim() || "cmf-local",
model_id: cmfModelIdVal,
threads: parseInt(cmfThreads.value) || 0,
gpu: cmfGpu.cb.checked,
},
@@ -372,13 +467,33 @@ export async function renderSettings() {
"div",
{ class: "card" },
h("div", { class: "card-head" }, h("h3", {}, t("settings.cmf"))),
cmfLocalOnly.node,
cmfManage.node,
h("div", { class: "hint" }, t("settings.cmf.intro")),
// Step 1 — install the cortiq runtime (the CLI that runs .cmf models).
h(
"label",
{ class: "field" },
h("span", {}, "1. " + t("settings.cmf.runtime"), " ", help(t("settings.cmf.runtimeHelp"))),
h("div", { class: "flex", style: "gap:8px;align-items:center;flex-wrap:wrap" }, cmfInstallBtn, cmfStatusEl)
),
h("div", { class: "row" }, cmfAutoInstall.node, cmfAutoUpdate.node),
field(t("settings.cmf.model"), cmfModel),
h("div", { class: "row" }, field(t("settings.cmf.port"), cmfPort), field(t("settings.cmf.modelId"), cmfModelId)),
h("div", { class: "row" }, field(t("settings.cmf.threads"), cmfThreads), cmfGpu.node),
h("div", { class: "hint" }, t("settings.cmf.note")),
h("div", { class: "divider" }),
// Step 2 — add a model. Models are imported in the Models/Import tab; here
// you just point the local server at the .cmf file it produced.
h("div", { class: "field" }, h("span", {}, "2. " + t("settings.cmf.addModel"), " ", help(t("settings.cmf.addModelHelp"))),
h("div", { class: "hint" }, t("settings.cmf.importNote"),
" ", h("a", { href: "#/import" }, t("settings.cmf.importLink") + " ↗"))),
field(t("settings.cmf.model"), cmfModel, null, t("settings.cmf.modelHelp")),
h("div", { class: "divider" }),
// Step 3 — run it locally + performance.
cmfManage.node,
cmfServingEl,
h("div", { class: "row" },
field(t("settings.cmf.port"),
h("div", { class: "flex wrap" }, cmfPort, cmfPortCheckBtn, cmfPortStatus),
null, t("settings.cmf.portHelp")),
field(t("settings.cmf.threads"), cmfThreads, null, t("settings.cmf.threadsHelp"))),
cmfGpu.node,
cmfLocalOnly.node,
h("div", { class: "hint" }, t("settings.cmf.speedNote"))
)
),