From 851cc7f1cb25e0480f8bb8d9df73e006e6bd822c Mon Sep 17 00:00:00 2001 From: infosave2007 Date: Tue, 14 Jul 2026 12:32:51 +0300 Subject: [PATCH] feat(cmf): per-server O(1) attention override at serve time Each managed local model row gains an O(1) selector (auto / off / all / deepN), exported to the child `cortiq serve` as CMF_O1. "off" forces exact attention, curing incoherent output from a too-aggressive `--o1` conversion hint without re-converting the file; "all"/"deepN" force-enable it. Auto (default) follows the hint recorded in the .cmf. Surfaced in /admin/api/cmf per-server status. --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/admin/mod.rs | 1 + src/cmf_runtime.rs | 5 +++++ src/config.rs | 7 +++++++ web/js/i18n/en.js | 4 +++- web/js/i18n/ru.js | 4 +++- web/js/views/settings.js | 10 ++++++++-- 8 files changed, 29 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c326f0..d22144d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -285,7 +285,7 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "cortiq-gateway" -version = "0.2.18" +version = "0.2.19" dependencies = [ "anyhow", "arc-swap", diff --git a/Cargo.toml b/Cargo.toml index e72356d..ad9a65d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cortiq-gateway" -version = "0.2.18" +version = "0.2.19" edition = "2021" rust-version = "1.75" license = "Apache-2.0" diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 9652c10..8e9c4ae 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -1009,6 +1009,7 @@ async fn cmf_status(State(state): State) -> ApiResult { "port": s.port, "threads": s.threads, "gpu": s.gpu, + "o1": s.o1, "running": live.map(|l| l.running).unwrap_or(false), "healthy": live.map(|l| l.healthy).unwrap_or(false), "last_error": live.and_then(|l| l.last_error.clone()), diff --git a/src/cmf_runtime.rs b/src/cmf_runtime.rs index 1873a2e..9b43b5c 100644 --- a/src/cmf_runtime.rs +++ b/src/cmf_runtime.rs @@ -295,6 +295,11 @@ fn spawn_serve(server: &CmfServer, cfg: &CmfCfg) -> Result { if server.gpu { cmd.env("CMF_GPU", "1"); } + // Serve-time O(1) attention override: "off" forces exact attention (undoes a + // too-aggressive --o1 conversion hint without re-converting the file). + if !server.o1.trim().is_empty() { + cmd.env("CMF_O1", server.o1.trim()); + } cmd.stdout(Stdio::null()) .stderr(Stdio::null()) .kill_on_drop(true) diff --git a/src/config.rs b/src/config.rs index b280281..329f0ea 100644 --- a/src/config.rs +++ b/src/config.rs @@ -83,6 +83,12 @@ pub struct CmfServer { /// Offload matvecs to the GPU (CMF_GPU=1). #[serde(default)] pub gpu: bool, + /// Serve-time O(1) attention override (exported as CMF_O1). Empty = follow + /// the hint recorded in the .cmf; "off" = force exact attention (fixes a + /// too-aggressive conversion without re-converting); "all"/"deepN"/"i,j,k" + /// = force-enable on those layers. + #[serde(default)] + pub o1: String, } /// CMF-format model factory: import a HuggingFace model, convert it to a @@ -172,6 +178,7 @@ impl CmfCfg { port: self.local_port, threads: self.threads, gpu: self.gpu, + o1: String::new(), }]; } Vec::new() diff --git a/web/js/i18n/en.js b/web/js/i18n/en.js index 0948473..101ba55 100644 --- a/web/js/i18n/en.js +++ b/web/js/i18n/en.js @@ -264,7 +264,9 @@ export default { "settings.cmf.pick": "select a model", "settings.cmf.models": "Local models", "settings.cmf.modelsHelp": "Each row is one model the gateway runs (its own cortiq serve on its own port) and registers in the pool. Add as many as you have .cmf files and free ports.", - "settings.cmf.rowHint": "id · .cmf file · port · threads · GPU. Changes apply on restart; deleting a row stops that server.", + "settings.cmf.rowHint": "id · .cmf file · port · threads · GPU · O(1). Changes apply on restart; deleting a row stops that server.", + "settings.cmf.o1Auto": "O(1): auto", + "settings.cmf.o1Help": "O(1) attention at serve time: auto — follow the hint recorded in the .cmf; off — force exact attention (fixes incoherent output after a too-aggressive --o1 conversion, no re-convert needed); all/deepN — force-enable on those layers.", "settings.cmf.addServer": "+ Add local model", "settings.cmf.checkPort": "Check port", "settings.cmf.portFree": "port is free", diff --git a/web/js/i18n/ru.js b/web/js/i18n/ru.js index 7daf6e4..a0d29a1 100644 --- a/web/js/i18n/ru.js +++ b/web/js/i18n/ru.js @@ -263,7 +263,9 @@ export default { "settings.cmf.pick": "выберите модель", "settings.cmf.models": "Локальные модели", "settings.cmf.modelsHelp": "Каждая строка — отдельная модель: шлюз поднимает свой cortiq serve на своём порту и регистрирует её в пуле. Добавляйте столько, сколько есть .cmf-файлов и свободных портов.", - "settings.cmf.rowHint": "id · файл .cmf · порт · потоки · GPU. Изменения применяются после перезапуска; удаление строки останавливает сервер.", + "settings.cmf.rowHint": "id · файл .cmf · порт · потоки · GPU · O(1). Изменения применяются после перезапуска; удаление строки останавливает сервер.", + "settings.cmf.o1Auto": "O(1): авто", + "settings.cmf.o1Help": "O(1)-внимание при запуске: авто — использовать подсказку из .cmf; off — принудительно точное внимание (лечит бессвязный вывод после слишком агрессивной --o1 конвертации, без переконвертации); all/deepN — принудительно включить на этих слоях.", "settings.cmf.addServer": "+ Добавить модель", "settings.cmf.checkPort": "Проверить порт", "settings.cmf.portFree": "порт свободен", diff --git a/web/js/views/settings.js b/web/js/views/settings.js index dc678a1..c3b1f83 100644 --- a/web/js/views/settings.js +++ b/web/js/views/settings.js @@ -234,6 +234,12 @@ export async function renderSettings() { const portIn = h("input", { type: "number", value: sv.port ?? 8090, style: "width:82px" }); const threadsIn = h("input", { type: "number", min: 0, value: sv.threads ?? 8, style: "width:60px", title: t("settings.cmf.threadsHelp") }); const gpuIn = h("input", { type: "checkbox", checked: sv.gpu ? true : null, title: t("settings.cmf.gpuHelp") }); + // serve-time O(1) override: off cures a too-aggressive --o1 conversion + const o1Opts = ["", "off", "all", "deep4", "deep8", "deep12"]; + const o1In = h("select", { style: "width:110px", title: t("settings.cmf.o1Help") }, + ...o1Opts.map((v) => h("option", { value: v, selected: (sv.o1 || "") === v ? true : null }, + v === "" ? t("settings.cmf.o1Auto") : v))); + if (sv.o1 && !o1Opts.includes(sv.o1)) o1In.appendChild(h("option", { value: sv.o1, selected: true }, sv.o1)); const checkBtn = h("button", { class: "btn small", type: "button" }, t("settings.cmf.checkPort")); const removeBtn = h("button", { class: "btn small danger", type: "button", title: t("common.delete") }, "✕"); const st = h("span", { class: "small" }); @@ -258,8 +264,8 @@ export async function renderSettings() { checkBtn.disabled = false; }); const row = h("div", { class: "flex wrap", style: "gap:6px;align-items:center;margin:6px 0" }, - idIn, modelSel, portIn, checkBtn, threadsIn, h("label", { class: "flex", style: "gap:4px" }, gpuIn, "GPU"), removeBtn, st); - const entry = { node: row, read: () => ({ id: idIn.value.trim() || autoId(modelSel.value), model: modelSel.value.trim(), port: parseInt(portIn.value) || 8090, threads: parseInt(threadsIn.value) || 0, gpu: gpuIn.checked }) }; + idIn, modelSel, portIn, checkBtn, threadsIn, h("label", { class: "flex", style: "gap:4px" }, gpuIn, "GPU"), o1In, removeBtn, st); + const entry = { node: row, read: () => ({ id: idIn.value.trim() || autoId(modelSel.value), model: modelSel.value.trim(), port: parseInt(portIn.value) || 8090, threads: parseInt(threadsIn.value) || 0, gpu: gpuIn.checked, o1: o1In.value }) }; removeBtn.addEventListener("click", () => { const i = serverRows.indexOf(entry); if (i >= 0) serverRows.splice(i, 1); row.remove(); }); serverRows.push(entry); serversWrap.appendChild(row);