From 6cf2608fc8f9bf08bbb72ac8dfd07af7ac86d273 Mon Sep 17 00:00:00 2001 From: infosave2007 Date: Wed, 8 Jul 2026 16:06:55 +0300 Subject: [PATCH] admin: surface CMF worker threads + GPU toggle in Settings Expose the two speed knobs in the /admin Settings panel instead of TOML-only: - [cmf].gpu (new) -> exports CMF_GPU=1 to the managed cortiq serve (Metal/wgpu). On Apple Silicon decode is memory-bandwidth bound so this is ~neutral for generation (helps prefill); off by default. - [cmf].threads (added earlier) now has a UI number input. - config.rs: CmfCfg.gpu bool (serde default false). - cmf_runtime.rs: spawn_serve sets CMF_GPU=1 when gpu is on. - admin/mod.rs: put_settings flags needs_restart on threads/gpu change (the server is spawned once at startup with those env vars). - settings.js: threads input + gpu checkbox + save payload + speed hint. - i18n en/ru: threads/gpu/speedNote keys (other locales fall back to en, as the whole cmf section already does there). --- src/admin/mod.rs | 11 ++++++++--- src/cmf_runtime.rs | 5 +++++ src/config.rs | 6 ++++++ web/js/i18n/en.js | 3 +++ web/js/i18n/ru.js | 3 +++ web/js/views/settings.js | 8 +++++++- 6 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 57a7036..07b66d3 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -655,9 +655,14 @@ async fn put_settings(State(state): State, Json(b): Json Result { if cfg.threads > 1 { cmd.env("CMF_THREADS", cfg.threads.to_string()); } + // Offload large matvecs to the GPU (Metal on macOS). Memory-bandwidth-bound + // on unified memory, so mostly a prefill win; opt-in. + if cfg.gpu { + cmd.env("CMF_GPU", "1"); + } cmd.stdout(Stdio::null()) .stderr(Stdio::null()) .kill_on_drop(true) diff --git a/src/config.rs b/src/config.rs index bf8ae55..94b0bbb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -116,6 +116,11 @@ pub struct CmfCfg { /// (measured ~2.8x faster than serial on a 15B model). Default 8. #[serde(default = "default_cmf_threads")] pub threads: usize, + /// Offload large matvecs to the GPU (exports CMF_GPU=1). On Apple Silicon the + /// decode loop is memory-bandwidth bound, so this is ~neutral for generation + /// (helps prefill of long prompts). Off by default. + #[serde(default)] + pub gpu: bool, } impl Default for CmfCfg { fn default() -> Self { @@ -135,6 +140,7 @@ impl Default for CmfCfg { model_id: default_cmf_model_id(), local_only: false, threads: default_cmf_threads(), + gpu: false, } } } diff --git a/web/js/i18n/en.js b/web/js/i18n/en.js index 4e43e4e..8e899d1 100644 --- a/web/js/i18n/en.js +++ b/web/js/i18n/en.js @@ -239,6 +239,9 @@ export default { "settings.cmf.port": "Local server port", "settings.cmf.modelId": "Model id in the pool", "settings.cmf.note": "Serves a local .cmf via cortiq (github.com/infosave2007/cmf). Works even when the router is unavailable.", + "settings.cmf.threads": "Worker threads", + "settings.cmf.gpu": "Use GPU (Metal / wgpu)", + "settings.cmf.speedNote": "Threads parallelise the decode matvecs (~3x faster; 0 = runtime default). GPU offload mainly helps prefill of long prompts — decode is memory-bandwidth-bound, so on Apple Silicon it is roughly neutral.", "settings.router.taxonomy": "Taxonomy id", "settings.router.getKey": "Get or top up a router key at", "settings.router.secret": "Router key (value)", diff --git a/web/js/i18n/ru.js b/web/js/i18n/ru.js index c686c05..2bc4a2a 100644 --- a/web/js/i18n/ru.js +++ b/web/js/i18n/ru.js @@ -238,6 +238,9 @@ export default { "settings.cmf.port": "Порт локального сервера", "settings.cmf.modelId": "ID модели в пуле", "settings.cmf.note": "Обслуживает локальный .cmf через cortiq (github.com/infosave2007/cmf). Работает даже когда роутер недоступен.", + "settings.cmf.threads": "Рабочие потоки", + "settings.cmf.gpu": "Использовать GPU (Metal / wgpu)", + "settings.cmf.speedNote": "Потоки распараллеливают декод-матвеки (~×3; 0 = по умолчанию рантайма). GPU-оффлоад помогает в основном на prefill длинных промптов — декод упирается в пропускную способность памяти, поэтому на Apple Silicon почти нейтрален.", "settings.router.taxonomy": "ID таксономии", "settings.router.getKey": "Получить или пополнить ключ роутера:", "settings.router.secret": "Ключ роутера (значение)", diff --git a/web/js/views/settings.js b/web/js/views/settings.js index 2b073d2..acf8a3e 100644 --- a/web/js/views/settings.js +++ b/web/js/views/settings.js @@ -193,6 +193,8 @@ export async function renderSettings() { const cmfModel = h("input", { value: s.cmf?.local_model || "", placeholder: "models/model.cmf" }); const cmfPort = h("input", { type: "number", value: s.cmf?.local_port ?? 8081 }); const cmfModelId = h("input", { value: s.cmf?.model_id || "cmf-local" }); + const cmfThreads = h("input", { type: "number", min: 0, value: s.cmf?.threads ?? 8 }); + const cmfGpu = check(t("settings.cmf.gpu"), s.cmf?.gpu); const saveBtn = h("button", { class: "btn primary" }, t("common.save")); saveBtn.addEventListener("click", async () => { @@ -241,6 +243,8 @@ export async function renderSettings() { local_model: cmfModel.value.trim(), local_port: parseInt(cmfPort.value) || 8081, model_id: cmfModelId.value.trim() || "cmf-local", + threads: parseInt(cmfThreads.value) || 0, + gpu: cmfGpu.cb.checked, }, }; // serde: drop null optionals @@ -373,7 +377,9 @@ export async function renderSettings() { h("div", { class: "row" }, cmfAutoInstall.node, cmfAutoUpdate.node), field(t("settings.cmf.model"), cmfModel), h("div", { class: "row" }, field(t("settings.cmf.port"), cmfPort), field(t("settings.cmf.modelId"), cmfModelId)), - h("div", { class: "hint" }, t("settings.cmf.note")) + h("div", { class: "row" }, field(t("settings.cmf.threads"), cmfThreads), cmfGpu.node), + h("div", { class: "hint" }, t("settings.cmf.note")), + h("div", { class: "hint" }, t("settings.cmf.speedNote")) ) ), h(