admin: surface CMF worker threads + GPU toggle in Settings
Expose the two speed knobs in the /admin Settings panel instead of TOML-only: - [cmf].gpu (new) -> exports CMF_GPU=1 to the managed cortiq serve (Metal/wgpu). On Apple Silicon decode is memory-bandwidth bound so this is ~neutral for generation (helps prefill); off by default. - [cmf].threads (added earlier) now has a UI number input. - config.rs: CmfCfg.gpu bool (serde default false). - cmf_runtime.rs: spawn_serve sets CMF_GPU=1 when gpu is on. - admin/mod.rs: put_settings flags needs_restart on threads/gpu change (the server is spawned once at startup with those env vars). - settings.js: threads input + gpu checkbox + save payload + speed hint. - i18n en/ru: threads/gpu/speedNote keys (other locales fall back to en, as the whole cmf section already does there).
This commit is contained in:
+8
-3
@@ -655,9 +655,14 @@ async fn put_settings(State(state): State<SharedState>, Json(b): Json<SettingsBo
|
|||||||
cfg.cache = v;
|
cfg.cache = v;
|
||||||
}
|
}
|
||||||
if let Some(v) = b.cmf {
|
if let Some(v) = b.cmf {
|
||||||
// manage_server / local_model changes apply on restart (the server is
|
// manage_server / local_model / threads / gpu apply on restart (the server
|
||||||
// spawned at startup); local_only / router routing apply immediately.
|
// is spawned once at startup with those env vars); local_only / router
|
||||||
if v.manage_server != cfg.cmf.manage_server || v.local_model != cfg.cmf.local_model {
|
// routing apply immediately.
|
||||||
|
if v.manage_server != cfg.cmf.manage_server
|
||||||
|
|| v.local_model != cfg.cmf.local_model
|
||||||
|
|| v.threads != cfg.cmf.threads
|
||||||
|
|| v.gpu != cfg.cmf.gpu
|
||||||
|
{
|
||||||
needs_restart = true;
|
needs_restart = true;
|
||||||
}
|
}
|
||||||
cfg.cmf = v;
|
cfg.cmf = v;
|
||||||
|
|||||||
@@ -167,6 +167,11 @@ fn spawn_serve(cfg: &CmfCfg) -> Result<Child, String> {
|
|||||||
if cfg.threads > 1 {
|
if cfg.threads > 1 {
|
||||||
cmd.env("CMF_THREADS", cfg.threads.to_string());
|
cmd.env("CMF_THREADS", cfg.threads.to_string());
|
||||||
}
|
}
|
||||||
|
// Offload large matvecs to the GPU (Metal on macOS). Memory-bandwidth-bound
|
||||||
|
// on unified memory, so mostly a prefill win; opt-in.
|
||||||
|
if cfg.gpu {
|
||||||
|
cmd.env("CMF_GPU", "1");
|
||||||
|
}
|
||||||
cmd.stdout(Stdio::null())
|
cmd.stdout(Stdio::null())
|
||||||
.stderr(Stdio::null())
|
.stderr(Stdio::null())
|
||||||
.kill_on_drop(true)
|
.kill_on_drop(true)
|
||||||
|
|||||||
@@ -116,6 +116,11 @@ pub struct CmfCfg {
|
|||||||
/// (measured ~2.8x faster than serial on a 15B model). Default 8.
|
/// (measured ~2.8x faster than serial on a 15B model). Default 8.
|
||||||
#[serde(default = "default_cmf_threads")]
|
#[serde(default = "default_cmf_threads")]
|
||||||
pub threads: usize,
|
pub threads: usize,
|
||||||
|
/// Offload large matvecs to the GPU (exports CMF_GPU=1). On Apple Silicon the
|
||||||
|
/// decode loop is memory-bandwidth bound, so this is ~neutral for generation
|
||||||
|
/// (helps prefill of long prompts). Off by default.
|
||||||
|
#[serde(default)]
|
||||||
|
pub gpu: bool,
|
||||||
}
|
}
|
||||||
impl Default for CmfCfg {
|
impl Default for CmfCfg {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
@@ -135,6 +140,7 @@ impl Default for CmfCfg {
|
|||||||
model_id: default_cmf_model_id(),
|
model_id: default_cmf_model_id(),
|
||||||
local_only: false,
|
local_only: false,
|
||||||
threads: default_cmf_threads(),
|
threads: default_cmf_threads(),
|
||||||
|
gpu: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Vendored
+3
@@ -239,6 +239,9 @@ export default {
|
|||||||
"settings.cmf.port": "Local server port",
|
"settings.cmf.port": "Local server port",
|
||||||
"settings.cmf.modelId": "Model id in the pool",
|
"settings.cmf.modelId": "Model id in the pool",
|
||||||
"settings.cmf.note": "Serves a local .cmf via cortiq (github.com/infosave2007/cmf). Works even when the router is unavailable.",
|
"settings.cmf.note": "Serves a local .cmf via cortiq (github.com/infosave2007/cmf). Works even when the router is unavailable.",
|
||||||
|
"settings.cmf.threads": "Worker threads",
|
||||||
|
"settings.cmf.gpu": "Use GPU (Metal / wgpu)",
|
||||||
|
"settings.cmf.speedNote": "Threads parallelise the decode matvecs (~3x faster; 0 = runtime default). GPU offload mainly helps prefill of long prompts — decode is memory-bandwidth-bound, so on Apple Silicon it is roughly neutral.",
|
||||||
"settings.router.taxonomy": "Taxonomy id",
|
"settings.router.taxonomy": "Taxonomy id",
|
||||||
"settings.router.getKey": "Get or top up a router key at",
|
"settings.router.getKey": "Get or top up a router key at",
|
||||||
"settings.router.secret": "Router key (value)",
|
"settings.router.secret": "Router key (value)",
|
||||||
|
|||||||
Vendored
+3
@@ -238,6 +238,9 @@ export default {
|
|||||||
"settings.cmf.port": "Порт локального сервера",
|
"settings.cmf.port": "Порт локального сервера",
|
||||||
"settings.cmf.modelId": "ID модели в пуле",
|
"settings.cmf.modelId": "ID модели в пуле",
|
||||||
"settings.cmf.note": "Обслуживает локальный .cmf через cortiq (github.com/infosave2007/cmf). Работает даже когда роутер недоступен.",
|
"settings.cmf.note": "Обслуживает локальный .cmf через cortiq (github.com/infosave2007/cmf). Работает даже когда роутер недоступен.",
|
||||||
|
"settings.cmf.threads": "Рабочие потоки",
|
||||||
|
"settings.cmf.gpu": "Использовать GPU (Metal / wgpu)",
|
||||||
|
"settings.cmf.speedNote": "Потоки распараллеливают декод-матвеки (~×3; 0 = по умолчанию рантайма). GPU-оффлоад помогает в основном на prefill длинных промптов — декод упирается в пропускную способность памяти, поэтому на Apple Silicon почти нейтрален.",
|
||||||
"settings.router.taxonomy": "ID таксономии",
|
"settings.router.taxonomy": "ID таксономии",
|
||||||
"settings.router.getKey": "Получить или пополнить ключ роутера:",
|
"settings.router.getKey": "Получить или пополнить ключ роутера:",
|
||||||
"settings.router.secret": "Ключ роутера (значение)",
|
"settings.router.secret": "Ключ роутера (значение)",
|
||||||
|
|||||||
Vendored
+7
-1
@@ -193,6 +193,8 @@ export async function renderSettings() {
|
|||||||
const cmfModel = h("input", { value: s.cmf?.local_model || "", placeholder: "models/model.cmf" });
|
const cmfModel = h("input", { value: s.cmf?.local_model || "", placeholder: "models/model.cmf" });
|
||||||
const cmfPort = h("input", { type: "number", value: s.cmf?.local_port ?? 8081 });
|
const cmfPort = h("input", { type: "number", value: s.cmf?.local_port ?? 8081 });
|
||||||
const cmfModelId = h("input", { value: s.cmf?.model_id || "cmf-local" });
|
const cmfModelId = h("input", { value: s.cmf?.model_id || "cmf-local" });
|
||||||
|
const cmfThreads = h("input", { type: "number", min: 0, value: s.cmf?.threads ?? 8 });
|
||||||
|
const cmfGpu = check(t("settings.cmf.gpu"), s.cmf?.gpu);
|
||||||
|
|
||||||
const saveBtn = h("button", { class: "btn primary" }, t("common.save"));
|
const saveBtn = h("button", { class: "btn primary" }, t("common.save"));
|
||||||
saveBtn.addEventListener("click", async () => {
|
saveBtn.addEventListener("click", async () => {
|
||||||
@@ -241,6 +243,8 @@ export async function renderSettings() {
|
|||||||
local_model: cmfModel.value.trim(),
|
local_model: cmfModel.value.trim(),
|
||||||
local_port: parseInt(cmfPort.value) || 8081,
|
local_port: parseInt(cmfPort.value) || 8081,
|
||||||
model_id: cmfModelId.value.trim() || "cmf-local",
|
model_id: cmfModelId.value.trim() || "cmf-local",
|
||||||
|
threads: parseInt(cmfThreads.value) || 0,
|
||||||
|
gpu: cmfGpu.cb.checked,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
// serde: drop null optionals
|
// serde: drop null optionals
|
||||||
@@ -373,7 +377,9 @@ export async function renderSettings() {
|
|||||||
h("div", { class: "row" }, cmfAutoInstall.node, cmfAutoUpdate.node),
|
h("div", { class: "row" }, cmfAutoInstall.node, cmfAutoUpdate.node),
|
||||||
field(t("settings.cmf.model"), cmfModel),
|
field(t("settings.cmf.model"), cmfModel),
|
||||||
h("div", { class: "row" }, field(t("settings.cmf.port"), cmfPort), field(t("settings.cmf.modelId"), cmfModelId)),
|
h("div", { class: "row" }, field(t("settings.cmf.port"), cmfPort), field(t("settings.cmf.modelId"), cmfModelId)),
|
||||||
h("div", { class: "hint" }, t("settings.cmf.note"))
|
h("div", { class: "row" }, field(t("settings.cmf.threads"), cmfThreads), cmfGpu.node),
|
||||||
|
h("div", { class: "hint" }, t("settings.cmf.note")),
|
||||||
|
h("div", { class: "hint" }, t("settings.cmf.speedNote"))
|
||||||
)
|
)
|
||||||
),
|
),
|
||||||
h(
|
h(
|
||||||
|
|||||||
Reference in New Issue
Block a user