diff --git a/src/cmf_runtime.rs b/src/cmf_runtime.rs index d17604e..1e34a9e 100644 --- a/src/cmf_runtime.rs +++ b/src/cmf_runtime.rs @@ -155,14 +155,19 @@ async fn cargo_install(force: bool) -> Result<(), String> { /// Spawn `cortiq serve --host --port `; the child is killed /// when its handle is dropped (i.e. when the gateway shuts down). fn spawn_serve(cfg: &CmfCfg) -> Result { - Command::new(&cfg.cortiq_bin) - .arg("serve") + let mut cmd = Command::new(&cfg.cortiq_bin); + cmd.arg("serve") .arg(&cfg.local_model) .arg("--host") .arg(&cfg.local_host) .arg("--port") - .arg(cfg.local_port.to_string()) - .stdout(Stdio::null()) + .arg(cfg.local_port.to_string()); + // Parallelise decode matvecs across cores (CMF_THREADS). 0 = leave the + // runtime's own default; N>1 gave ~2.8x on a 15B CPU model. + if cfg.threads > 1 { + cmd.env("CMF_THREADS", cfg.threads.to_string()); + } + cmd.stdout(Stdio::null()) .stderr(Stdio::null()) .kill_on_drop(true) .spawn() diff --git a/src/config.rs b/src/config.rs index 3e48780..bf8ae55 100644 --- a/src/config.rs +++ b/src/config.rs @@ -111,6 +111,11 @@ pub struct CmfCfg { /// the router and any external providers ("use only local models"). #[serde(default)] pub local_only: bool, + /// Worker threads for the managed `cortiq serve` (exported as `CMF_THREADS`). + /// 0 = let the runtime decide; N>1 parallelises decode matvecs across N cores + /// (measured ~2.8x faster than serial on a 15B model). Default 8. + #[serde(default = "default_cmf_threads")] + pub threads: usize, } impl Default for CmfCfg { fn default() -> Self { @@ -129,9 +134,13 @@ impl Default for CmfCfg { local_port: default_cmf_local_port(), model_id: default_cmf_model_id(), local_only: false, + threads: default_cmf_threads(), } } } +fn default_cmf_threads() -> usize { + 8 +} fn default_cortiq_bin() -> String { "cortiq".into() }