cmf: parallelise the managed local server (CMF_THREADS)
The gateway spawned `cortiq serve` with no thread hint, so the runtime ran the decode loop single-threaded (~0.25 tok/s on a 15B CPU model). Export CMF_THREADS from a new [cmf].threads config field (default 8) in spawn_serve. Measured: 0.25 -> 0.81 tok/s (~3.2x) on qwopus-vbit7 (15B). Scaling is sublinear (GatedDeltaNet state recurrence is serial per token, Amdahl-bound); a real jump needs the GPU path.
This commit is contained in:
+9
-4
@@ -155,14 +155,19 @@ async fn cargo_install(force: bool) -> Result<(), String> {
|
|||||||
/// Spawn `cortiq serve <model> --host <host> --port <port>`; the child is killed
|
/// Spawn `cortiq serve <model> --host <host> --port <port>`; the child is killed
|
||||||
/// when its handle is dropped (i.e. when the gateway shuts down).
|
/// when its handle is dropped (i.e. when the gateway shuts down).
|
||||||
fn spawn_serve(cfg: &CmfCfg) -> Result<Child, String> {
|
fn spawn_serve(cfg: &CmfCfg) -> Result<Child, String> {
|
||||||
Command::new(&cfg.cortiq_bin)
|
let mut cmd = Command::new(&cfg.cortiq_bin);
|
||||||
.arg("serve")
|
cmd.arg("serve")
|
||||||
.arg(&cfg.local_model)
|
.arg(&cfg.local_model)
|
||||||
.arg("--host")
|
.arg("--host")
|
||||||
.arg(&cfg.local_host)
|
.arg(&cfg.local_host)
|
||||||
.arg("--port")
|
.arg("--port")
|
||||||
.arg(cfg.local_port.to_string())
|
.arg(cfg.local_port.to_string());
|
||||||
.stdout(Stdio::null())
|
// Parallelise decode matvecs across cores (CMF_THREADS). 0 = leave the
|
||||||
|
// runtime's own default; N>1 gave ~2.8x on a 15B CPU model.
|
||||||
|
if cfg.threads > 1 {
|
||||||
|
cmd.env("CMF_THREADS", cfg.threads.to_string());
|
||||||
|
}
|
||||||
|
cmd.stdout(Stdio::null())
|
||||||
.stderr(Stdio::null())
|
.stderr(Stdio::null())
|
||||||
.kill_on_drop(true)
|
.kill_on_drop(true)
|
||||||
.spawn()
|
.spawn()
|
||||||
|
|||||||
@@ -111,6 +111,11 @@ pub struct CmfCfg {
|
|||||||
/// the router and any external providers ("use only local models").
|
/// the router and any external providers ("use only local models").
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub local_only: bool,
|
pub local_only: bool,
|
||||||
|
/// Worker threads for the managed `cortiq serve` (exported as `CMF_THREADS`).
|
||||||
|
/// 0 = let the runtime decide; N>1 parallelises decode matvecs across N cores
|
||||||
|
/// (measured ~2.8x faster than serial on a 15B model). Default 8.
|
||||||
|
#[serde(default = "default_cmf_threads")]
|
||||||
|
pub threads: usize,
|
||||||
}
|
}
|
||||||
impl Default for CmfCfg {
|
impl Default for CmfCfg {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
@@ -129,9 +134,13 @@ impl Default for CmfCfg {
|
|||||||
local_port: default_cmf_local_port(),
|
local_port: default_cmf_local_port(),
|
||||||
model_id: default_cmf_model_id(),
|
model_id: default_cmf_model_id(),
|
||||||
local_only: false,
|
local_only: false,
|
||||||
|
threads: default_cmf_threads(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
fn default_cmf_threads() -> usize {
|
||||||
|
8
|
||||||
|
}
|
||||||
fn default_cortiq_bin() -> String {
|
fn default_cortiq_bin() -> String {
|
||||||
"cortiq".into()
|
"cortiq".into()
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user