feat(release): v0.2.30 — per-model generation, O(1) attention, MTP & reasoning parameters

src/admin/mod.rs

@@ -1330,6 +1330,13 @@ async fn register_import(State(state): State<SharedState>, Path(job): Path<Strin
             port,
             threads: cfg.cmf.threads,
             gpu: cfg.cmf.gpu,
+            temperature: None,
+            top_p: None,
+            max_tokens: None,
+            think_budget: None,
+            system_prompt: None,
+            o1: None,
+            skip_mtp: false,
         });
     } else {
         cfg.models.push(crate::config::ModelCfg {
@@ -1343,6 +1350,13 @@ async fn register_import(State(state): State<SharedState>, Path(job): Path<Strin
             kind: "chat".into(),
             api_key_env: None,
             caps: Vec::new(),
+            temperature: None,
+            top_p: None,
+            max_tokens: None,
+            think_budget: None,
+            system_prompt: None,
+            o1: None,
+            skip_mtp: false,
         });
     }
     state.reload(cfg)?;

src/cmf_runtime.rs

@@ -290,11 +290,17 @@ fn spawn_serve(server: &CmfServer, cfg: &CmfCfg) -> Result<Child, String> {
     if server.threads > 1 {
         cmd.env("CMF_THREADS", server.threads.to_string());
     }
-    // Offload large matvecs to the GPU (Metal on macOS). Memory-bandwidth-bound
-    // on unified memory, so mostly a prefill win; opt-in.
     if server.gpu {
         cmd.env("CMF_GPU", "1");
     }
+    if let Some(o1) = &server.o1 {
+        if !o1.trim().is_empty() && o1 != "off" {
+            cmd.arg("--o1").arg(o1);
+        }
+    }
+    if server.skip_mtp {
+        cmd.arg("--skip-mtp");
+    }
     cmd.stdout(Stdio::null())
         .stderr(Stdio::null())
         .kill_on_drop(true)

src/config.rs

@@ -83,6 +83,27 @@ pub struct CmfServer {
     /// Offload matvecs to the GPU (CMF_GPU=1).
     #[serde(default)]
     pub gpu: bool,
+    /// Default generation temperature (0.0 .. 2.0).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub temperature: Option<f32>,
+    /// Default nucleus sampling threshold (0.0 .. 1.0).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub top_p: Option<f32>,
+    /// Default maximum completion tokens.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub max_tokens: Option<u32>,
+    /// Reasoning / thinking token budget.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub think_budget: Option<u32>,
+    /// Default system prompt prepended if request lacks one.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub system_prompt: Option<String>,
+    /// O(1) constant-memory streaming attention hint (all, deep, custom, off).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub o1: Option<String>,
+    /// Skip Multi-Token Prediction (MTP) heads.
+    #[serde(default)]
+    pub skip_mtp: bool,
 }
 
 /// CMF-format model factory: import a HuggingFace model, convert it to a
@@ -172,6 +193,13 @@ impl CmfCfg {
                 port: self.local_port,
                 threads: self.threads,
                 gpu: self.gpu,
+                temperature: None,
+                top_p: None,
+                max_tokens: None,
+                think_budget: None,
+                system_prompt: None,
+                o1: None,
+                skip_mtp: false,
             }];
         }
         Vec::new()
@@ -306,6 +334,27 @@ pub struct ModelCfg {
     pub api_key_env: Option<String>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub caps: Vec<String>,
+    /// Default generation temperature (0.0 .. 2.0).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub temperature: Option<f32>,
+    /// Default nucleus sampling threshold (0.0 .. 1.0).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub top_p: Option<f32>,
+    /// Default maximum completion tokens.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub max_tokens: Option<u32>,
+    /// Reasoning / thinking token budget.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub think_budget: Option<u32>,
+    /// Default system prompt prepended if request lacks one.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub system_prompt: Option<String>,
+    /// O(1) constant-memory streaming attention hint (all, deep, custom, off).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub o1: Option<String>,
+    /// Skip Multi-Token Prediction (MTP) heads.
+    #[serde(default)]
+    pub skip_mtp: bool,
 }
 fn default_kind() -> String {
     "chat".into()

src/model.rs

@@ -29,6 +29,7 @@ pub struct GenParams {
     pub temperature: Option<f32>,
     pub max_tokens: Option<u32>,
     pub top_p: Option<f32>,
+    pub think_budget: Option<u32>,
     pub stop: Vec<String>,
     /// Fields we do not interpret but must proxy through to the provider.
     pub passthrough: serde_json::Map<String, serde_json::Value>,

src/pipeline.rs

@@ -341,7 +341,64 @@ impl Pipeline {
                 route_source
             );
 
-            match provider.chat(req.clone()).await {
+            let mut req_call = req.clone();
+            if let Some(m_cfg) = live.cfg.models.iter().find(|m| m.id == model_id) {
+                if req_call.params.temperature.is_none() {
+                    req_call.params.temperature = m_cfg.temperature;
+                }
+                if req_call.params.top_p.is_none() {
+                    req_call.params.top_p = m_cfg.top_p;
+                }
+                if req_call.params.max_tokens.is_none() {
+                    req_call.params.max_tokens = m_cfg.max_tokens;
+                }
+                if req_call.params.think_budget.is_none() {
+                    req_call.params.think_budget = m_cfg.think_budget;
+                }
+                if let Some(sys) = &m_cfg.system_prompt {
+                    if !sys.trim().is_empty()
+                        && !req_call.messages.iter().any(|m| m.role == "system")
+                    {
+                        req_call.messages.insert(
+                            0,
+                            Message {
+                                role: "system".into(),
+                                content: sys.clone(),
+                                tool_calls: Vec::new(),
+                            },
+                        );
+                    }
+                }
+            } else if let Some(s_cfg) = live.cfg.cmf.servers.iter().find(|s| s.id == model_id) {
+                if req_call.params.temperature.is_none() {
+                    req_call.params.temperature = s_cfg.temperature;
+                }
+                if req_call.params.top_p.is_none() {
+                    req_call.params.top_p = s_cfg.top_p;
+                }
+                if req_call.params.max_tokens.is_none() {
+                    req_call.params.max_tokens = s_cfg.max_tokens;
+                }
+                if req_call.params.think_budget.is_none() {
+                    req_call.params.think_budget = s_cfg.think_budget;
+                }
+                if let Some(sys) = &s_cfg.system_prompt {
+                    if !sys.trim().is_empty()
+                        && !req_call.messages.iter().any(|m| m.role == "system")
+                    {
+                        req_call.messages.insert(
+                            0,
+                            Message {
+                                role: "system".into(),
+                                content: sys.clone(),
+                                tool_calls: Vec::new(),
+                            },
+                        );
+                    }
+                }
+            }
+
+            match provider.chat(req_call).await {
                 Ok(mut resp) => {
                     let cost =
                         provider.price(resp.usage.prompt_tokens, resp.usage.completion_tokens);

src/protocols/anthropic_messages.rs

@@ -101,6 +101,7 @@ async fn handler(
             temperature: req.temperature,
             max_tokens: req.max_tokens,
             top_p: req.top_p,
+            think_budget: None,
             stop: Vec::new(),
             passthrough: Default::default(),
         },

src/protocols/openai_chat.rs

@@ -49,6 +49,10 @@ struct OpenAiChatRequest {
     #[serde(default)]
     top_p: Option<f32>,
     #[serde(default)]
+    think_budget: Option<u32>,
+    #[serde(default)]
+    reasoning_effort: Option<u32>,
+    #[serde(default)]
     stream: bool,
     #[serde(default)]
     tools: Vec<serde_json::Value>,
@@ -93,6 +97,7 @@ async fn handler(
             temperature: req.temperature,
             max_tokens: req.max_tokens,
             top_p: req.top_p,
+            think_budget: req.think_budget.or(req.reasoning_effort),
             stop: Vec::new(),
             passthrough: req.rest,
         },

src/protocols/openai_completions.rs

@@ -82,6 +82,7 @@ async fn handler(
             temperature: req.temperature,
             max_tokens: req.max_tokens,
             top_p: req.top_p,
+            think_budget: None,
             stop: Vec::new(),
             passthrough: Default::default(),
         },

src/providers/openai.rs

@@ -42,9 +42,16 @@ impl OpenAiProvider {
         if let Some(t) = req.params.temperature {
             body["temperature"] = t.into();
         }
+        if let Some(p) = req.params.top_p {
+            body["top_p"] = p.into();
+        }
         if let Some(m) = req.params.max_tokens {
             body["max_tokens"] = m.into();
         }
+        if let Some(tb) = req.params.think_budget {
+            body["think_budget"] = tb.into();
+            body["reasoning_effort"] = tb.into();
+        }
         if !req.tools.is_empty() {
             body["tools"] = serde_json::Value::Array(req.tools.clone());
         }

src/registry.rs

@@ -49,6 +49,13 @@ impl Registry {
                 kind: "chat".into(),
                 api_key_env: None,
                 caps: Vec::new(),
+                temperature: s.temperature,
+                top_p: s.top_p,
+                max_tokens: s.max_tokens,
+                think_budget: s.think_budget,
+                system_prompt: s.system_prompt.clone(),
+                o1: s.o1.clone(),
+                skip_mtp: s.skip_mtp,
             };
             by_id.insert(m.id.clone(), Arc::new(OpenAiProvider::new(&m, None)?));
         }