From 94c8551206e297baa980138a371d1d2ae55460bb Mon Sep 17 00:00:00 2001 From: infosave2007 Date: Wed, 22 Jul 2026 10:40:44 +0300 Subject: [PATCH] =?UTF-8?q?feat(release):=20v0.2.30=20=E2=80=94=20per-mode?= =?UTF-8?q?l=20generation,=20O(1)=20attention,=20MTP=20&=20reasoning=20par?= =?UTF-8?q?ameters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 6 ++- Cargo.lock | 2 +- Cargo.toml | 2 +- src/admin/mod.rs | 14 +++++++ src/cmf_runtime.rs | 10 ++++- src/config.rs | 49 ++++++++++++++++++++++++ src/model.rs | 1 + src/pipeline.rs | 59 ++++++++++++++++++++++++++++- src/protocols/anthropic_messages.rs | 1 + src/protocols/openai_chat.rs | 5 +++ src/protocols/openai_completions.rs | 1 + src/providers/openai.rs | 7 ++++ src/registry.rs | 7 ++++ web/js/i18n/en.js | 12 ++++++ web/js/i18n/ru.js | 12 ++++++ web/js/views/models.js | 40 ++++++++++++++++++- web/js/views/settings.js | 46 +++++++++++++++++++++- 17 files changed, 265 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11e7123..72fe135 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,11 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/). -## [0.2.29] - 2026-07-21 +## [0.2.30] - 2026-07-22 + +### Added +- **Per-Model Generation & Reasoning Parameters**: Full support in Web UI and Gateway core for per-model generation defaults: Temperature, Top-P, Max Tokens, Think Budget (Reasoning mode), O(1) Constant-Memory Streaming Attention (`o1`), MTP (Multi-Token Prediction toggle `skip_mtp`), and default System Prompt. + ### Fixed - **Duplicate Local Model Registration**: Fixed `duplicate managed model id` conflict in `register_import` by cleanly routing managed CMF models directly into `[[cmf.servers]]` with automatic free port allocation without pushing duplicate entries to `[[models]]`. diff --git a/Cargo.lock b/Cargo.lock index 88ed782..09a6171 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -285,7 +285,7 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "cortiq-gateway" -version = "0.2.29" +version = "0.2.30" dependencies = [ "anyhow", "arc-swap", diff --git a/Cargo.toml b/Cargo.toml index f4d91fd..50ba7da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cortiq-gateway" -version = "0.2.29" +version = "0.2.30" edition = "2021" rust-version = "1.75" license = "Apache-2.0" diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 38fa425..014b08c 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -1330,6 +1330,13 @@ async fn register_import(State(state): State, Path(job): Path, Path(job): Path Result { if server.threads > 1 { cmd.env("CMF_THREADS", server.threads.to_string()); } - // Offload large matvecs to the GPU (Metal on macOS). Memory-bandwidth-bound - // on unified memory, so mostly a prefill win; opt-in. if server.gpu { cmd.env("CMF_GPU", "1"); } + if let Some(o1) = &server.o1 { + if !o1.trim().is_empty() && o1 != "off" { + cmd.arg("--o1").arg(o1); + } + } + if server.skip_mtp { + cmd.arg("--skip-mtp"); + } cmd.stdout(Stdio::null()) .stderr(Stdio::null()) .kill_on_drop(true) diff --git a/src/config.rs b/src/config.rs index b280281..0b6a443 100644 --- a/src/config.rs +++ b/src/config.rs @@ -83,6 +83,27 @@ pub struct CmfServer { /// Offload matvecs to the GPU (CMF_GPU=1). #[serde(default)] pub gpu: bool, + /// Default generation temperature (0.0 .. 2.0). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub temperature: Option, + /// Default nucleus sampling threshold (0.0 .. 1.0). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub top_p: Option, + /// Default maximum completion tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, + /// Reasoning / thinking token budget. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub think_budget: Option, + /// Default system prompt prepended if request lacks one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub system_prompt: Option, + /// O(1) constant-memory streaming attention hint (all, deep, custom, off). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub o1: Option, + /// Skip Multi-Token Prediction (MTP) heads. + #[serde(default)] + pub skip_mtp: bool, } /// CMF-format model factory: import a HuggingFace model, convert it to a @@ -172,6 +193,13 @@ impl CmfCfg { port: self.local_port, threads: self.threads, gpu: self.gpu, + temperature: None, + top_p: None, + max_tokens: None, + think_budget: None, + system_prompt: None, + o1: None, + skip_mtp: false, }]; } Vec::new() @@ -306,6 +334,27 @@ pub struct ModelCfg { pub api_key_env: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub caps: Vec, + /// Default generation temperature (0.0 .. 2.0). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub temperature: Option, + /// Default nucleus sampling threshold (0.0 .. 1.0). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub top_p: Option, + /// Default maximum completion tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, + /// Reasoning / thinking token budget. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub think_budget: Option, + /// Default system prompt prepended if request lacks one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub system_prompt: Option, + /// O(1) constant-memory streaming attention hint (all, deep, custom, off). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub o1: Option, + /// Skip Multi-Token Prediction (MTP) heads. + #[serde(default)] + pub skip_mtp: bool, } fn default_kind() -> String { "chat".into() diff --git a/src/model.rs b/src/model.rs index ec9efe6..aa840a2 100644 --- a/src/model.rs +++ b/src/model.rs @@ -29,6 +29,7 @@ pub struct GenParams { pub temperature: Option, pub max_tokens: Option, pub top_p: Option, + pub think_budget: Option, pub stop: Vec, /// Fields we do not interpret but must proxy through to the provider. pub passthrough: serde_json::Map, diff --git a/src/pipeline.rs b/src/pipeline.rs index 9f6139e..226b5e7 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -341,7 +341,64 @@ impl Pipeline { route_source ); - match provider.chat(req.clone()).await { + let mut req_call = req.clone(); + if let Some(m_cfg) = live.cfg.models.iter().find(|m| m.id == model_id) { + if req_call.params.temperature.is_none() { + req_call.params.temperature = m_cfg.temperature; + } + if req_call.params.top_p.is_none() { + req_call.params.top_p = m_cfg.top_p; + } + if req_call.params.max_tokens.is_none() { + req_call.params.max_tokens = m_cfg.max_tokens; + } + if req_call.params.think_budget.is_none() { + req_call.params.think_budget = m_cfg.think_budget; + } + if let Some(sys) = &m_cfg.system_prompt { + if !sys.trim().is_empty() + && !req_call.messages.iter().any(|m| m.role == "system") + { + req_call.messages.insert( + 0, + Message { + role: "system".into(), + content: sys.clone(), + tool_calls: Vec::new(), + }, + ); + } + } + } else if let Some(s_cfg) = live.cfg.cmf.servers.iter().find(|s| s.id == model_id) { + if req_call.params.temperature.is_none() { + req_call.params.temperature = s_cfg.temperature; + } + if req_call.params.top_p.is_none() { + req_call.params.top_p = s_cfg.top_p; + } + if req_call.params.max_tokens.is_none() { + req_call.params.max_tokens = s_cfg.max_tokens; + } + if req_call.params.think_budget.is_none() { + req_call.params.think_budget = s_cfg.think_budget; + } + if let Some(sys) = &s_cfg.system_prompt { + if !sys.trim().is_empty() + && !req_call.messages.iter().any(|m| m.role == "system") + { + req_call.messages.insert( + 0, + Message { + role: "system".into(), + content: sys.clone(), + tool_calls: Vec::new(), + }, + ); + } + } + } + + match provider.chat(req_call).await { Ok(mut resp) => { let cost = provider.price(resp.usage.prompt_tokens, resp.usage.completion_tokens); diff --git a/src/protocols/anthropic_messages.rs b/src/protocols/anthropic_messages.rs index 65c04b9..3f6e3d4 100644 --- a/src/protocols/anthropic_messages.rs +++ b/src/protocols/anthropic_messages.rs @@ -101,6 +101,7 @@ async fn handler( temperature: req.temperature, max_tokens: req.max_tokens, top_p: req.top_p, + think_budget: None, stop: Vec::new(), passthrough: Default::default(), }, diff --git a/src/protocols/openai_chat.rs b/src/protocols/openai_chat.rs index 777bda9..d17592e 100644 --- a/src/protocols/openai_chat.rs +++ b/src/protocols/openai_chat.rs @@ -49,6 +49,10 @@ struct OpenAiChatRequest { #[serde(default)] top_p: Option, #[serde(default)] + think_budget: Option, + #[serde(default)] + reasoning_effort: Option, + #[serde(default)] stream: bool, #[serde(default)] tools: Vec, @@ -93,6 +97,7 @@ async fn handler( temperature: req.temperature, max_tokens: req.max_tokens, top_p: req.top_p, + think_budget: req.think_budget.or(req.reasoning_effort), stop: Vec::new(), passthrough: req.rest, }, diff --git a/src/protocols/openai_completions.rs b/src/protocols/openai_completions.rs index 92e9b5f..df47d7d 100644 --- a/src/protocols/openai_completions.rs +++ b/src/protocols/openai_completions.rs @@ -82,6 +82,7 @@ async fn handler( temperature: req.temperature, max_tokens: req.max_tokens, top_p: req.top_p, + think_budget: None, stop: Vec::new(), passthrough: Default::default(), }, diff --git a/src/providers/openai.rs b/src/providers/openai.rs index 99275d3..33aeed1 100644 --- a/src/providers/openai.rs +++ b/src/providers/openai.rs @@ -42,9 +42,16 @@ impl OpenAiProvider { if let Some(t) = req.params.temperature { body["temperature"] = t.into(); } + if let Some(p) = req.params.top_p { + body["top_p"] = p.into(); + } if let Some(m) = req.params.max_tokens { body["max_tokens"] = m.into(); } + if let Some(tb) = req.params.think_budget { + body["think_budget"] = tb.into(); + body["reasoning_effort"] = tb.into(); + } if !req.tools.is_empty() { body["tools"] = serde_json::Value::Array(req.tools.clone()); } diff --git a/src/registry.rs b/src/registry.rs index 3e2fefb..2c2689b 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -49,6 +49,13 @@ impl Registry { kind: "chat".into(), api_key_env: None, caps: Vec::new(), + temperature: s.temperature, + top_p: s.top_p, + max_tokens: s.max_tokens, + think_budget: s.think_budget, + system_prompt: s.system_prompt.clone(), + o1: s.o1.clone(), + skip_mtp: s.skip_mtp, }; by_id.insert(m.id.clone(), Arc::new(OpenAiProvider::new(&m, None)?)); } diff --git a/web/js/i18n/en.js b/web/js/i18n/en.js index c21cd0d..de80b72 100644 --- a/web/js/i18n/en.js +++ b/web/js/i18n/en.js @@ -139,6 +139,18 @@ export default { "models.form.apiKeyHint": "Stored securely on the gateway; it is not shown again.", "models.form.keyStored": "A key is saved — leave blank to keep it, or paste a new one.", "models.form.keyFromEnv": "Using a key from an environment variable.", + "models.form.paramsTitle": "Generation & Thinking Parameters (O(1), MTP, Temperature)", + "models.form.temperature": "Temperature", + "models.form.temperatureHint": "Generation temperature (0.0 = deterministic, 1.0+ = creative).", + "models.form.topP": "Top-P (Nucleus Sampling)", + "models.form.topPHint": "Sampling threshold (0.0 .. 1.0).", + "models.form.maxTokens": "Max Tokens", + "models.form.maxTokensHint": "Maximum completion tokens to generate.", + "models.form.thinkBudget": "Thinking Mode / Think Budget", + "models.form.thinkBudgetHint": "Reasoning token budget (0 = disabled, N > 0 = max reasoning tokens).", + "models.form.systemPrompt": "Default System Prompt", + "models.form.systemPromptHint": "Prepend automatically if request lacks a system prompt.", + "models.form.systemPromptPlaceholder": "You are a helpful assistant...", "models.form.testKey": "Test & load models", "models.form.testing": "checking…", "models.form.foundModels": "{n} models loaded", diff --git a/web/js/i18n/ru.js b/web/js/i18n/ru.js index 8db2116..3a304cb 100644 --- a/web/js/i18n/ru.js +++ b/web/js/i18n/ru.js @@ -138,6 +138,18 @@ export default { "models.form.apiKeyHint": "Хранится безопасно на шлюзе; больше не показывается.", "models.form.keyStored": "Ключ сохранён — оставьте пустым, чтобы не менять, или вставьте новый.", "models.form.keyFromEnv": "Используется ключ из переменной окружения.", + "models.form.paramsTitle": "Параметры генерации, O(1) и режим размышления", + "models.form.temperature": "Температура", + "models.form.temperatureHint": "Температура генерации (0.0 — точные факты, 1.0+ — творчество).", + "models.form.topP": "Top-P (Nucleus Sampling)", + "models.form.topPHint": "Порог сэмплирования токенов (0.0 .. 1.0).", + "models.form.maxTokens": "Макс. токенов (Max Tokens)", + "models.form.maxTokensHint": "Максимальное количество генерируемых токенов.", + "models.form.thinkBudget": "Режим размышления / Think Budget", + "models.form.thinkBudgetHint": "Бюджет токенов мыслей (0 = выключен, N > 0 = лимит рассуждений).", + "models.form.systemPrompt": "Системный промпт по умолчанию", + "models.form.systemPromptHint": "Автоматически подставляется, если запрос не содержит системное сообщение.", + "models.form.systemPromptPlaceholder": "Вы — полезный ассистент...", "models.form.testKey": "Проверить и загрузить модели", "models.form.testing": "проверка…", "models.form.foundModels": "загружено моделей: {n}", diff --git a/web/js/views/models.js b/web/js/views/models.js index c1b809b..88e26c8 100644 --- a/web/js/views/models.js +++ b/web/js/views/models.js @@ -118,6 +118,36 @@ function modelForm(meta, existing) { : existing && existing.key_source === "env" ? t("models.form.keyFromEnv") : t("models.form.apiKeyHint"); + // ⚙ Generation & Thinking Parameters + const tempIn = h("input", { type: "number", step: "0.05", min: "0", max: "2.0", placeholder: "0.7", value: m.temperature ?? "" }); + const topPIn = h("input", { type: "number", step: "0.05", min: "0", max: "1.0", placeholder: "0.9", value: m.top_p ?? "" }); + const maxTokIn = h("input", { type: "number", min: "1", placeholder: "2048", value: m.max_tokens ?? "" }); + const thinkIn = h("input", { type: "number", min: "0", placeholder: "1024", value: m.think_budget ?? "" }); + const o1Sel = h("select", {}, + opt("", t("import.adv.auto")), + opt("all", "all — O(1) Attention"), + opt("deep", "deep — O(1) Deep"), + opt("off", "off — Standard")); + if (m.o1) o1Sel.value = m.o1; + const skipMtpCb = h("input", { type: "checkbox", checked: m.skip_mtp ? true : null }); + const sysPromptIn = h("textarea", { rows: 2, placeholder: t("models.form.systemPromptPlaceholder"), value: m.system_prompt || "" }); + + const advParams = h( + "details", + { class: "adv-params", style: "margin-top:12px;padding:10px;border:1px solid var(--border);border-radius:6px;" }, + h("summary", { style: "font-weight:600;cursor:pointer;" }, "⚙ " + t("models.form.paramsTitle")), + h("div", { class: "row", style: "margin-top:8px" }, + field(t("models.form.temperature"), tempIn, t("models.form.temperatureHint")), + field(t("models.form.topP"), topPIn, t("models.form.topPHint"))), + h("div", { class: "row" }, + field(t("models.form.maxTokens"), maxTokIn, t("models.form.maxTokensHint")), + field(t("models.form.thinkBudget"), thinkIn, t("models.form.thinkBudgetHint"))), + h("div", { class: "row" }, + field("O(1) Attention", o1Sel, t("import.adv.o1Hint")), + field("MTP", h("label", { class: "check" }, skipMtpCb, t("import.adv.skipMtp")))), + field(t("models.form.systemPrompt"), sysPromptIn, t("models.form.systemPromptHint")) + ); + const node = h( "div", {}, @@ -128,7 +158,8 @@ function modelForm(meta, existing) { dl, h("div", { class: "row" }, field(t("models.form.kind"), kindSel), field(t("models.form.costTier"), tierSel)), h("div", { class: "row" }, field(t("models.form.priceIn"), priceInIn), field(t("models.form.priceOut"), priceOutIn)), - field(t("models.form.caps"), h("div", { class: "flex wrap" }, ...capInputs.map(({ c, cb }) => h("label", { class: "check" }, cb, c)))) + field(t("models.form.caps"), h("div", { class: "flex wrap" }, ...capInputs.map(({ c, cb }) => h("label", { class: "check" }, cb, c)))), + advParams ); const getValue = () => { @@ -147,6 +178,13 @@ function modelForm(meta, existing) { price_out: parseFloat(priceOutIn.value) || 0, api_key_env: hasKey ? keyEnv : null, caps: capInputs.filter(({ cb }) => cb.checked).map(({ c }) => c), + temperature: tempIn.value !== "" ? parseFloat(tempIn.value) : null, + top_p: topPIn.value !== "" ? parseFloat(topPIn.value) : null, + max_tokens: maxTokIn.value !== "" ? parseInt(maxTokIn.value) : null, + think_budget: thinkIn.value !== "" ? parseInt(thinkIn.value) : null, + o1: o1Sel.value || null, + skip_mtp: skipMtpCb.checked, + system_prompt: sysPromptIn.value.trim() || null, }, secret: typedKey, keyEnv, diff --git a/web/js/views/settings.js b/web/js/views/settings.js index 55827aa..84b3ab1 100644 --- a/web/js/views/settings.js +++ b/web/js/views/settings.js @@ -253,9 +253,51 @@ export async function renderSettings() { } catch (e) { mount(st, String(e.message || e)); } checkBtn.disabled = false; }); + // ⚙ Model Generation & Thinking Parameters drawer + const tempIn = h("input", { type: "number", step: "0.05", min: "0", max: "2.0", placeholder: "0.7", value: sv.temperature ?? "", style: "width:65px" }); + const topPIn = h("input", { type: "number", step: "0.05", min: "0", max: "1.0", placeholder: "0.9", value: sv.top_p ?? "", style: "width:65px" }); + const maxTokIn = h("input", { type: "number", min: "1", placeholder: "2048", value: sv.max_tokens ?? "", style: "width:80px" }); + const thinkIn = h("input", { type: "number", min: "0", placeholder: "1024", value: sv.think_budget ?? "", style: "width:80px" }); + const o1Sel = h("select", { style: "width:110px" }, + opt("", "O(1) auto"), + opt("all", "O(1) all"), + opt("deep", "O(1) deep"), + opt("off", "O(1) off")); + if (sv.o1) o1Sel.value = sv.o1; + const skipMtpCb = h("input", { type: "checkbox", checked: sv.skip_mtp ? true : null }); + const sysIn = h("input", { placeholder: t("models.form.systemPromptPlaceholder"), value: sv.system_prompt || "", style: "flex:1" }); + + const drawer = h("div", { class: "flex wrap gap-sm", style: "display:none;width:100%;margin-top:4px;padding:6px;background:var(--bg-subtle);border-radius:4px;align-items:center" }, + h("span", { class: "small muted" }, "Temp:"), tempIn, + h("span", { class: "small muted" }, "Top-P:"), topPIn, + h("span", { class: "small muted" }, "MaxTokens:"), maxTokIn, + h("span", { class: "small muted" }, "ThinkBudget:"), thinkIn, + o1Sel, + h("label", { class: "check small" }, skipMtpCb, "Skip MTP"), + h("span", { class: "small muted" }, "System:"), sysIn + ); + + const gearBtn = h("button", { class: "btn small", type: "button", title: "⚙ " + t("models.form.paramsTitle") }, "⚙"); + gearBtn.addEventListener("click", () => { + drawer.style.display = drawer.style.display === "none" ? "flex" : "none"; + }); + const row = h("div", { class: "flex wrap", style: "gap:6px;align-items:center;margin:6px 0" }, - idIn, modelSel, portIn, checkBtn, threadsIn, h("label", { class: "flex", style: "gap:4px" }, gpuIn, "GPU"), removeBtn, st); - const entry = { node: row, read: () => ({ id: idIn.value.trim() || autoId(modelSel.value), model: modelSel.value.trim(), port: parseInt(portIn.value) || 8090, threads: parseInt(threadsIn.value) || 0, gpu: gpuIn.checked }) }; + idIn, modelSel, portIn, checkBtn, threadsIn, h("label", { class: "flex", style: "gap:4px" }, gpuIn, "GPU"), gearBtn, removeBtn, st, drawer); + const entry = { node: row, read: () => ({ + id: idIn.value.trim() || autoId(modelSel.value), + model: modelSel.value.trim(), + port: parseInt(portIn.value) || 8090, + threads: parseInt(threadsIn.value) || 0, + gpu: gpuIn.checked, + temperature: tempIn.value !== "" ? parseFloat(tempIn.value) : null, + top_p: topPIn.value !== "" ? parseFloat(topPIn.value) : null, + max_tokens: maxTokIn.value !== "" ? parseInt(maxTokIn.value) : null, + think_budget: thinkIn.value !== "" ? parseInt(thinkIn.value) : null, + o1: o1Sel.value || null, + skip_mtp: skipMtpCb.checked, + system_prompt: sysIn.value.trim() || null, + }) }; removeBtn.addEventListener("click", () => { const i = serverRows.indexOf(entry); if (i >= 0) serverRows.splice(i, 1); row.remove(); }); serverRows.push(entry); serversWrap.appendChild(row);