Compare commits

...
2 Commits
Author SHA1 Message Date
infosave2007 8e369f99af feat(playground): top-p and a thoughts switch (auto/off/on)
Docker / publish (push) Canceled after 0s
Release / crates-io (push) Canceled after 0s
Release / release (macos-14, aarch64-apple-darwin) (push) Canceled after 0s
Release / release (ubuntu-latest, x86_64-unknown-linux-gnu) (push) Canceled after 0s
Release / release (windows-latest, x86_64-pc-windows-msvc) (push) Canceled after 0s
Playground gains top-p and a reasoning selector matching the Local-models row:
auto follows the model/row default; off/on override per request via
enable_thinking (cortiq >= 0.2.1 for local models). /admin/api/test(+stream)
accept top_p and enable_thinking, proxied through GenParams passthrough.
2026-07-14 16:13:19 +03:00
infosave2007 4754e1a065 feat(cmf): per-model reasoning switch — 'thoughts: off' for thinking models
Docker / publish (push) Canceled after 0s
Release / crates-io (push) Canceled after 0s
Release / release (macos-14, aarch64-apple-darwin) (push) Canceled after 0s
Release / release (ubuntu-latest, x86_64-unknown-linux-gnu) (push) Canceled after 0s
Release / release (windows-latest, x86_64-pc-windows-msvc) (push) Canceled after 0s
Each Local-models row gains a thoughts selector (auto/off). Off injects
enable_thinking=false (cortiq >= 0.2.1 renders the chat template with an empty
<think> block) into requests that don't set it themselves — the model answers
directly, faster and shorter. A client-supplied enable_thinking always wins.
2026-07-14 15:55:43 +03:00
9 changed files with 79 additions and 14 deletions
Generated
+1 -1
View File
@@ -285,7 +285,7 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
[[package]]
name = "cortiq-gateway"
version = "0.2.21"
version = "0.2.23"
dependencies = [
"anyhow",
"arc-swap",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "cortiq-gateway"
version = "0.2.21"
version = "0.2.23"
edition = "2021"
rust-version = "1.75"
license = "Apache-2.0"
+27 -10
View File
@@ -1447,24 +1447,44 @@ struct TestBody {
temperature: Option<f32>,
#[serde(default)]
max_tokens: Option<u32>,
#[serde(default)]
top_p: Option<f32>,
/// Reasoning switch (cortiq ≥ 0.2.1): false = answer without the thinking
/// phase; true = force it on. Absent = model/row default.
#[serde(default)]
enable_thinking: Option<bool>,
}
fn default_test_model() -> String {
"cortiq-auto".into()
}
/// GenParams for a playground request: native sampling fields + the reasoning
/// switch proxied via passthrough (providers forward unknown fields as-is).
fn test_gen_params(b: &TestBody) -> GenParams {
let mut params = GenParams {
temperature: b.temperature,
max_tokens: b.max_tokens,
top_p: b.top_p,
..Default::default()
};
if let Some(v) = b.enable_thinking {
params
.passthrough
.insert("enable_thinking".into(), serde_json::Value::Bool(v));
}
params
}
async fn run_test(State(state): State<SharedState>, Json(b): Json<TestBody>) -> ApiResult {
if b.messages.is_empty() {
return Err(ApiError::bad("messages must not be empty"));
}
let params = test_gen_params(&b);
let req = ChatRequest {
routing: parse_routing(&b.model),
messages: b.messages,
tools: vec![],
params: GenParams {
temperature: b.temperature,
max_tokens: b.max_tokens,
..Default::default()
},
params,
stream: false,
meta: RequestMeta {
account: "playground".into(),
@@ -1513,15 +1533,12 @@ async fn run_test_stream(State(state): State<SharedState>, Json(b): Json<TestBod
if b.messages.is_empty() {
return ApiError::bad("messages must not be empty").into_response();
}
let params = test_gen_params(&b);
let req = ChatRequest {
routing: parse_routing(&b.model),
messages: b.messages,
tools: vec![],
params: GenParams {
temperature: b.temperature,
max_tokens: b.max_tokens,
..Default::default()
},
params,
stream: true,
meta: RequestMeta {
account: "playground".into(),
+7
View File
@@ -97,6 +97,12 @@ pub struct CmfServer {
pub top_p: Option<f32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_tokens: Option<u32>,
/// Reasoning switch for thinking models (Qwen3/3.5 …): "off" injects
/// enable_thinking=false into requests that don't set it themselves —
/// the model answers directly, skipping the <think> phase. Needs
/// cortiq ≥ 0.2.1. Empty = model/template default.
#[serde(default)]
pub thinking: String,
}
/// CMF-format model factory: import a HuggingFace model, convert it to a
@@ -190,6 +196,7 @@ impl CmfCfg {
temperature: None,
top_p: None,
max_tokens: None,
thinking: String::new(),
}];
}
Vec::new()
+8
View File
@@ -319,6 +319,14 @@ impl Pipeline {
if req.params.max_tokens.is_none() {
req.params.max_tokens = s.max_tokens;
}
// reasoning switch: "off" → enable_thinking=false (cortiq ≥ 0.2.1
// renders the chat template with an empty <think> block); a client
// that sets enable_thinking itself wins.
if s.thinking == "off" && !req.params.passthrough.contains_key("enable_thinking") {
req.params
.passthrough
.insert("enable_thinking".into(), serde_json::Value::Bool(false));
}
}
}
+9
View File
@@ -223,6 +223,12 @@ export default {
"pg.model": "Model",
"pg.temp": "Temperature",
"pg.maxTokens": "Max tokens",
"pg.think": "Thoughts",
"pg.think.auto": "auto (model default)",
"pg.think.off": "off — answer directly",
"pg.think.on": "on — force reasoning",
"pg.thinkHelp": "Reasoning phase of thinking models. Auto = model/row default; off/on override for this request (needs cortiq >= 0.2.1 for local models).",
"pg.topPHelp": "Nucleus sampling: only tokens within this cumulative probability are sampled. Empty = default.",
"pg.send": "Send",
"pg.stream": "Stream",
"pg.sending": "Sending…",
@@ -265,6 +271,9 @@ export default {
"settings.cmf.models": "Local models",
"settings.cmf.modelsHelp": "Each row is one model the gateway runs (its own cortiq serve on its own port) and registers in the pool. Add as many as you have .cmf files and free ports.",
"settings.cmf.rowHint": "id · .cmf file · port · threads · GPU · O(1) · T°/top-p/max tok. Applied on save: a removed row stops its server (memory freed), a new row starts one.",
"settings.cmf.thinkAuto": "thoughts: auto",
"settings.cmf.thinkOff": "thoughts: off",
"settings.cmf.thinkHelp": "Reasoning models (Qwen3/3.5) write out a thinking phase before the answer. Off = answer directly without it (faster, shorter; needs cortiq >= 0.2.1). A request that sets enable_thinking itself always wins.",
"settings.cmf.tempHelp": "Default temperature for this model (0 = deterministic, 0.7 = balanced, >1 = creative). Used when the request doesn't set one; a request value always wins. Empty = provider default.",
"settings.cmf.topPHelp": "Default nucleus sampling (top-p) for this model: only tokens within this cumulative probability are sampled (0.9 typical). Empty = provider default.",
"settings.cmf.maxTokHelp": "Default max output tokens for this model when the request doesn't set one. Protects against runaway long generations. Empty = provider default.",
+9
View File
@@ -222,6 +222,12 @@ export default {
"pg.model": "Модель",
"pg.temp": "Температура",
"pg.maxTokens": "Макс. токены",
"pg.think": "Мысли",
"pg.think.auto": "авто (по настройке модели)",
"pg.think.off": "выкл — отвечать сразу",
"pg.think.on": "вкл — принудительно размышлять",
"pg.thinkHelp": "Фаза размышлений reasoning-моделей. Авто = настройка модели; выкл/вкл переопределяют для этого запроса (для локальных нужен cortiq ≥ 0.2.1).",
"pg.topPHelp": "Nucleus-сэмплинг: выбираются токены в пределах этой суммарной вероятности. Пусто = по умолчанию.",
"pg.send": "Отправить",
"pg.stream": "Стриминг",
"pg.sending": "Отправка…",
@@ -264,6 +270,9 @@ export default {
"settings.cmf.models": "Локальные модели",
"settings.cmf.modelsHelp": "Каждая строка — отдельная модель: шлюз поднимает свой cortiq serve на своём порту и регистрирует её в пуле. Добавляйте столько, сколько есть .cmf-файлов и свободных портов.",
"settings.cmf.rowHint": "id · файл .cmf · порт · потоки · GPU · O(1) · T°/top-p/max tok. Применяется сразу при сохранении: удалённая строка останавливает сервер (память освобождается), новая — запускает.",
"settings.cmf.thinkAuto": "мысли: авто",
"settings.cmf.thinkOff": "мысли: выкл",
"settings.cmf.thinkHelp": "Рассуждающие модели (Qwen3/3.5) сначала пишут фазу размышлений. Выкл = отвечать сразу, без неё (быстрее и короче; нужен cortiq ≥ 0.2.1). Если запрос сам задал enable_thinking — он важнее.",
"settings.cmf.tempHelp": "Температура по умолчанию для этой модели (0 = детерминированно, 0.7 = баланс, >1 = креативно). Подставляется, если запрос её не задал; значение из запроса всегда важнее. Пусто = дефолт провайдера.",
"settings.cmf.topPHelp": "Nucleus-сэмплинг (top-p) по умолчанию: выбираются только токены в пределах этой суммарной вероятности (обычно 0.9). Пусто = дефолт провайдера.",
"settings.cmf.maxTokHelp": "Максимум выходных токенов по умолчанию, если запрос не задал свой. Защищает от бесконечных генераций. Пусто = дефолт провайдера.",
+11
View File
@@ -16,6 +16,12 @@ export async function renderPlayground() {
const promptIn = h("textarea", { placeholder: t("pg.promptPlaceholder"), rows: 5 });
const tempIn = h("input", { type: "number", step: "0.1", value: 0.7 });
const maxTokIn = h("input", { type: "number", value: 512 });
const topPIn = h("input", { type: "number", step: "0.05", min: 0, max: 1, placeholder: "0.9", title: t("pg.topPHelp") });
// reasoning switch: auto = model/row default; off/on override per request
const thinkSel = h("select", { title: t("pg.thinkHelp") },
h("option", { value: "" }, t("pg.think.auto")),
h("option", { value: "off" }, t("pg.think.off")),
h("option", { value: "on" }, t("pg.think.on")));
const streamCb = h("input", { type: "checkbox", checked: true });
const profileSel = h("select", {}, ...(meta.profiles || []).map((p) => h("option", { value: p }, p)));
const modelSel = h("select", {}, ...ids.map((id) => h("option", { value: id }, id)));
@@ -69,6 +75,8 @@ export async function renderPlayground() {
temperature: parseFloat(tempIn.value),
max_tokens: parseInt(maxTokIn.value) || null,
};
if (topPIn.value !== "") body.top_p = parseFloat(topPIn.value);
if (thinkSel.value) body.enable_thinking = thinkSel.value === "on";
try {
if (streamCb.checked) {
const started = performance.now();
@@ -151,6 +159,9 @@ export async function renderPlayground() {
h("label", { class: "field" }, h("span", {}, t("pg.mode")), modeSeg),
modeExtra,
h("div", { class: "row" }, h("label", { class: "field" }, h("span", {}, t("pg.temp")), tempIn), h("label", { class: "field" }, h("span", {}, t("pg.maxTokens")), maxTokIn)),
h("div", { class: "row" },
h("label", { class: "field" }, h("span", {}, "top-p"), topPIn),
h("label", { class: "field" }, h("span", {}, t("pg.think")), thinkSel)),
h(
"div",
{ class: "flex", style: "justify-content:space-between" },
+6 -2
View File
@@ -240,6 +240,10 @@ export async function renderSettings() {
...o1Opts.map((v) => h("option", { value: v, selected: (sv.o1 || "") === v ? true : null },
v === "" ? t("settings.cmf.o1Auto") : v)));
if (sv.o1 && !o1Opts.includes(sv.o1)) o1In.appendChild(h("option", { value: sv.o1, selected: true }, sv.o1));
// reasoning switch for thinking models (off → enable_thinking=false)
const thinkIn = h("select", { style: "width:118px", title: t("settings.cmf.thinkHelp") },
h("option", { value: "", selected: !sv.thinking ? true : null }, t("settings.cmf.thinkAuto")),
h("option", { value: "off", selected: sv.thinking === "off" ? true : null }, t("settings.cmf.thinkOff")));
// per-model generation defaults (applied when the request doesn't set them)
const tempIn = h("input", { type: "number", step: "0.1", min: 0, max: 2, value: sv.temperature ?? "", placeholder: "T° 0.7", style: "width:74px", title: t("settings.cmf.tempHelp") });
const topPIn = h("input", { type: "number", step: "0.05", min: 0, max: 1, value: sv.top_p ?? "", placeholder: "top-p", style: "width:70px", title: t("settings.cmf.topPHelp") });
@@ -268,9 +272,9 @@ export async function renderSettings() {
checkBtn.disabled = false;
});
const row = h("div", { class: "flex wrap", style: "gap:6px;align-items:center;margin:6px 0" },
idIn, modelSel, portIn, checkBtn, threadsIn, h("label", { class: "flex", style: "gap:4px" }, gpuIn, "GPU"), o1In, tempIn, topPIn, maxTokIn, removeBtn, st);
idIn, modelSel, portIn, checkBtn, threadsIn, h("label", { class: "flex", style: "gap:4px" }, gpuIn, "GPU"), o1In, thinkIn, tempIn, topPIn, maxTokIn, removeBtn, st);
const entry = { node: row, read: () => {
const v = { id: idIn.value.trim() || autoId(modelSel.value), model: modelSel.value.trim(), port: parseInt(portIn.value) || 8090, threads: parseInt(threadsIn.value) || 0, gpu: gpuIn.checked, o1: o1In.value };
const v = { id: idIn.value.trim() || autoId(modelSel.value), model: modelSel.value.trim(), port: parseInt(portIn.value) || 8090, threads: parseInt(threadsIn.value) || 0, gpu: gpuIn.checked, o1: o1In.value, thinking: thinkIn.value };
if (tempIn.value !== "") v.temperature = parseFloat(tempIn.value);
if (topPIn.value !== "") v.top_p = parseFloat(topPIn.value);
if (maxTokIn.value !== "") v.max_tokens = parseInt(maxTokIn.value);