diff --git a/Cargo.lock b/Cargo.lock index deb3c1b..82ddbef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -285,7 +285,7 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "cortiq-gateway" -version = "0.2.14" +version = "0.2.15" dependencies = [ "anyhow", "arc-swap", diff --git a/Cargo.toml b/Cargo.toml index f947605..9f6fde0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cortiq-gateway" -version = "0.2.14" +version = "0.2.15" edition = "2021" rust-version = "1.75" license = "Apache-2.0" diff --git a/src/cmf_runtime.rs b/src/cmf_runtime.rs index 648bbcc..1873a2e 100644 --- a/src/cmf_runtime.rs +++ b/src/cmf_runtime.rs @@ -238,7 +238,7 @@ fn parse_ver(s: &str) -> (u32, u32, u32) { ) } -fn version_lt(a: &str, b: &str) -> bool { +pub(crate) fn version_lt(a: &str, b: &str) -> bool { parse_ver(a) < parse_ver(b) } diff --git a/src/import.rs b/src/import.rs index 767e851..7fbac61 100644 --- a/src/import.rs +++ b/src/import.rs @@ -71,6 +71,20 @@ pub struct ImportParams { pub shard_max_gb: Option, #[serde(default)] pub skip_mtp: bool, + /// O(1) Nyström attention hint (native converter, cortiq ≥ 0.2.0): + /// `all` | `deepN` | `i,j,k`. Weights pass through unchanged — the runtime + /// reads the hint at load. Empty/None/"off" = exact attention. + #[serde(default)] + pub o1: Option, + /// Landmark budget for the o1 hint (validated default 32). + #[serde(default)] + pub o1_m: Option, + /// Exact-window width for the o1 hint (validated default 128). + #[serde(default)] + pub o1_window: Option, + /// Permanent exact sink keys for the o1 hint (validated default 4). + #[serde(default)] + pub o1_sink: Option, } fn default_quant() -> String { "Q8_2F".into() @@ -269,6 +283,34 @@ pub fn start_import(store: Arc, cfg: &CmfCfg, p: ImportParams) -> Resu cfg.converter )); } + // O(1) attention hint — native-converter feature (cortiq ≥ 0.2.0). Fail fast + // with a clear message instead of a mid-job "unexpected argument" error. + let o1 = + p.o1.as_deref() + .map(str::trim) + .filter(|s| !s.is_empty() && *s != "off") + .map(str::to_string); + if o1.is_some() { + if use_python { + return Err( + "O(1) attention is supported by the native converter only — remove the \ + linear-core / v-bit shape / mean-bits options to use it" + .into(), + ); + } + if p.repo.to_ascii_lowercase().contains("gguf") { + return Err("O(1) attention is not supported for GGUF imports yet".into()); + } + let ver = crate::cmf_runtime::installed_version(&cfg.cortiq_bin); + if let Some(v) = &ver { + if crate::cmf_runtime::version_lt(v, "0.2.0") { + return Err(format!( + "O(1) attention needs cortiq ≥ 0.2.0 (installed: {v}) — update the runtime \ + in Settings → Local models" + )); + } + } + } let _ = std::fs::create_dir_all(&cfg.models_dir); let base = if p.name.trim().is_empty() { sanitize(p.repo.rsplit('/').next().unwrap_or(&p.repo)) @@ -384,6 +426,23 @@ pub fn start_import(store: Arc, cfg: &CmfCfg, p: ImportParams) -> Resu a.push("--hf-token".into()); a.push(t.clone()); } + // O(1) attention hint (weights unchanged; the runtime reads it at load) + if let Some(spec) = &o1 { + a.push("--o1".into()); + a.push(spec.clone()); + if let Some(m) = p.o1_m { + a.push("--o1-m".into()); + a.push(m.to_string()); + } + if let Some(w) = p.o1_window { + a.push("--o1-window".into()); + a.push(w.to_string()); + } + if let Some(s) = p.o1_sink { + a.push("--o1-sink".into()); + a.push(s.to_string()); + } + } ( cfg.cortiq_bin.clone(), a, diff --git a/web/js/i18n/en.js b/web/js/i18n/en.js index c188d6c..7858a2f 100644 --- a/web/js/i18n/en.js +++ b/web/js/i18n/en.js @@ -400,4 +400,15 @@ export default { "import.adv.shard": "Shard max (GB)", "import.adv.noShard": "no sharding", "import.adv.skipMtp": "skip MTP head", + "import.adv.o1": "O(1) attention", + "import.adv.o1Hint": "Constant-memory streaming attention: the KV cache is released after prefill, so long contexts don't grow memory. Weights stay byte-identical — only a load-time hint is recorded; you can switch back anytime (serve with --o1 off). Validated on Qwen3-0.6B: all layers ≈ ×1.18 perplexity.", + "import.adv.o1Off": "off — exact attention (default)", + "import.adv.o1All": "all — convert every softmax layer", + "import.adv.o1Deep": "deep N — only the N deepest layers", + "import.adv.o1Custom": "custom — explicit layer list", + "import.adv.o1DeepN": "N deep layers", + "import.adv.o1Layers": "Layers (comma-separated)", + "import.adv.o1mHint": "landmark budget (default 32)", + "import.adv.o1windowHint": "exact-window width (default 128)", + "import.adv.o1sinkHint": "permanent exact sink keys (default 4)", }; diff --git a/web/js/i18n/ru.js b/web/js/i18n/ru.js index 58464f3..ebb57ce 100644 --- a/web/js/i18n/ru.js +++ b/web/js/i18n/ru.js @@ -399,4 +399,15 @@ export default { "import.adv.shard": "Шард макс (ГБ)", "import.adv.noShard": "без шардинга", "import.adv.skipMtp": "пропустить MTP-голову", + "import.adv.o1": "O(1)-внимание", + "import.adv.o1Hint": "Потоковое внимание с постоянной памятью: KV-кэш освобождается после prefill, длинный контекст не раздувает память. Веса не меняются — записывается лишь подсказка на загрузку; вернуться можно всегда (запуск с --o1 off). Проверено на Qwen3-0.6B: все слои ≈ ×1.18 перплексии.", + "import.adv.o1Off": "выкл — точное внимание (по умолчанию)", + "import.adv.o1All": "all — конвертировать все softmax-слои", + "import.adv.o1Deep": "deep N — только N самых глубоких слоёв", + "import.adv.o1Custom": "вручную — явный список слоёв", + "import.adv.o1DeepN": "N глубоких слоёв", + "import.adv.o1Layers": "Слои (через запятую)", + "import.adv.o1mHint": "число опорных точек (по умолчанию 32)", + "import.adv.o1windowHint": "ширина точного окна (по умолчанию 128)", + "import.adv.o1sinkHint": "постоянные точные sink-ключи (по умолчанию 4)", }; diff --git a/web/js/views/import.js b/web/js/views/import.js index a2ad279..97ebba5 100644 --- a/web/js/views/import.js +++ b/web/js/views/import.js @@ -125,6 +125,32 @@ export function renderImport() { const meanBitsIn = h("input", { type: "number", step: "0.1", placeholder: "5.0" }); const shardIn = h("input", { type: "number", step: "0.5", placeholder: t("import.adv.noShard") }); const skipMtp = h("input", { type: "checkbox" }); + // O(1) attention (cortiq ≥ 0.2.0): constant-memory streaming attention hint, + // weights byte-identical — the runtime applies it at load. + const o1Sel = h("select", {}, + opt("", t("import.adv.o1Off")), + opt("all", t("import.adv.o1All")), + opt("deep", t("import.adv.o1Deep")), + opt("custom", t("import.adv.o1Custom"))); + const o1DeepIn = h("input", { type: "number", min: "1", placeholder: "12" }); + const o1CustomIn = h("input", { placeholder: "0,4,8" }); + const o1M = h("input", { type: "number", min: "1", placeholder: "32" }); + const o1Window = h("input", { type: "number", min: "0", placeholder: "128" }); + const o1Sink = h("input", { type: "number", min: "0", placeholder: "4" }); + const o1SpecRow = h("div", { class: "row", style: "display:none" }, + field(t("import.adv.o1DeepN"), o1DeepIn), + field(t("import.adv.o1Layers"), o1CustomIn)); + const o1Knobs = h("div", { class: "row", style: "display:none" }, + field("--o1-m", o1M, t("import.adv.o1mHint")), + field("--o1-window", o1Window, t("import.adv.o1windowHint")), + field("--o1-sink", o1Sink, t("import.adv.o1sinkHint"))); + o1Sel.addEventListener("change", () => { + const on = !!o1Sel.value; + o1Knobs.style.display = on ? "" : "none"; + o1SpecRow.style.display = o1Sel.value === "deep" || o1Sel.value === "custom" ? "" : "none"; + o1DeepIn.parentElement.style.display = o1Sel.value === "deep" ? "" : "none"; + o1CustomIn.parentElement.style.display = o1Sel.value === "custom" ? "" : "none"; + }); const advBody = h( "div", { class: "adv-body", style: "display:none" }, @@ -137,6 +163,9 @@ export function renderImport() { h("div", { class: "row" }, field(t("import.adv.shard"), shardIn), field("MTP", h("label", { class: "check" }, skipMtp, t("import.adv.skipMtp")))), + field(t("import.adv.o1"), o1Sel, t("import.adv.o1Hint")), + o1SpecRow, + o1Knobs, ); const advToggle = h("button", { class: "btn ghost sm" }, "⚙ " + t("import.adv.title")); advToggle.addEventListener("click", () => { @@ -156,6 +185,19 @@ export function renderImport() { if (vbitSel.value) params.vbit_shape = vbitSel.value; if (meanBitsIn.value) params.mean_bits = parseFloat(meanBitsIn.value); if (shardIn.value) params.shard_max_gb = parseFloat(shardIn.value); + if (o1Sel.value) { + // spec: all | deepN | explicit layer list + params.o1 = + o1Sel.value === "deep" ? "deep" + (parseInt(o1DeepIn.value) || 12) + : o1Sel.value === "custom" ? o1CustomIn.value.trim() + : "all"; + if (!params.o1) delete params.o1; + else { + if (o1M.value) params.o1_m = parseInt(o1M.value); + if (o1Window.value) params.o1_window = parseInt(o1Window.value); + if (o1Sink.value) params.o1_sink = parseInt(o1Sink.value); + } + } goBtn.disabled = true; try { await api.startImport(params);