feat(import): O(1) attention options in Advanced (cortiq >= 0.2.0)

- New Advanced controls on the import page: O(1) attention spec (off / all /
  deep N / explicit layer list) plus the --o1-m / --o1-window / --o1-sink knobs
  with validated defaults (32 / 128 / 4) and explanatory hints.
- Backend passes --o1* to the native `cortiq convert`; fails fast with clear
  messages when combined with Python-only options, used on GGUF imports, or
  when the installed cortiq predates 0.2.0 (the hint needs the new runtime).
- The hint is recorded in the .cmf header (weights byte-identical); the runtime
  applies it at load. Verified end-to-end: converted a model with --o1 all and
  the runtime logged "o1 nystrom attention: 2/2 layer(s)".
This commit is contained in:
infosave2007
2026-07-14 10:16:54 +03:00
parent b55e72789f
commit c59ed9c949
7 changed files with 126 additions and 3 deletions
+59
View File
@@ -71,6 +71,20 @@ pub struct ImportParams {
pub shard_max_gb: Option<f32>,
#[serde(default)]
pub skip_mtp: bool,
/// O(1) Nyström attention hint (native converter, cortiq ≥ 0.2.0):
/// `all` | `deepN` | `i,j,k`. Weights pass through unchanged — the runtime
/// reads the hint at load. Empty/None/"off" = exact attention.
#[serde(default)]
pub o1: Option<String>,
/// Landmark budget for the o1 hint (validated default 32).
#[serde(default)]
pub o1_m: Option<usize>,
/// Exact-window width for the o1 hint (validated default 128).
#[serde(default)]
pub o1_window: Option<usize>,
/// Permanent exact sink keys for the o1 hint (validated default 4).
#[serde(default)]
pub o1_sink: Option<usize>,
}
fn default_quant() -> String {
"Q8_2F".into()
@@ -269,6 +283,34 @@ pub fn start_import(store: Arc<JobStore>, cfg: &CmfCfg, p: ImportParams) -> Resu
cfg.converter
));
}
// O(1) attention hint — native-converter feature (cortiq ≥ 0.2.0). Fail fast
// with a clear message instead of a mid-job "unexpected argument" error.
let o1 =
p.o1.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty() && *s != "off")
.map(str::to_string);
if o1.is_some() {
if use_python {
return Err(
"O(1) attention is supported by the native converter only — remove the \
linear-core / v-bit shape / mean-bits options to use it"
.into(),
);
}
if p.repo.to_ascii_lowercase().contains("gguf") {
return Err("O(1) attention is not supported for GGUF imports yet".into());
}
let ver = crate::cmf_runtime::installed_version(&cfg.cortiq_bin);
if let Some(v) = &ver {
if crate::cmf_runtime::version_lt(v, "0.2.0") {
return Err(format!(
"O(1) attention needs cortiq ≥ 0.2.0 (installed: {v}) — update the runtime \
in Settings → Local models"
));
}
}
}
let _ = std::fs::create_dir_all(&cfg.models_dir);
let base = if p.name.trim().is_empty() {
sanitize(p.repo.rsplit('/').next().unwrap_or(&p.repo))
@@ -384,6 +426,23 @@ pub fn start_import(store: Arc<JobStore>, cfg: &CmfCfg, p: ImportParams) -> Resu
a.push("--hf-token".into());
a.push(t.clone());
}
// O(1) attention hint (weights unchanged; the runtime reads it at load)
if let Some(spec) = &o1 {
a.push("--o1".into());
a.push(spec.clone());
if let Some(m) = p.o1_m {
a.push("--o1-m".into());
a.push(m.to_string());
}
if let Some(w) = p.o1_window {
a.push("--o1-window".into());
a.push(w.to_string());
}
if let Some(s) = p.o1_sink {
a.push("--o1-sink".into());
a.push(s.to_string());
}
}
(
cfg.cortiq_bin.clone(),
a,