Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4754e1a065 | ||
|
|
3c4be5e966 | ||
|
|
46c11b9b88 | ||
|
|
851cc7f1cb | ||
|
|
d93ac537c8 | ||
|
|
4220bc27c2 | ||
|
|
6fdbe0e159 | ||
|
|
243d8f4c7d | ||
|
|
4f72711a1e | ||
|
|
7855472499 | ||
|
|
a6ed974ece | ||
|
|
4c11b97197 | ||
|
|
87f00ee502 | ||
|
|
4fa1b8ea22 |
Generated
+1
-1
@@ -285,7 +285,7 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cortiq-gateway"
|
name = "cortiq-gateway"
|
||||||
version = "0.2.8"
|
version = "0.2.22"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "cortiq-gateway"
|
name = "cortiq-gateway"
|
||||||
version = "0.2.8"
|
version = "0.2.22"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
|
|||||||
+278
-30
@@ -165,7 +165,7 @@ pub fn api_routes(admin_token: String) -> Router<SharedState> {
|
|||||||
"/admin/api/secrets",
|
"/admin/api/secrets",
|
||||||
get(list_secrets).put(set_secret).delete(clear_secret),
|
get(list_secrets).put(set_secret).delete(clear_secret),
|
||||||
)
|
)
|
||||||
.route("/admin/api/stats", get(get_stats))
|
.route("/admin/api/stats", get(get_stats).delete(clear_stats))
|
||||||
.route("/admin/api/shadow", get(get_shadow))
|
.route("/admin/api/shadow", get(get_shadow))
|
||||||
.route("/admin/api/hf/search", get(hf_search))
|
.route("/admin/api/hf/search", get(hf_search))
|
||||||
.route("/admin/api/import", get(list_imports).post(start_import))
|
.route("/admin/api/import", get(list_imports).post(start_import))
|
||||||
@@ -178,6 +178,8 @@ pub fn api_routes(admin_token: String) -> Router<SharedState> {
|
|||||||
.route("/admin/api/cmf", get(cmf_status))
|
.route("/admin/api/cmf", get(cmf_status))
|
||||||
.route("/admin/api/cmf/install", post(cmf_install))
|
.route("/admin/api/cmf/install", post(cmf_install))
|
||||||
.route("/admin/api/cmf/port", get(cmf_port_check))
|
.route("/admin/api/cmf/port", get(cmf_port_check))
|
||||||
|
.route("/admin/api/cmf/files", get(cmf_files))
|
||||||
|
.route("/admin/api/provider/models", post(provider_models))
|
||||||
.route("/admin/api/requests", get(get_requests))
|
.route("/admin/api/requests", get(get_requests))
|
||||||
.route("/admin/api/test", post(run_test))
|
.route("/admin/api/test", post(run_test))
|
||||||
.route("/admin/api/test/stream", post(run_test_stream))
|
.route("/admin/api/test/stream", post(run_test_stream))
|
||||||
@@ -390,18 +392,105 @@ async fn router_probe(State(state): State<SharedState>) -> ApiResult {
|
|||||||
ok(json!({ "ok": true, "status": "ok", "latency_ms": latency_ms, "usage": usage }))
|
ok(json!({ "ok": true, "status": "ok", "latency_ms": latency_ms, "usage": usage }))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct ProviderModelsBody {
|
||||||
|
provider: String,
|
||||||
|
base_url: String,
|
||||||
|
/// Key typed into the form (takes priority)…
|
||||||
|
#[serde(default)]
|
||||||
|
api_key: Option<String>,
|
||||||
|
/// …or the name of a stored secret to reuse if no key was typed.
|
||||||
|
#[serde(default)]
|
||||||
|
api_key_env: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Concise error line from a provider's JSON error body.
|
||||||
|
fn provider_err(body: &str, status: u16) -> String {
|
||||||
|
if let Ok(v) = serde_json::from_str::<Value>(body) {
|
||||||
|
if let Some(m) = v["error"]["message"]
|
||||||
|
.as_str()
|
||||||
|
.or_else(|| v["error"].as_str())
|
||||||
|
.or_else(|| v["message"].as_str())
|
||||||
|
{
|
||||||
|
return format!("{status}: {m}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
format!("HTTP {status}")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Validate a provider API key and list its models by calling the provider's
|
||||||
|
/// `/models` endpoint — powers the "Test key & load models" button. The key is
|
||||||
|
/// used transiently (from the form) or resolved from a stored secret; it is not
|
||||||
|
/// persisted here.
|
||||||
|
async fn provider_models(
|
||||||
|
State(state): State<SharedState>,
|
||||||
|
Json(b): Json<ProviderModelsBody>,
|
||||||
|
) -> ApiResult {
|
||||||
|
let base = b.base_url.trim_end_matches('/').to_string();
|
||||||
|
if base.is_empty() {
|
||||||
|
return Err(ApiError::bad("base URL is required"));
|
||||||
|
}
|
||||||
|
let key = b.api_key.filter(|k| !k.trim().is_empty()).or_else(|| {
|
||||||
|
b.api_key_env
|
||||||
|
.as_deref()
|
||||||
|
.and_then(|e| state.secrets.resolve(e))
|
||||||
|
});
|
||||||
|
let key = key.as_deref();
|
||||||
|
|
||||||
|
let client = reqwest::Client::builder()
|
||||||
|
.timeout(std::time::Duration::from_secs(15))
|
||||||
|
.build()
|
||||||
|
.map_err(|e| ApiError::bad(e.to_string()))?;
|
||||||
|
|
||||||
|
// Anthropic uses x-api-key + anthropic-version and /v1/models; every other
|
||||||
|
// (OpenAI-compatible) provider uses a Bearer token and /models.
|
||||||
|
let resp = if b.provider == "anthropic" {
|
||||||
|
let mut r = client
|
||||||
|
.get(format!("{base}/v1/models"))
|
||||||
|
.header("anthropic-version", "2023-06-01");
|
||||||
|
if let Some(k) = key {
|
||||||
|
r = r.header("x-api-key", k);
|
||||||
|
}
|
||||||
|
r.send().await
|
||||||
|
} else {
|
||||||
|
let mut r = client.get(format!("{base}/models"));
|
||||||
|
if let Some(k) = key {
|
||||||
|
r = r.header("authorization", format!("Bearer {k}"));
|
||||||
|
}
|
||||||
|
r.send().await
|
||||||
|
};
|
||||||
|
let resp = resp.map_err(|e| ApiError::bad(format!("request failed: {e}")))?;
|
||||||
|
let status = resp.status().as_u16();
|
||||||
|
let body = resp.text().await.unwrap_or_default();
|
||||||
|
if !(200..300).contains(&status) {
|
||||||
|
return ok(json!({ "ok": false, "status": status, "error": provider_err(&body, status) }));
|
||||||
|
}
|
||||||
|
// OpenAI + Anthropic both return { "data": [ { "id": ... } ] }.
|
||||||
|
let v: Value = serde_json::from_str(&body).unwrap_or(json!({}));
|
||||||
|
let mut models: Vec<String> = v["data"]
|
||||||
|
.as_array()
|
||||||
|
.map(|arr| {
|
||||||
|
arr.iter()
|
||||||
|
.filter_map(|m| m["id"].as_str().map(|s| s.to_string()))
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
models.sort();
|
||||||
|
ok(json!({ "ok": true, "count": models.len(), "models": models }))
|
||||||
|
}
|
||||||
|
|
||||||
async fn meta() -> ApiResult {
|
async fn meta() -> ApiResult {
|
||||||
ok(json!({
|
ok(json!({
|
||||||
"version": env!("CARGO_PKG_VERSION"),
|
"version": env!("CARGO_PKG_VERSION"),
|
||||||
"providers": ["openai", "anthropic", "ollama", "http"],
|
"providers": ["openai", "anthropic", "openrouter", "lmstudio", "ollama", "http"],
|
||||||
"provider_impl": { "openai": true, "ollama": true, "http": true, "anthropic": true },
|
"provider_impl": { "openai": true, "anthropic": true, "openrouter": true, "lmstudio": true, "ollama": true, "http": true },
|
||||||
"cost_tiers": ["cheap", "mid", "expensive"],
|
"cost_tiers": ["cheap", "mid", "expensive", "local"],
|
||||||
"kinds": ["chat", "embedding"],
|
"kinds": ["chat", "embedding"],
|
||||||
"profiles": ["cost-saver", "balanced", "quality-first"],
|
"profiles": ["cost-saver", "balanced", "quality-first"],
|
||||||
"text_strategies": ["last_user", "last_user_plus_system", "concat_all"],
|
"text_strategies": ["last_user", "last_user_plus_system", "concat_all"],
|
||||||
"policy_modes": ["fixed_table", "cost_aware"],
|
"policy_modes": ["fixed_table", "cost_aware"],
|
||||||
"tiers": ["low", "medium", "high"],
|
"tiers": ["low", "medium", "high"],
|
||||||
"caps": ["tools", "vision"],
|
"caps": ["tools", "vision", "audio", "reasoning", "json", "caching"],
|
||||||
"protocols_impl": {
|
"protocols_impl": {
|
||||||
"openai_chat": true, "openai_completions": true, "openai_embeddings": true,
|
"openai_chat": true, "openai_completions": true, "openai_embeddings": true,
|
||||||
"openai_models": true, "anthropic_messages": true, "mcp": true, "native_passthrough": true
|
"openai_models": true, "anthropic_messages": true, "mcp": true, "native_passthrough": true
|
||||||
@@ -517,7 +606,10 @@ fn cmf_file_for_model(
|
|||||||
cmf: &crate::config::CmfCfg,
|
cmf: &crate::config::CmfCfg,
|
||||||
m: &crate::config::ModelCfg,
|
m: &crate::config::ModelCfg,
|
||||||
) -> Option<std::path::PathBuf> {
|
) -> Option<std::path::PathBuf> {
|
||||||
|
// Any loopback URL counts as local — managed servers may sit on any port.
|
||||||
let local = m.base_url == cmf.cortiq_server_url
|
let local = m.base_url == cmf.cortiq_server_url
|
||||||
|
|| m.base_url.contains("//127.0.0.1:")
|
||||||
|
|| m.base_url.contains("//localhost:")
|
||||||
|| m.base_url
|
|| m.base_url
|
||||||
.contains(&format!("{}:{}", cmf.local_host, cmf.local_port));
|
.contains(&format!("{}:{}", cmf.local_host, cmf.local_port));
|
||||||
let stem = m.model.trim();
|
let stem = m.model.trim();
|
||||||
@@ -557,12 +649,24 @@ fn cmf_path_referenced(cfg: &crate::config::Config, path: &std::path::Path) -> b
|
|||||||
.any(|m| cmf_file_for_model(&cfg.cmf, m).as_deref() == Some(path))
|
.any(|m| cmf_file_for_model(&cfg.cmf, m).as_deref() == Some(path))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn delete_model(State(state): State<SharedState>, Path(id): Path<String>) -> ApiResult {
|
#[derive(Deserialize)]
|
||||||
|
struct DeleteModelQuery {
|
||||||
|
/// Also remove the local `.cmf` from disk. Defaults to true (the historical
|
||||||
|
/// behaviour); the UI asks the user and passes it explicitly.
|
||||||
|
delete_file: Option<bool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_model(
|
||||||
|
State(state): State<SharedState>,
|
||||||
|
Path(id): Path<String>,
|
||||||
|
Query(q): Query<DeleteModelQuery>,
|
||||||
|
) -> ApiResult {
|
||||||
let mut cfg = current_cfg(&state);
|
let mut cfg = current_cfg(&state);
|
||||||
|
let delete_file = q.delete_file.unwrap_or(true);
|
||||||
|
|
||||||
// Managed local models live in [cmf], not [[models]]. Deleting one stops its
|
// Managed local models live in [cmf], not [[models]]. Deleting one stops its
|
||||||
// server, removes it from the managed config, and deletes its file — unless
|
// server, removes it from the managed config, and (when asked) deletes its
|
||||||
// another model still points at the same file.
|
// file — unless another model still points at the same file.
|
||||||
let managed = cfg.cmf.effective_servers().into_iter().find(|s| s.id == id);
|
let managed = cfg.cmf.effective_servers().into_iter().find(|s| s.id == id);
|
||||||
if let Some(server) = managed {
|
if let Some(server) = managed {
|
||||||
if !cfg.models.iter().any(|x| x.id == id) {
|
if !cfg.models.iter().any(|x| x.id == id) {
|
||||||
@@ -577,10 +681,10 @@ async fn delete_model(State(state): State<SharedState>, Path(id): Path<String>)
|
|||||||
let shared = cmf_path_referenced(&cfg, &path);
|
let shared = cmf_path_referenced(&cfg, &path);
|
||||||
state.cmf.stop_one(&id).await;
|
state.cmf.stop_one(&id).await;
|
||||||
state.reload(cfg)?;
|
state.reload(cfg)?;
|
||||||
let file_removed = if shared {
|
let file_removed = if delete_file && !shared {
|
||||||
false
|
|
||||||
} else {
|
|
||||||
remove_cmf_path(&path)
|
remove_cmf_path(&path)
|
||||||
|
} else {
|
||||||
|
false
|
||||||
};
|
};
|
||||||
return ok(json!({ "ok": true, "file_removed": file_removed }));
|
return ok(json!({ "ok": true, "file_removed": file_removed }));
|
||||||
}
|
}
|
||||||
@@ -596,8 +700,8 @@ async fn delete_model(State(state): State<SharedState>, Path(id): Path<String>)
|
|||||||
let file = victim
|
let file = victim
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|m| cmf_file_for_model(&cfg.cmf, m));
|
.and_then(|m| cmf_file_for_model(&cfg.cmf, m));
|
||||||
// Only remove the file if no remaining model still references it.
|
// Only remove the file if asked to AND no remaining model still references it.
|
||||||
let file = file.filter(|p| !cmf_path_referenced(&cfg, p));
|
let file = file.filter(|p| delete_file && !cmf_path_referenced(&cfg, p));
|
||||||
state.reload(cfg)?;
|
state.reload(cfg)?;
|
||||||
let file_removed = file.map(|p| remove_cmf_path(&p)).unwrap_or(false);
|
let file_removed = file.map(|p| remove_cmf_path(&p)).unwrap_or(false);
|
||||||
ok(json!({ "ok": true, "file_removed": file_removed }))
|
ok(json!({ "ok": true, "file_removed": file_removed }))
|
||||||
@@ -605,15 +709,25 @@ async fn delete_model(State(state): State<SharedState>, Path(id): Path<String>)
|
|||||||
|
|
||||||
async fn probe_model(State(state): State<SharedState>, Path(id): Path<String>) -> ApiResult {
|
async fn probe_model(State(state): State<SharedState>, Path(id): Path<String>) -> ApiResult {
|
||||||
let live = state.live();
|
let live = state.live();
|
||||||
let mcfg = live
|
// Provider string: from the static pool, or "openai" for a managed local
|
||||||
|
// model (which is synthesized into the registry, not [[models]]).
|
||||||
|
let provider_str = live
|
||||||
.cfg
|
.cfg
|
||||||
.models
|
.models
|
||||||
.iter()
|
.iter()
|
||||||
.find(|m| m.id == id)
|
.find(|m| m.id == id)
|
||||||
.ok_or_else(|| ApiError::not_found(format!("model '{id}' not found")))?
|
.map(|m| m.provider.clone())
|
||||||
.clone();
|
.or_else(|| {
|
||||||
|
live.cfg
|
||||||
|
.cmf
|
||||||
|
.effective_servers()
|
||||||
|
.iter()
|
||||||
|
.find(|s| s.id == id)
|
||||||
|
.map(|_| "openai".to_string())
|
||||||
|
})
|
||||||
|
.ok_or_else(|| ApiError::not_found(format!("model '{id}' not found")))?;
|
||||||
|
|
||||||
if mcfg.provider == "anthropic" {
|
if provider_str == "anthropic" {
|
||||||
return ok(json!({
|
return ok(json!({
|
||||||
"ok": false,
|
"ok": false,
|
||||||
"error": "anthropic provider not yet implemented (planned v0.2)",
|
"error": "anthropic provider not yet implemented (planned v0.2)",
|
||||||
@@ -650,7 +764,21 @@ async fn probe_model(State(state): State<SharedState>, Path(id): Path<String>) -
|
|||||||
let latency_ms = started.elapsed().as_millis() as u64;
|
let latency_ms = started.elapsed().as_millis() as u64;
|
||||||
match res {
|
match res {
|
||||||
Ok(_) => ok(json!({ "ok": true, "latency_ms": latency_ms })),
|
Ok(_) => ok(json!({ "ok": true, "latency_ms": latency_ms })),
|
||||||
Err(e) => ok(json!({ "ok": false, "latency_ms": latency_ms, "error": e.to_string() })),
|
Err(e) => {
|
||||||
|
let msg = e.to_string();
|
||||||
|
// Make auth failures actionable — a raw "User not found" (OpenRouter's
|
||||||
|
// 401) usually means the API key is wrong or the wrong type.
|
||||||
|
let hint = if msg.contains("401")
|
||||||
|
|| msg.contains("Unauthorized")
|
||||||
|
|| msg.contains("User not found")
|
||||||
|
|| msg.contains("No auth")
|
||||||
|
{
|
||||||
|
" — check the API key (for OpenRouter use an inference key, not a provisioning/management key)"
|
||||||
|
} else {
|
||||||
|
""
|
||||||
|
};
|
||||||
|
ok(json!({ "ok": false, "latency_ms": latency_ms, "error": format!("{msg}{hint}") }))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -780,22 +908,101 @@ async fn put_settings(State(state): State<SharedState>, Json(b): Json<SettingsBo
|
|||||||
needs_restart = true;
|
needs_restart = true;
|
||||||
cfg.cache = v;
|
cfg.cache = v;
|
||||||
}
|
}
|
||||||
|
let mut cmf_apply: Option<(Vec<crate::config::CmfServer>, CmfCfg)> = None;
|
||||||
if let Some(v) = b.cmf {
|
if let Some(v) = b.cmf {
|
||||||
// manage_server / local_model / threads / gpu apply on restart (the server
|
let old_servers = cfg.cmf.effective_servers();
|
||||||
// is spawned once at startup with those env vars); local_only / router
|
|
||||||
// routing apply immediately.
|
|
||||||
if v.manage_server != cfg.cmf.manage_server
|
|
||||||
|| v.local_model != cfg.cmf.local_model
|
|
||||||
|| v.threads != cfg.cmf.threads
|
|
||||||
|| v.gpu != cfg.cmf.gpu
|
|
||||||
|| v.servers != cfg.cmf.servers
|
|
||||||
{
|
|
||||||
needs_restart = true;
|
|
||||||
}
|
|
||||||
cfg.cmf = v;
|
cfg.cmf = v;
|
||||||
|
// A managed row supersedes a same-id STATIC pool entry that points at a
|
||||||
|
// local server (the shape Import→register creates): drop the static one
|
||||||
|
// instead of failing validation with "duplicate managed model id".
|
||||||
|
let managed_ids: std::collections::HashSet<String> = cfg
|
||||||
|
.cmf
|
||||||
|
.effective_servers()
|
||||||
|
.into_iter()
|
||||||
|
.map(|s| s.id)
|
||||||
|
.collect();
|
||||||
|
cfg.models.retain(|m| {
|
||||||
|
!(managed_ids.contains(&m.id)
|
||||||
|
&& (m.base_url.contains("//127.0.0.1:") || m.base_url.contains("//localhost:")))
|
||||||
|
});
|
||||||
|
// Replacing or renaming a managed model must not brick the save: routing
|
||||||
|
// may still reference the old id. Map a vanished id to its replacement
|
||||||
|
// (the new server on the same port), strip ids that are truly gone, and
|
||||||
|
// repair the default so validate() passes.
|
||||||
|
heal_routing_after_cmf_change(&mut cfg, &old_servers);
|
||||||
|
// Server-list changes apply LIVE after a successful reload: removed /
|
||||||
|
// redefined servers are stopped (RAM freed), new ones are spawned.
|
||||||
|
if cfg.cmf.effective_servers() != old_servers {
|
||||||
|
cmf_apply = Some((old_servers, cfg.cmf.clone()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
state.reload(cfg)?;
|
state.reload(cfg)?;
|
||||||
ok(json!({ "ok": true, "needs_restart": needs_restart }))
|
let applied_live = cmf_apply.is_some();
|
||||||
|
if let Some((old, new_cfg)) = cmf_apply {
|
||||||
|
tokio::spawn(crate::cmf_runtime::apply_server_changes(
|
||||||
|
state.cmf.clone(),
|
||||||
|
old,
|
||||||
|
new_cfg,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
ok(json!({ "ok": true, "needs_restart": needs_restart, "applied_live": applied_live }))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// After the managed-servers list changed, rewrite [routing] so it only
|
||||||
|
/// references ids that still exist. A removed server whose port is now held by
|
||||||
|
/// a NEW server is treated as a replacement (old id → new id); other vanished
|
||||||
|
/// ids are stripped. An emptied default falls back to the first managed server,
|
||||||
|
/// then the first pool model.
|
||||||
|
fn heal_routing_after_cmf_change(cfg: &mut Config, old_servers: &[crate::config::CmfServer]) {
|
||||||
|
let servers = cfg.cmf.effective_servers();
|
||||||
|
let known: std::collections::HashSet<String> = cfg
|
||||||
|
.models
|
||||||
|
.iter()
|
||||||
|
.map(|m| m.id.clone())
|
||||||
|
.chain(servers.iter().map(|s| s.id.clone()))
|
||||||
|
.collect();
|
||||||
|
let new_by_port: HashMap<u16, String> =
|
||||||
|
servers.iter().map(|s| (s.port, s.id.clone())).collect();
|
||||||
|
let mut rename: HashMap<String, String> = HashMap::new();
|
||||||
|
for old in old_servers {
|
||||||
|
if !known.contains(&old.id) {
|
||||||
|
if let Some(new_id) = new_by_port.get(&old.port) {
|
||||||
|
rename.insert(old.id.clone(), new_id.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let fix = |id: &str| -> Option<String> {
|
||||||
|
if known.contains(id) {
|
||||||
|
Some(id.to_string())
|
||||||
|
} else {
|
||||||
|
rename.get(id).cloned()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if !cfg.routing.default.is_empty() {
|
||||||
|
cfg.routing.default = fix(&cfg.routing.default).unwrap_or_default();
|
||||||
|
}
|
||||||
|
if cfg.routing.default.is_empty() {
|
||||||
|
cfg.routing.default = servers
|
||||||
|
.first()
|
||||||
|
.map(|s| s.id.clone())
|
||||||
|
.or_else(|| cfg.models.first().map(|m| m.id.clone()))
|
||||||
|
.unwrap_or_default();
|
||||||
|
}
|
||||||
|
for tt in cfg.routing.tiers.values_mut() {
|
||||||
|
let list = match tt {
|
||||||
|
TierTargets::List(v) => v.clone(),
|
||||||
|
TierTargets::One(s) => vec![s.clone()],
|
||||||
|
};
|
||||||
|
let mut healed: Vec<String> = Vec::new();
|
||||||
|
for id in &list {
|
||||||
|
if let Some(new_id) = fix(id) {
|
||||||
|
if !healed.contains(&new_id) {
|
||||||
|
healed.push(new_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
*tt = TierTargets::List(healed);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ───────────────────────────── cmf runtime ─────────────────────────────────
|
// ───────────────────────────── cmf runtime ─────────────────────────────────
|
||||||
@@ -818,6 +1025,7 @@ async fn cmf_status(State(state): State<SharedState>) -> ApiResult {
|
|||||||
"port": s.port,
|
"port": s.port,
|
||||||
"threads": s.threads,
|
"threads": s.threads,
|
||||||
"gpu": s.gpu,
|
"gpu": s.gpu,
|
||||||
|
"o1": s.o1,
|
||||||
"running": live.map(|l| l.running).unwrap_or(false),
|
"running": live.map(|l| l.running).unwrap_or(false),
|
||||||
"healthy": live.map(|l| l.healthy).unwrap_or(false),
|
"healthy": live.map(|l| l.healthy).unwrap_or(false),
|
||||||
"last_error": live.and_then(|l| l.last_error.clone()),
|
"last_error": live.and_then(|l| l.last_error.clone()),
|
||||||
@@ -843,6 +1051,36 @@ async fn cmf_install(State(state): State<SharedState>) -> ApiResult {
|
|||||||
ok(json!({ "started": true }))
|
ok(json!({ "started": true }))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// List the `.cmf` model files available under `models_dir` so the UI can offer
|
||||||
|
/// them as a dropdown instead of a hand-typed path. Includes tensor-dir `.cmf`.
|
||||||
|
async fn cmf_files(State(state): State<SharedState>) -> ApiResult {
|
||||||
|
let dir = state.live().cfg.cmf.models_dir.clone();
|
||||||
|
let mut entries: Vec<(String, String, Option<u64>, bool)> = Vec::new();
|
||||||
|
if let Ok(rd) = std::fs::read_dir(&dir) {
|
||||||
|
for e in rd.flatten() {
|
||||||
|
let name = e.file_name().to_string_lossy().to_string();
|
||||||
|
if !name.ends_with(".cmf") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let p = e.path();
|
||||||
|
let is_dir = p.is_dir();
|
||||||
|
let size = if is_dir {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
std::fs::metadata(&p).ok().map(|m| m.len())
|
||||||
|
};
|
||||||
|
let path = format!("{}/{}", dir.trim_end_matches('/'), name);
|
||||||
|
entries.push((name, path, size, is_dir));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entries.sort_by(|a, b| a.0.cmp(&b.0));
|
||||||
|
let files: Vec<Value> = entries
|
||||||
|
.into_iter()
|
||||||
|
.map(|(name, path, size, is_dir)| json!({ "name": name, "path": path, "size": size, "is_dir": is_dir }))
|
||||||
|
.collect();
|
||||||
|
ok(json!({ "dir": dir, "files": files }))
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
struct PortQuery {
|
struct PortQuery {
|
||||||
port: u16,
|
port: u16,
|
||||||
@@ -885,6 +1123,9 @@ async fn cmf_port_check(State(state): State<SharedState>, Query(q): Query<PortQu
|
|||||||
ok(json!({
|
ok(json!({
|
||||||
"port": port,
|
"port": port,
|
||||||
"available": available,
|
"available": available,
|
||||||
|
// true = held by this gateway's own managed server (a replaced/edited
|
||||||
|
// row frees it on restart) — the UI shows this as info, not an error
|
||||||
|
"managed": running_here,
|
||||||
"detail": detail,
|
"detail": detail,
|
||||||
"suggested": suggested,
|
"suggested": suggested,
|
||||||
}))
|
}))
|
||||||
@@ -1054,6 +1295,13 @@ async fn get_stats(State(state): State<SharedState>, Query(q): Query<StatsQuery>
|
|||||||
ok(snap)
|
ok(snap)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Clear all recorded stats/logs (in-memory aggregates, series, recent requests)
|
||||||
|
/// and truncate the JSONL log — the dashboard "Clear logs" button.
|
||||||
|
async fn clear_stats(State(state): State<SharedState>) -> ApiResult {
|
||||||
|
state.stats.clear();
|
||||||
|
ok(json!({ "ok": true }))
|
||||||
|
}
|
||||||
|
|
||||||
/// Self-warming shadow loop: per-task-type promotion state, Wilson-LB of the
|
/// Self-warming shadow loop: per-task-type promotion state, Wilson-LB of the
|
||||||
/// judged pass-rate, and whether the local model is serving it yet.
|
/// judged pass-rate, and whether the local model is serving it yet.
|
||||||
async fn get_shadow(State(state): State<SharedState>) -> ApiResult {
|
async fn get_shadow(State(state): State<SharedState>) -> ApiResult {
|
||||||
|
|||||||
+87
-1
@@ -173,6 +173,31 @@ impl CmfRuntime {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Ensure a status slot exists for `server` (used when servers are added at
|
||||||
|
/// runtime); refreshes model/port if the slot already exists.
|
||||||
|
fn add_server_slot(&self, server: &CmfServer, host: &str) {
|
||||||
|
let mut s = self.status.lock().unwrap();
|
||||||
|
s.enabled = true;
|
||||||
|
if let Some(sv) = s.servers.iter_mut().find(|x| x.id == server.id) {
|
||||||
|
sv.model = server.model.clone();
|
||||||
|
sv.port = server.port;
|
||||||
|
sv.base_url = format!("http://{host}:{}/v1", server.port);
|
||||||
|
sv.running = false;
|
||||||
|
sv.healthy = false;
|
||||||
|
sv.last_error = None;
|
||||||
|
} else {
|
||||||
|
s.servers.push(CmfServerStatus {
|
||||||
|
id: server.id.clone(),
|
||||||
|
model: server.model.clone(),
|
||||||
|
port: server.port,
|
||||||
|
base_url: format!("http://{host}:{}/v1", server.port),
|
||||||
|
running: false,
|
||||||
|
healthy: false,
|
||||||
|
last_error: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Stop and forget a single managed server (used when a model is deleted).
|
/// Stop and forget a single managed server (used when a model is deleted).
|
||||||
pub async fn stop_one(&self, id: &str) {
|
pub async fn stop_one(&self, id: &str) {
|
||||||
let removed = {
|
let removed = {
|
||||||
@@ -238,7 +263,7 @@ fn parse_ver(s: &str) -> (u32, u32, u32) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn version_lt(a: &str, b: &str) -> bool {
|
pub(crate) fn version_lt(a: &str, b: &str) -> bool {
|
||||||
parse_ver(a) < parse_ver(b)
|
parse_ver(a) < parse_ver(b)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -295,6 +320,11 @@ fn spawn_serve(server: &CmfServer, cfg: &CmfCfg) -> Result<Child, String> {
|
|||||||
if server.gpu {
|
if server.gpu {
|
||||||
cmd.env("CMF_GPU", "1");
|
cmd.env("CMF_GPU", "1");
|
||||||
}
|
}
|
||||||
|
// Serve-time O(1) attention override: "off" forces exact attention (undoes a
|
||||||
|
// too-aggressive --o1 conversion hint without re-converting the file).
|
||||||
|
if !server.o1.trim().is_empty() {
|
||||||
|
cmd.env("CMF_O1", server.o1.trim());
|
||||||
|
}
|
||||||
cmd.stdout(Stdio::null())
|
cmd.stdout(Stdio::null())
|
||||||
.stderr(Stdio::null())
|
.stderr(Stdio::null())
|
||||||
.kill_on_drop(true)
|
.kill_on_drop(true)
|
||||||
@@ -322,6 +352,62 @@ async fn wait_healthy(host: &str, port: u16, tries: u32) -> bool {
|
|||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Apply a changed managed-servers list LIVE (no gateway restart): stop servers
|
||||||
|
/// that were removed or redefined — freeing their memory immediately — and
|
||||||
|
/// spawn servers that are new or redefined.
|
||||||
|
pub async fn apply_server_changes(rt: Arc<CmfRuntime>, old: Vec<CmfServer>, cfg: CmfCfg) {
|
||||||
|
let new = cfg.effective_servers();
|
||||||
|
// 1. Stop what's gone or changed (kills the child → RAM released).
|
||||||
|
for o in &old {
|
||||||
|
let unchanged = new.iter().any(|n| n == o);
|
||||||
|
if !unchanged {
|
||||||
|
rt.stop_one(&o.id).await;
|
||||||
|
rt.log(format!("stopped '{}' (removed or changed)", o.id));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if new.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// 2. Spawn what's new or changed.
|
||||||
|
if installed_version(&cfg.cortiq_bin).is_none() {
|
||||||
|
rt.fail("cortiq binary not found — cannot start local model servers");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for n in &new {
|
||||||
|
let unchanged = old.iter().any(|o| o == n);
|
||||||
|
if unchanged {
|
||||||
|
continue; // already running with the same definition
|
||||||
|
}
|
||||||
|
rt.add_server_slot(n, &cfg.local_host);
|
||||||
|
if !std::path::Path::new(&n.model).exists() {
|
||||||
|
rt.fail_server(&n.id, format!("model file not found: {}", n.model));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
rt.log(format!(
|
||||||
|
"starting: cortiq serve {} --host {} --port {}",
|
||||||
|
n.model, cfg.local_host, n.port
|
||||||
|
));
|
||||||
|
match spawn_serve(n, &cfg) {
|
||||||
|
Ok(child) => rt.add_child(n.id.clone(), child),
|
||||||
|
Err(e) => {
|
||||||
|
rt.fail_server(&n.id, format!("failed to spawn: {e}"));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let (rtc, host, id, port) = (rt.clone(), cfg.local_host.clone(), n.id.clone(), n.port);
|
||||||
|
tokio::spawn(async move {
|
||||||
|
if wait_healthy(&host, port, 240).await {
|
||||||
|
rtc.set_healthy(&id, true);
|
||||||
|
rtc.log(format!(
|
||||||
|
"local CMF server ready — '{id}' at http://{host}:{port}/v1"
|
||||||
|
));
|
||||||
|
} else {
|
||||||
|
rtc.fail_server(&id, "did not become healthy in time");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Orchestrate the whole lifecycle: ensure installed → maybe update → spawn each
|
/// Orchestrate the whole lifecycle: ensure installed → maybe update → spawn each
|
||||||
/// configured local model → wait healthy. Safe to call in a background task; it
|
/// configured local model → wait healthy. Safe to call in a background task; it
|
||||||
/// never panics.
|
/// never panics.
|
||||||
|
|||||||
+36
-1
@@ -83,6 +83,26 @@ pub struct CmfServer {
|
|||||||
/// Offload matvecs to the GPU (CMF_GPU=1).
|
/// Offload matvecs to the GPU (CMF_GPU=1).
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub gpu: bool,
|
pub gpu: bool,
|
||||||
|
/// Serve-time O(1) attention override (exported as CMF_O1). Empty = follow
|
||||||
|
/// the hint recorded in the .cmf; "off" = force exact attention (fixes a
|
||||||
|
/// too-aggressive conversion without re-converting); "all"/"deepN"/"i,j,k"
|
||||||
|
/// = force-enable on those layers.
|
||||||
|
#[serde(default)]
|
||||||
|
pub o1: String,
|
||||||
|
/// Default generation params for this model — applied by the gateway when
|
||||||
|
/// the incoming request does not set them (an explicit request value wins).
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub temperature: Option<f32>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub top_p: Option<f32>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub max_tokens: Option<u32>,
|
||||||
|
/// Reasoning switch for thinking models (Qwen3/3.5 …): "off" injects
|
||||||
|
/// enable_thinking=false into requests that don't set it themselves —
|
||||||
|
/// the model answers directly, skipping the <think> phase. Needs
|
||||||
|
/// cortiq ≥ 0.2.1. Empty = model/template default.
|
||||||
|
#[serde(default)]
|
||||||
|
pub thinking: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// CMF-format model factory: import a HuggingFace model, convert it to a
|
/// CMF-format model factory: import a HuggingFace model, convert it to a
|
||||||
@@ -172,6 +192,11 @@ impl CmfCfg {
|
|||||||
port: self.local_port,
|
port: self.local_port,
|
||||||
threads: self.threads,
|
threads: self.threads,
|
||||||
gpu: self.gpu,
|
gpu: self.gpu,
|
||||||
|
o1: String::new(),
|
||||||
|
temperature: None,
|
||||||
|
top_p: None,
|
||||||
|
max_tokens: None,
|
||||||
|
thinking: String::new(),
|
||||||
}];
|
}];
|
||||||
}
|
}
|
||||||
Vec::new()
|
Vec::new()
|
||||||
@@ -675,9 +700,19 @@ impl Config {
|
|||||||
self.models.iter().map(|m| m.id.as_str()).collect();
|
self.models.iter().map(|m| m.id.as_str()).collect();
|
||||||
// managed local servers are known ids too; also enforce unique ports/ids
|
// managed local servers are known ids too; also enforce unique ports/ids
|
||||||
let mut ports = std::collections::HashSet::new();
|
let mut ports = std::collections::HashSet::new();
|
||||||
|
let mut managed_seen = std::collections::HashSet::new();
|
||||||
for s in &servers {
|
for s in &servers {
|
||||||
|
if !managed_seen.insert(s.id.as_str()) {
|
||||||
|
anyhow::bail!(
|
||||||
|
"config: two managed local models share the id '{}' — give one of the rows a different id",
|
||||||
|
s.id
|
||||||
|
);
|
||||||
|
}
|
||||||
if !ids.insert(s.id.as_str()) {
|
if !ids.insert(s.id.as_str()) {
|
||||||
anyhow::bail!("config: duplicate managed model id '{}'", s.id);
|
anyhow::bail!(
|
||||||
|
"config: managed local model '{}' has the same id as a pool model — rename the row or delete the pool entry",
|
||||||
|
s.id
|
||||||
|
);
|
||||||
}
|
}
|
||||||
if !ports.insert(s.port) {
|
if !ports.insert(s.port) {
|
||||||
anyhow::bail!("config: two managed models share port {}", s.port);
|
anyhow::bail!("config: two managed models share port {}", s.port);
|
||||||
|
|||||||
@@ -71,6 +71,20 @@ pub struct ImportParams {
|
|||||||
pub shard_max_gb: Option<f32>,
|
pub shard_max_gb: Option<f32>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub skip_mtp: bool,
|
pub skip_mtp: bool,
|
||||||
|
/// O(1) Nyström attention hint (native converter, cortiq ≥ 0.2.0):
|
||||||
|
/// `all` | `deepN` | `i,j,k`. Weights pass through unchanged — the runtime
|
||||||
|
/// reads the hint at load. Empty/None/"off" = exact attention.
|
||||||
|
#[serde(default)]
|
||||||
|
pub o1: Option<String>,
|
||||||
|
/// Landmark budget for the o1 hint (validated default 32).
|
||||||
|
#[serde(default)]
|
||||||
|
pub o1_m: Option<usize>,
|
||||||
|
/// Exact-window width for the o1 hint (validated default 128).
|
||||||
|
#[serde(default)]
|
||||||
|
pub o1_window: Option<usize>,
|
||||||
|
/// Permanent exact sink keys for the o1 hint (validated default 4).
|
||||||
|
#[serde(default)]
|
||||||
|
pub o1_sink: Option<usize>,
|
||||||
}
|
}
|
||||||
fn default_quant() -> String {
|
fn default_quant() -> String {
|
||||||
"Q8_2F".into()
|
"Q8_2F".into()
|
||||||
@@ -269,6 +283,34 @@ pub fn start_import(store: Arc<JobStore>, cfg: &CmfCfg, p: ImportParams) -> Resu
|
|||||||
cfg.converter
|
cfg.converter
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
// O(1) attention hint — native-converter feature (cortiq ≥ 0.2.0). Fail fast
|
||||||
|
// with a clear message instead of a mid-job "unexpected argument" error.
|
||||||
|
let o1 =
|
||||||
|
p.o1.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty() && *s != "off")
|
||||||
|
.map(str::to_string);
|
||||||
|
if o1.is_some() {
|
||||||
|
if use_python {
|
||||||
|
return Err(
|
||||||
|
"O(1) attention is supported by the native converter only — remove the \
|
||||||
|
linear-core / v-bit shape / mean-bits options to use it"
|
||||||
|
.into(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if p.repo.to_ascii_lowercase().contains("gguf") {
|
||||||
|
return Err("O(1) attention is not supported for GGUF imports yet".into());
|
||||||
|
}
|
||||||
|
let ver = crate::cmf_runtime::installed_version(&cfg.cortiq_bin);
|
||||||
|
if let Some(v) = &ver {
|
||||||
|
if crate::cmf_runtime::version_lt(v, "0.2.0") {
|
||||||
|
return Err(format!(
|
||||||
|
"O(1) attention needs cortiq ≥ 0.2.0 (installed: {v}) — update the runtime \
|
||||||
|
in Settings → Local models"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
let _ = std::fs::create_dir_all(&cfg.models_dir);
|
let _ = std::fs::create_dir_all(&cfg.models_dir);
|
||||||
let base = if p.name.trim().is_empty() {
|
let base = if p.name.trim().is_empty() {
|
||||||
sanitize(p.repo.rsplit('/').next().unwrap_or(&p.repo))
|
sanitize(p.repo.rsplit('/').next().unwrap_or(&p.repo))
|
||||||
@@ -384,6 +426,23 @@ pub fn start_import(store: Arc<JobStore>, cfg: &CmfCfg, p: ImportParams) -> Resu
|
|||||||
a.push("--hf-token".into());
|
a.push("--hf-token".into());
|
||||||
a.push(t.clone());
|
a.push(t.clone());
|
||||||
}
|
}
|
||||||
|
// O(1) attention hint (weights unchanged; the runtime reads it at load)
|
||||||
|
if let Some(spec) = &o1 {
|
||||||
|
a.push("--o1".into());
|
||||||
|
a.push(spec.clone());
|
||||||
|
if let Some(m) = p.o1_m {
|
||||||
|
a.push("--o1-m".into());
|
||||||
|
a.push(m.to_string());
|
||||||
|
}
|
||||||
|
if let Some(w) = p.o1_window {
|
||||||
|
a.push("--o1-window".into());
|
||||||
|
a.push(w.to_string());
|
||||||
|
}
|
||||||
|
if let Some(s) = p.o1_sink {
|
||||||
|
a.push("--o1-sink".into());
|
||||||
|
a.push(s.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
(
|
(
|
||||||
cfg.cortiq_bin.clone(),
|
cfg.cortiq_bin.clone(),
|
||||||
a,
|
a,
|
||||||
|
|||||||
+35
-2
@@ -301,6 +301,35 @@ impl Pipeline {
|
|||||||
Ok((decision, route_source, final_candidates))
|
Ok((decision, route_source, final_candidates))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Fill in the model's default generation params (from its Local-models row)
|
||||||
|
/// for anything the request left unset — an explicit request value wins.
|
||||||
|
fn apply_gen_defaults(req: &mut ChatRequest, cfg: &crate::config::Config, model_id: &str) {
|
||||||
|
if let Some(s) = cfg
|
||||||
|
.cmf
|
||||||
|
.effective_servers()
|
||||||
|
.into_iter()
|
||||||
|
.find(|s| s.id == model_id)
|
||||||
|
{
|
||||||
|
if req.params.temperature.is_none() {
|
||||||
|
req.params.temperature = s.temperature;
|
||||||
|
}
|
||||||
|
if req.params.top_p.is_none() {
|
||||||
|
req.params.top_p = s.top_p;
|
||||||
|
}
|
||||||
|
if req.params.max_tokens.is_none() {
|
||||||
|
req.params.max_tokens = s.max_tokens;
|
||||||
|
}
|
||||||
|
// reasoning switch: "off" → enable_thinking=false (cortiq ≥ 0.2.1
|
||||||
|
// renders the chat template with an empty <think> block); a client
|
||||||
|
// that sets enable_thinking itself wins.
|
||||||
|
if s.thinking == "off" && !req.params.passthrough.contains_key("enable_thinking") {
|
||||||
|
req.params
|
||||||
|
.passthrough
|
||||||
|
.insert("enable_thinking".into(), serde_json::Value::Bool(false));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn run_inner(&self, req: ChatRequest, state: &SharedState) -> Result<ChatResponse> {
|
async fn run_inner(&self, req: ChatRequest, state: &SharedState) -> Result<ChatResponse> {
|
||||||
let live = state.live();
|
let live = state.live();
|
||||||
let (decision, route_source, mut final_candidates) = self.resolve(&req, &live).await?;
|
let (decision, route_source, mut final_candidates) = self.resolve(&req, &live).await?;
|
||||||
@@ -341,7 +370,9 @@ impl Pipeline {
|
|||||||
route_source
|
route_source
|
||||||
);
|
);
|
||||||
|
|
||||||
match provider.chat(req.clone()).await {
|
let mut attempt_req = req.clone();
|
||||||
|
Self::apply_gen_defaults(&mut attempt_req, &live.cfg, &model_id);
|
||||||
|
match provider.chat(attempt_req).await {
|
||||||
Ok(mut resp) => {
|
Ok(mut resp) => {
|
||||||
let cost =
|
let cost =
|
||||||
provider.price(resp.usage.prompt_tokens, resp.usage.completion_tokens);
|
provider.price(resp.usage.prompt_tokens, resp.usage.completion_tokens);
|
||||||
@@ -415,7 +446,9 @@ impl Pipeline {
|
|||||||
model_id,
|
model_id,
|
||||||
route_source
|
route_source
|
||||||
);
|
);
|
||||||
match provider.chat_stream(req.clone()).await {
|
let mut attempt_req = req.clone();
|
||||||
|
Self::apply_gen_defaults(&mut attempt_req, &live.cfg, &model_id);
|
||||||
|
match provider.chat_stream(attempt_req).await {
|
||||||
Ok(stream) => {
|
Ok(stream) => {
|
||||||
let info = RouteInfo {
|
let info = RouteInfo {
|
||||||
task_label: decision.task_label.clone(),
|
task_label: decision.task_label.clone(),
|
||||||
|
|||||||
@@ -45,6 +45,9 @@ impl OpenAiProvider {
|
|||||||
if let Some(m) = req.params.max_tokens {
|
if let Some(m) = req.params.max_tokens {
|
||||||
body["max_tokens"] = m.into();
|
body["max_tokens"] = m.into();
|
||||||
}
|
}
|
||||||
|
if let Some(p) = req.params.top_p {
|
||||||
|
body["top_p"] = p.into();
|
||||||
|
}
|
||||||
if !req.tools.is_empty() {
|
if !req.tools.is_empty() {
|
||||||
body["tools"] = serde_json::Value::Array(req.tools.clone());
|
body["tools"] = serde_json::Value::Array(req.tools.clone());
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-1
@@ -21,7 +21,10 @@ impl Registry {
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|name| secrets.resolve(name));
|
.and_then(|name| secrets.resolve(name));
|
||||||
let provider: Arc<dyn Provider> = match m.provider.as_str() {
|
let provider: Arc<dyn Provider> = match m.provider.as_str() {
|
||||||
"openai" | "ollama" | "http" => Arc::new(OpenAiProvider::new(m, api_key)?),
|
// openrouter + lmstudio are OpenAI-compatible (Bearer + /chat/completions)
|
||||||
|
"openai" | "ollama" | "http" | "openrouter" | "lmstudio" => {
|
||||||
|
Arc::new(OpenAiProvider::new(m, api_key)?)
|
||||||
|
}
|
||||||
"anthropic" => Arc::new(AnthropicProvider::new(m, api_key)?),
|
"anthropic" => Arc::new(AnthropicProvider::new(m, api_key)?),
|
||||||
other => anyhow::bail!("unknown provider '{}' for model '{}'", other, m.id),
|
other => anyhow::bail!("unknown provider '{}' for model '{}'", other, m.id),
|
||||||
};
|
};
|
||||||
|
|||||||
+93
-41
@@ -83,9 +83,23 @@ impl Totals {
|
|||||||
self.cost_usd += r.cost_usd;
|
self.cost_usd += r.cost_usd;
|
||||||
self.latency_ms_sum += r.latency_ms;
|
self.latency_ms_sum += r.latency_ms;
|
||||||
}
|
}
|
||||||
|
/// Accumulate another Totals into this one (for windowed aggregation).
|
||||||
|
fn merge(&mut self, o: &Totals) {
|
||||||
|
self.requests += o.requests;
|
||||||
|
self.ok += o.ok;
|
||||||
|
self.errors += o.errors;
|
||||||
|
self.failovers += o.failovers;
|
||||||
|
self.prompt_tokens += o.prompt_tokens;
|
||||||
|
self.completion_tokens += o.completion_tokens;
|
||||||
|
self.cost_usd += o.cost_usd;
|
||||||
|
self.latency_ms_sum += o.latency_ms_sum;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Per-minute bucket for time-series charts.
|
/// Per-minute bucket. Beyond the chart fields it carries the per-group breakdown
|
||||||
|
/// and failovers so the admin snapshot can be computed for ANY time window by
|
||||||
|
/// summing the in-window buckets (the internal fields are not serialized into
|
||||||
|
/// the `series` payload).
|
||||||
#[derive(Clone, Debug, Default, Serialize)]
|
#[derive(Clone, Debug, Default, Serialize)]
|
||||||
pub struct Bucket {
|
pub struct Bucket {
|
||||||
pub minute: u64, // unix seconds, rounded down to the minute
|
pub minute: u64, // unix seconds, rounded down to the minute
|
||||||
@@ -95,15 +109,25 @@ pub struct Bucket {
|
|||||||
pub completion_tokens: u64,
|
pub completion_tokens: u64,
|
||||||
pub cost_usd: f64,
|
pub cost_usd: f64,
|
||||||
pub latency_ms_sum: u64,
|
pub latency_ms_sum: u64,
|
||||||
|
#[serde(skip)]
|
||||||
|
pub failovers: u64,
|
||||||
|
#[serde(skip)]
|
||||||
|
pub by_model: HashMap<String, Totals>,
|
||||||
|
#[serde(skip)]
|
||||||
|
pub by_account: HashMap<String, Totals>,
|
||||||
|
#[serde(skip)]
|
||||||
|
pub by_tier: HashMap<String, Totals>,
|
||||||
|
#[serde(skip)]
|
||||||
|
pub by_label: HashMap<String, Totals>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct Inner {
|
struct Inner {
|
||||||
|
/// All-time totals + per-model (used by the Prometheus counters, which must
|
||||||
|
/// be monotonic — the windowed admin snapshot is derived from the buckets).
|
||||||
total: Totals,
|
total: Totals,
|
||||||
by_model: HashMap<String, Totals>,
|
by_model: HashMap<String, Totals>,
|
||||||
by_account: HashMap<String, Totals>,
|
/// Per-minute buckets — the single source for the windowed admin snapshot.
|
||||||
by_tier: HashMap<String, Totals>,
|
|
||||||
by_label: HashMap<String, Totals>,
|
|
||||||
buckets: VecDeque<Bucket>,
|
buckets: VecDeque<Bucket>,
|
||||||
recent: VecDeque<RequestRecord>,
|
recent: VecDeque<RequestRecord>,
|
||||||
}
|
}
|
||||||
@@ -166,31 +190,14 @@ impl Stats {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn apply_aggregates(inner: &mut Inner, rec: &RequestRecord, retention_secs: u64) {
|
fn apply_aggregates(inner: &mut Inner, rec: &RequestRecord, retention_secs: u64) {
|
||||||
|
// all-time counters for Prometheus; the windowed snapshot is derived
|
||||||
|
// from the buckets below.
|
||||||
inner.total.apply(rec);
|
inner.total.apply(rec);
|
||||||
inner
|
inner
|
||||||
.by_model
|
.by_model
|
||||||
.entry(rec.model_id.clone())
|
.entry(rec.model_id.clone())
|
||||||
.or_default()
|
.or_default()
|
||||||
.apply(rec);
|
.apply(rec);
|
||||||
inner
|
|
||||||
.by_account
|
|
||||||
.entry(if rec.account.is_empty() {
|
|
||||||
"anonymous".to_string()
|
|
||||||
} else {
|
|
||||||
rec.account.clone()
|
|
||||||
})
|
|
||||||
.or_default()
|
|
||||||
.apply(rec);
|
|
||||||
inner
|
|
||||||
.by_tier
|
|
||||||
.entry(rec.tier.clone())
|
|
||||||
.or_default()
|
|
||||||
.apply(rec);
|
|
||||||
inner
|
|
||||||
.by_label
|
|
||||||
.entry(rec.task_label.clone())
|
|
||||||
.or_default()
|
|
||||||
.apply(rec);
|
|
||||||
|
|
||||||
let minute = rec.ts - (rec.ts % 60);
|
let minute = rec.ts - (rec.ts % 60);
|
||||||
match inner.buckets.back_mut() {
|
match inner.buckets.back_mut() {
|
||||||
@@ -250,13 +257,31 @@ impl Stats {
|
|||||||
.filter(|b| b.minute >= cutoff)
|
.filter(|b| b.minute >= cutoff)
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let breakdown_src = match groupby {
|
// Windowed totals + breakdown = sum of the in-window buckets, so the KPIs
|
||||||
"account" => &inner.by_account,
|
// and the breakdown recompute whenever the range changes.
|
||||||
"tier" => &inner.by_tier,
|
let mut win = Totals::default();
|
||||||
"label" => &inner.by_label,
|
let mut group: HashMap<String, Totals> = HashMap::new();
|
||||||
_ => &inner.by_model,
|
for b in &series {
|
||||||
};
|
win.requests += b.requests;
|
||||||
let mut breakdown: Vec<serde_json::Value> = breakdown_src
|
win.errors += b.errors;
|
||||||
|
win.ok += b.requests.saturating_sub(b.errors);
|
||||||
|
win.failovers += b.failovers;
|
||||||
|
win.prompt_tokens += b.prompt_tokens;
|
||||||
|
win.completion_tokens += b.completion_tokens;
|
||||||
|
win.cost_usd += b.cost_usd;
|
||||||
|
win.latency_ms_sum += b.latency_ms_sum;
|
||||||
|
let src = match groupby {
|
||||||
|
"account" => &b.by_account,
|
||||||
|
"tier" => &b.by_tier,
|
||||||
|
"label" => &b.by_label,
|
||||||
|
_ => &b.by_model,
|
||||||
|
};
|
||||||
|
for (k, t) in src {
|
||||||
|
group.entry(k.clone()).or_default().merge(t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut breakdown: Vec<serde_json::Value> = group
|
||||||
.iter()
|
.iter()
|
||||||
.map(|(k, t)| {
|
.map(|(k, t)| {
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
@@ -280,17 +305,17 @@ impl Stats {
|
|||||||
|
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
"totals": {
|
"totals": {
|
||||||
"requests": inner.total.requests,
|
"requests": win.requests,
|
||||||
"ok": inner.total.ok,
|
"ok": win.ok,
|
||||||
"errors": inner.total.errors,
|
"errors": win.errors,
|
||||||
"failovers": inner.total.failovers,
|
"failovers": win.failovers,
|
||||||
"prompt_tokens": inner.total.prompt_tokens,
|
"prompt_tokens": win.prompt_tokens,
|
||||||
"completion_tokens": inner.total.completion_tokens,
|
"completion_tokens": win.completion_tokens,
|
||||||
"total_tokens": inner.total.prompt_tokens + inner.total.completion_tokens,
|
"total_tokens": win.prompt_tokens + win.completion_tokens,
|
||||||
"cost_usd": inner.total.cost_usd,
|
"cost_usd": win.cost_usd,
|
||||||
"avg_latency_ms": avg(inner.total.latency_ms_sum, inner.total.requests),
|
"avg_latency_ms": avg(win.latency_ms_sum, win.requests),
|
||||||
"success_rate": if inner.total.requests > 0 {
|
"success_rate": if win.requests > 0 {
|
||||||
inner.total.ok as f64 / inner.total.requests as f64
|
win.ok as f64 / win.requests as f64
|
||||||
} else { 0.0 },
|
} else { 0.0 },
|
||||||
},
|
},
|
||||||
"groupby": groupby,
|
"groupby": groupby,
|
||||||
@@ -299,6 +324,18 @@ impl Stats {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reset all in-memory stats and truncate the JSONL log — the "clear logs"
|
||||||
|
/// admin action.
|
||||||
|
pub fn clear(&self) {
|
||||||
|
{
|
||||||
|
let mut inner = self.inner.lock().unwrap();
|
||||||
|
*inner = Inner::default();
|
||||||
|
}
|
||||||
|
if let Some(path) = &self.file {
|
||||||
|
let _ = std::fs::write(path, "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Recent requests (newest first), with pagination.
|
/// Recent requests (newest first), with pagination.
|
||||||
pub fn recent(&self, limit: usize, offset: usize) -> Vec<RequestRecord> {
|
pub fn recent(&self, limit: usize, offset: usize) -> Vec<RequestRecord> {
|
||||||
let inner = self.inner.lock().unwrap();
|
let inner = self.inner.lock().unwrap();
|
||||||
@@ -358,10 +395,25 @@ fn fill_bucket(b: &mut Bucket, r: &RequestRecord) {
|
|||||||
if r.outcome != "ok" {
|
if r.outcome != "ok" {
|
||||||
b.errors += 1;
|
b.errors += 1;
|
||||||
}
|
}
|
||||||
|
if r.failover {
|
||||||
|
b.failovers += 1;
|
||||||
|
}
|
||||||
b.prompt_tokens += r.prompt_tokens as u64;
|
b.prompt_tokens += r.prompt_tokens as u64;
|
||||||
b.completion_tokens += r.completion_tokens as u64;
|
b.completion_tokens += r.completion_tokens as u64;
|
||||||
b.cost_usd += r.cost_usd;
|
b.cost_usd += r.cost_usd;
|
||||||
b.latency_ms_sum += r.latency_ms;
|
b.latency_ms_sum += r.latency_ms;
|
||||||
|
// per-group breakdown, so the snapshot can be windowed by summing buckets
|
||||||
|
b.by_model.entry(r.model_id.clone()).or_default().apply(r);
|
||||||
|
b.by_account
|
||||||
|
.entry(if r.account.is_empty() {
|
||||||
|
"anonymous".to_string()
|
||||||
|
} else {
|
||||||
|
r.account.clone()
|
||||||
|
})
|
||||||
|
.or_default()
|
||||||
|
.apply(r);
|
||||||
|
b.by_tier.entry(r.tier.clone()).or_default().apply(r);
|
||||||
|
b.by_label.entry(r.task_label.clone()).or_default().apply(r);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn avg(sum: u64, n: u64) -> u64 {
|
fn avg(sum: u64, n: u64) -> u64 {
|
||||||
|
|||||||
Vendored
+5
-1
@@ -60,7 +60,8 @@ export const api = {
|
|||||||
listModels: () => j("GET", "/models"),
|
listModels: () => j("GET", "/models"),
|
||||||
createModel: (b) => j("POST", "/models", b),
|
createModel: (b) => j("POST", "/models", b),
|
||||||
updateModel: (id, b) => j("PUT", "/models/" + encodeURIComponent(id), b),
|
updateModel: (id, b) => j("PUT", "/models/" + encodeURIComponent(id), b),
|
||||||
deleteModel: (id) => j("DELETE", "/models/" + encodeURIComponent(id)),
|
deleteModel: (id, deleteFile = true) =>
|
||||||
|
j("DELETE", "/models/" + encodeURIComponent(id) + "?delete_file=" + (deleteFile ? "true" : "false")),
|
||||||
probeModel: (id) => j("POST", "/models/" + encodeURIComponent(id) + "/probe"),
|
probeModel: (id) => j("POST", "/models/" + encodeURIComponent(id) + "/probe"),
|
||||||
|
|
||||||
// routing
|
// routing
|
||||||
@@ -87,6 +88,7 @@ export const api = {
|
|||||||
|
|
||||||
// stats
|
// stats
|
||||||
stats: (range, groupby) => j("GET", `/stats?range=${range}&groupby=${groupby}`),
|
stats: (range, groupby) => j("GET", `/stats?range=${range}&groupby=${groupby}`),
|
||||||
|
clearStats: () => j("DELETE", "/stats"),
|
||||||
requests: (limit = 50, offset = 0) => j("GET", `/requests?limit=${limit}&offset=${offset}`),
|
requests: (limit = 50, offset = 0) => j("GET", `/requests?limit=${limit}&offset=${offset}`),
|
||||||
|
|
||||||
// playground
|
// playground
|
||||||
@@ -99,6 +101,8 @@ export const api = {
|
|||||||
cmfStatus: () => j("GET", "/cmf"),
|
cmfStatus: () => j("GET", "/cmf"),
|
||||||
cmfInstall: () => j("POST", "/cmf/install"),
|
cmfInstall: () => j("POST", "/cmf/install"),
|
||||||
cmfPortCheck: (port) => j("GET", "/cmf/port?port=" + encodeURIComponent(port)),
|
cmfPortCheck: (port) => j("GET", "/cmf/port?port=" + encodeURIComponent(port)),
|
||||||
|
cmfFiles: () => j("GET", "/cmf/files"),
|
||||||
|
providerModels: (b) => j("POST", "/provider/models", b),
|
||||||
|
|
||||||
// CMF model factory (HuggingFace → local .cmf)
|
// CMF model factory (HuggingFace → local .cmf)
|
||||||
hfSearch: (q, limit = 24) =>
|
hfSearch: (q, limit = 24) =>
|
||||||
|
|||||||
Vendored
+49
-1
@@ -59,6 +59,9 @@ export default {
|
|||||||
// dashboard
|
// dashboard
|
||||||
"dash.title": "Dashboard",
|
"dash.title": "Dashboard",
|
||||||
"dash.subtitle": "Live traffic, cost and routing analytics for your model pool.",
|
"dash.subtitle": "Live traffic, cost and routing analytics for your model pool.",
|
||||||
|
"dash.clear": "Clear logs",
|
||||||
|
"dash.clearConfirm": "Clear all recorded stats and logs? This cannot be undone.",
|
||||||
|
"dash.cleared": "Logs cleared",
|
||||||
"dash.range": "Range",
|
"dash.range": "Range",
|
||||||
"dash.range.1h": "1h",
|
"dash.range.1h": "1h",
|
||||||
"dash.range.24h": "24h",
|
"dash.range.24h": "24h",
|
||||||
@@ -129,6 +132,20 @@ export default {
|
|||||||
"models.form.secretMissing": "not set",
|
"models.form.secretMissing": "not set",
|
||||||
"models.form.secretPlaceholder": "set or replace key (write-only)",
|
"models.form.secretPlaceholder": "set or replace key (write-only)",
|
||||||
"models.deleteConfirm": "Delete model \"{id}\"?",
|
"models.deleteConfirm": "Delete model \"{id}\"?",
|
||||||
|
"models.deleteFile": "also delete the model file from disk",
|
||||||
|
"models.deletedWithFile": "Deleted (file removed from disk)",
|
||||||
|
"models.form.idOrModel": "Enter an id or a model name",
|
||||||
|
"models.form.apiKey": "API Key",
|
||||||
|
"models.form.apiKeyPlaceholder": "paste your API key",
|
||||||
|
"models.form.apiKeyNotNeeded": "not required (local server)",
|
||||||
|
"models.form.apiKeyHint": "Stored securely on the gateway; it is not shown again.",
|
||||||
|
"models.form.keyStored": "A key is saved — leave blank to keep it, or paste a new one.",
|
||||||
|
"models.form.keyFromEnv": "Using a key from an environment variable.",
|
||||||
|
"models.form.testKey": "Test & load models",
|
||||||
|
"models.form.testing": "checking…",
|
||||||
|
"models.form.foundModels": "{n} models loaded",
|
||||||
|
"models.form.modelPlaceholder": "type to search…",
|
||||||
|
"models.form.modelHint": "Click “Test & load models” to pull the list from your key, then pick one.",
|
||||||
"models.managed": "managed",
|
"models.managed": "managed",
|
||||||
"models.managedHint": "Provided by the managed CMF server (Settings → Local models). Configure it there.",
|
"models.managedHint": "Provided by the managed CMF server (Settings → Local models). Configure it there.",
|
||||||
|
|
||||||
@@ -244,13 +261,23 @@ export default {
|
|||||||
"settings.cmf.threads": "Worker threads",
|
"settings.cmf.threads": "Worker threads",
|
||||||
"settings.cmf.gpu": "Use GPU (Metal / wgpu)",
|
"settings.cmf.gpu": "Use GPU (Metal / wgpu)",
|
||||||
"settings.cmf.speedNote": "Threads parallelise the decode matvecs (~3x faster; 0 = runtime default). GPU offload mainly helps prefill of long prompts — decode is memory-bandwidth-bound, so on Apple Silicon it is roughly neutral.",
|
"settings.cmf.speedNote": "Threads parallelise the decode matvecs (~3x faster; 0 = runtime default). GPU offload mainly helps prefill of long prompts — decode is memory-bandwidth-bound, so on Apple Silicon it is roughly neutral.",
|
||||||
|
"settings.cmf.pick": "select a model",
|
||||||
"settings.cmf.models": "Local models",
|
"settings.cmf.models": "Local models",
|
||||||
"settings.cmf.modelsHelp": "Each row is one model the gateway runs (its own cortiq serve on its own port) and registers in the pool. Add as many as you have .cmf files and free ports.",
|
"settings.cmf.modelsHelp": "Each row is one model the gateway runs (its own cortiq serve on its own port) and registers in the pool. Add as many as you have .cmf files and free ports.",
|
||||||
"settings.cmf.rowHint": "id · path to .cmf · port · threads · GPU. Changes apply on restart; deleting a row stops that server.",
|
"settings.cmf.rowHint": "id · .cmf file · port · threads · GPU · O(1) · T°/top-p/max tok. Applied on save: a removed row stops its server (memory freed), a new row starts one.",
|
||||||
|
"settings.cmf.thinkAuto": "thoughts: auto",
|
||||||
|
"settings.cmf.thinkOff": "thoughts: off",
|
||||||
|
"settings.cmf.thinkHelp": "Reasoning models (Qwen3/3.5) write out a thinking phase before the answer. Off = answer directly without it (faster, shorter; needs cortiq >= 0.2.1). A request that sets enable_thinking itself always wins.",
|
||||||
|
"settings.cmf.tempHelp": "Default temperature for this model (0 = deterministic, 0.7 = balanced, >1 = creative). Used when the request doesn't set one; a request value always wins. Empty = provider default.",
|
||||||
|
"settings.cmf.topPHelp": "Default nucleus sampling (top-p) for this model: only tokens within this cumulative probability are sampled (0.9 typical). Empty = provider default.",
|
||||||
|
"settings.cmf.maxTokHelp": "Default max output tokens for this model when the request doesn't set one. Protects against runaway long generations. Empty = provider default.",
|
||||||
|
"settings.cmf.o1Auto": "O(1): auto",
|
||||||
|
"settings.cmf.o1Help": "O(1) attention at serve time: auto — follow the hint recorded in the .cmf; off — force exact attention (fixes incoherent output after a too-aggressive --o1 conversion, no re-convert needed); all/deepN — force-enable on those layers.",
|
||||||
"settings.cmf.addServer": "+ Add local model",
|
"settings.cmf.addServer": "+ Add local model",
|
||||||
"settings.cmf.checkPort": "Check port",
|
"settings.cmf.checkPort": "Check port",
|
||||||
"settings.cmf.portFree": "port is free",
|
"settings.cmf.portFree": "port is free",
|
||||||
"settings.cmf.portBusy": "port is in use",
|
"settings.cmf.portBusy": "port is in use",
|
||||||
|
"settings.cmf.portOwn": "in use by this gateway's current server — freed on restart, replacing is fine",
|
||||||
"settings.cmf.useSuggested": "use {port}",
|
"settings.cmf.useSuggested": "use {port}",
|
||||||
"settings.cmf.serving": "serving {model} on :{port}",
|
"settings.cmf.serving": "serving {model} on :{port}",
|
||||||
"settings.cmf.intro": "Run models locally with the cortiq engine — works even offline or when the router is down.",
|
"settings.cmf.intro": "Run models locally with the cortiq engine — works even offline or when the router is down.",
|
||||||
@@ -303,6 +330,10 @@ export default {
|
|||||||
"settings.route.maxChars": "Max chars",
|
"settings.route.maxChars": "Max chars",
|
||||||
"settings.route.cacheTtl": "Cache TTL",
|
"settings.route.cacheTtl": "Cache TTL",
|
||||||
"settings.route.profile": "Default profile",
|
"settings.route.profile": "Default profile",
|
||||||
|
"settings.route.strategyHelp": "What text is sent to the router for classification: last_user = only the last user message; last_user_plus_system = last user message + system prompt; concat_all = the whole conversation.",
|
||||||
|
"settings.route.maxCharsHelp": "Max characters of that text sent to the router (longer is truncated). More = more accurate classification, but slower/pricier router calls.",
|
||||||
|
"settings.route.cacheTtlHelp": "How long a routing decision is cached for an identical request before the router is asked again (e.g. 60s, 5m).",
|
||||||
|
"settings.route.profileHelp": "Default routing profile when a request doesn't specify one: cost-saver (cheapest), balanced, or quality-first (best model). Biases which model the router picks.",
|
||||||
"settings.breaker": "Circuit breaker",
|
"settings.breaker": "Circuit breaker",
|
||||||
"settings.breaker.threshold": "Error threshold",
|
"settings.breaker.threshold": "Error threshold",
|
||||||
"settings.breaker.cooldown": "Cooldown",
|
"settings.breaker.cooldown": "Cooldown",
|
||||||
@@ -380,4 +411,21 @@ export default {
|
|||||||
"import.adv.shard": "Shard max (GB)",
|
"import.adv.shard": "Shard max (GB)",
|
||||||
"import.adv.noShard": "no sharding",
|
"import.adv.noShard": "no sharding",
|
||||||
"import.adv.skipMtp": "skip MTP head",
|
"import.adv.skipMtp": "skip MTP head",
|
||||||
|
"import.adv.o1": "O(1) attention",
|
||||||
|
"import.adv.o1Hint": "Constant-memory streaming attention: the KV cache is released after prefill, so long contexts don't grow memory. Weights stay byte-identical — only a load-time hint is recorded; you can switch back anytime (serve with --o1 off). Validated on Qwen3-0.6B: all layers ≈ ×1.18 perplexity.",
|
||||||
|
"import.adv.o1Off": "off — exact attention (default)",
|
||||||
|
"import.adv.o1All": "all — convert every softmax layer",
|
||||||
|
"import.adv.o1Deep": "deep N — only the N deepest layers",
|
||||||
|
"import.adv.o1Custom": "custom — explicit layer list",
|
||||||
|
"import.adv.o1DeepN": "N deep layers",
|
||||||
|
"import.adv.o1Layers": "Layers (comma-separated)",
|
||||||
|
"import.adv.o1mHint": "landmark budget (default 32)",
|
||||||
|
"import.adv.o1mLabel": "Landmarks (--o1-m)",
|
||||||
|
"import.adv.o1mHelp": "How many landmark tokens summarize the whole older history when attention is approximated. More = closer to exact attention but slower and more memory; fewer = faster but rougher. 32 is the validated sweet spot — change only if you know why.",
|
||||||
|
"import.adv.o1windowLabel": "Exact window (--o1-window)",
|
||||||
|
"import.adv.o1windowHelp": "The last N tokens always get exact (non-approximated) attention. This protects quality on the text the model just wrote/read. Bigger = better local quality, more memory. Default 128.",
|
||||||
|
"import.adv.o1sinkLabel": "Sink keys (--o1-sink)",
|
||||||
|
"import.adv.o1sinkHelp": "The first N tokens of the context are kept exact forever ('attention sinks'). Models lean on these anchors — keeping them exact stabilizes long generations. Default 4; rarely needs changing.",
|
||||||
|
"import.adv.o1windowHint": "exact-window width (default 128)",
|
||||||
|
"import.adv.o1sinkHint": "permanent exact sink keys (default 4)",
|
||||||
};
|
};
|
||||||
|
|||||||
Vendored
+49
-1
@@ -58,6 +58,9 @@ export default {
|
|||||||
// dashboard
|
// dashboard
|
||||||
"dash.title": "Дашборд",
|
"dash.title": "Дашборд",
|
||||||
"dash.subtitle": "Трафик в реальном времени, расходы и аналитика роутинга по пулу моделей.",
|
"dash.subtitle": "Трафик в реальном времени, расходы и аналитика роутинга по пулу моделей.",
|
||||||
|
"dash.clear": "Очистить логи",
|
||||||
|
"dash.clearConfirm": "Очистить всю статистику и логи? Отменить нельзя.",
|
||||||
|
"dash.cleared": "Логи очищены",
|
||||||
"dash.range": "Период",
|
"dash.range": "Период",
|
||||||
"dash.range.1h": "1ч",
|
"dash.range.1h": "1ч",
|
||||||
"dash.range.24h": "24ч",
|
"dash.range.24h": "24ч",
|
||||||
@@ -128,6 +131,20 @@ export default {
|
|||||||
"models.form.secretMissing": "не задан",
|
"models.form.secretMissing": "не задан",
|
||||||
"models.form.secretPlaceholder": "задать или заменить ключ (только запись)",
|
"models.form.secretPlaceholder": "задать или заменить ключ (только запись)",
|
||||||
"models.deleteConfirm": "Удалить модель \"{id}\"?",
|
"models.deleteConfirm": "Удалить модель \"{id}\"?",
|
||||||
|
"models.deleteFile": "также удалить файл модели с диска",
|
||||||
|
"models.deletedWithFile": "Удалено (файл стёрт с диска)",
|
||||||
|
"models.form.idOrModel": "Укажите id или имя модели",
|
||||||
|
"models.form.apiKey": "API-ключ",
|
||||||
|
"models.form.apiKeyPlaceholder": "вставьте API-ключ",
|
||||||
|
"models.form.apiKeyNotNeeded": "не нужен (локальный сервер)",
|
||||||
|
"models.form.apiKeyHint": "Хранится безопасно на шлюзе; больше не показывается.",
|
||||||
|
"models.form.keyStored": "Ключ сохранён — оставьте пустым, чтобы не менять, или вставьте новый.",
|
||||||
|
"models.form.keyFromEnv": "Используется ключ из переменной окружения.",
|
||||||
|
"models.form.testKey": "Проверить и загрузить модели",
|
||||||
|
"models.form.testing": "проверка…",
|
||||||
|
"models.form.foundModels": "загружено моделей: {n}",
|
||||||
|
"models.form.modelPlaceholder": "введите для поиска…",
|
||||||
|
"models.form.modelHint": "Нажмите «Проверить и загрузить модели», чтобы подтянуть список по ключу, затем выберите.",
|
||||||
"models.managed": "управляемая",
|
"models.managed": "управляемая",
|
||||||
"models.managedHint": "Предоставляется CMF-сервером (Настройки → Локальные модели). Настраивается там.",
|
"models.managedHint": "Предоставляется CMF-сервером (Настройки → Локальные модели). Настраивается там.",
|
||||||
|
|
||||||
@@ -243,13 +260,23 @@ export default {
|
|||||||
"settings.cmf.threads": "Рабочие потоки",
|
"settings.cmf.threads": "Рабочие потоки",
|
||||||
"settings.cmf.gpu": "Использовать GPU (Metal / wgpu)",
|
"settings.cmf.gpu": "Использовать GPU (Metal / wgpu)",
|
||||||
"settings.cmf.speedNote": "Потоки распараллеливают декод-матвеки (~×3; 0 = по умолчанию рантайма). GPU-оффлоад помогает в основном на prefill длинных промптов — декод упирается в пропускную способность памяти, поэтому на Apple Silicon почти нейтрален.",
|
"settings.cmf.speedNote": "Потоки распараллеливают декод-матвеки (~×3; 0 = по умолчанию рантайма). GPU-оффлоад помогает в основном на prefill длинных промптов — декод упирается в пропускную способность памяти, поэтому на Apple Silicon почти нейтрален.",
|
||||||
|
"settings.cmf.pick": "выберите модель",
|
||||||
"settings.cmf.models": "Локальные модели",
|
"settings.cmf.models": "Локальные модели",
|
||||||
"settings.cmf.modelsHelp": "Каждая строка — отдельная модель: шлюз поднимает свой cortiq serve на своём порту и регистрирует её в пуле. Добавляйте столько, сколько есть .cmf-файлов и свободных портов.",
|
"settings.cmf.modelsHelp": "Каждая строка — отдельная модель: шлюз поднимает свой cortiq serve на своём порту и регистрирует её в пуле. Добавляйте столько, сколько есть .cmf-файлов и свободных портов.",
|
||||||
"settings.cmf.rowHint": "id · путь к .cmf · порт · потоки · GPU. Изменения применяются после перезапуска; удаление строки останавливает сервер.",
|
"settings.cmf.rowHint": "id · файл .cmf · порт · потоки · GPU · O(1) · T°/top-p/max tok. Применяется сразу при сохранении: удалённая строка останавливает сервер (память освобождается), новая — запускает.",
|
||||||
|
"settings.cmf.thinkAuto": "мысли: авто",
|
||||||
|
"settings.cmf.thinkOff": "мысли: выкл",
|
||||||
|
"settings.cmf.thinkHelp": "Рассуждающие модели (Qwen3/3.5) сначала пишут фазу размышлений. Выкл = отвечать сразу, без неё (быстрее и короче; нужен cortiq ≥ 0.2.1). Если запрос сам задал enable_thinking — он важнее.",
|
||||||
|
"settings.cmf.tempHelp": "Температура по умолчанию для этой модели (0 = детерминированно, 0.7 = баланс, >1 = креативно). Подставляется, если запрос её не задал; значение из запроса всегда важнее. Пусто = дефолт провайдера.",
|
||||||
|
"settings.cmf.topPHelp": "Nucleus-сэмплинг (top-p) по умолчанию: выбираются только токены в пределах этой суммарной вероятности (обычно 0.9). Пусто = дефолт провайдера.",
|
||||||
|
"settings.cmf.maxTokHelp": "Максимум выходных токенов по умолчанию, если запрос не задал свой. Защищает от бесконечных генераций. Пусто = дефолт провайдера.",
|
||||||
|
"settings.cmf.o1Auto": "O(1): авто",
|
||||||
|
"settings.cmf.o1Help": "O(1)-внимание при запуске: авто — использовать подсказку из .cmf; off — принудительно точное внимание (лечит бессвязный вывод после слишком агрессивной --o1 конвертации, без переконвертации); all/deepN — принудительно включить на этих слоях.",
|
||||||
"settings.cmf.addServer": "+ Добавить модель",
|
"settings.cmf.addServer": "+ Добавить модель",
|
||||||
"settings.cmf.checkPort": "Проверить порт",
|
"settings.cmf.checkPort": "Проверить порт",
|
||||||
"settings.cmf.portFree": "порт свободен",
|
"settings.cmf.portFree": "порт свободен",
|
||||||
"settings.cmf.portBusy": "порт занят",
|
"settings.cmf.portBusy": "порт занят",
|
||||||
|
"settings.cmf.portOwn": "занят текущим сервером шлюза — освободится после перезапуска, замена допустима",
|
||||||
"settings.cmf.useSuggested": "занять {port}",
|
"settings.cmf.useSuggested": "занять {port}",
|
||||||
"settings.cmf.serving": "работает: {model} на :{port}",
|
"settings.cmf.serving": "работает: {model} на :{port}",
|
||||||
"settings.cmf.intro": "Запускайте модели локально движком cortiq — работает даже офлайн или когда роутер недоступен.",
|
"settings.cmf.intro": "Запускайте модели локально движком cortiq — работает даже офлайн или когда роутер недоступен.",
|
||||||
@@ -302,6 +329,10 @@ export default {
|
|||||||
"settings.route.maxChars": "Макс. символов",
|
"settings.route.maxChars": "Макс. символов",
|
||||||
"settings.route.cacheTtl": "TTL кэша",
|
"settings.route.cacheTtl": "TTL кэша",
|
||||||
"settings.route.profile": "Профиль по умолчанию",
|
"settings.route.profile": "Профиль по умолчанию",
|
||||||
|
"settings.route.strategyHelp": "Какой текст отправляется роутеру для классификации: last_user — только последнее сообщение пользователя; last_user_plus_system — последнее сообщение + системный промпт; concat_all — вся переписка.",
|
||||||
|
"settings.route.maxCharsHelp": "Максимум символов этого текста, отправляемого роутеру (длиннее — обрезается). Больше = точнее классификация, но медленнее/дороже вызовы роутера.",
|
||||||
|
"settings.route.cacheTtlHelp": "Сколько кэшируется решение роутинга для одинакового запроса, прежде чем роутер спрашивается снова (напр. 60s, 5m).",
|
||||||
|
"settings.route.profileHelp": "Профиль роутинга по умолчанию, если запрос его не задаёт: cost-saver (дешевле), balanced (баланс) или quality-first (качество). Влияет на выбор модели роутером.",
|
||||||
"settings.breaker": "Автоматический выключатель",
|
"settings.breaker": "Автоматический выключатель",
|
||||||
"settings.breaker.threshold": "Порог ошибок",
|
"settings.breaker.threshold": "Порог ошибок",
|
||||||
"settings.breaker.cooldown": "Время восстановления",
|
"settings.breaker.cooldown": "Время восстановления",
|
||||||
@@ -379,4 +410,21 @@ export default {
|
|||||||
"import.adv.shard": "Шард макс (ГБ)",
|
"import.adv.shard": "Шард макс (ГБ)",
|
||||||
"import.adv.noShard": "без шардинга",
|
"import.adv.noShard": "без шардинга",
|
||||||
"import.adv.skipMtp": "пропустить MTP-голову",
|
"import.adv.skipMtp": "пропустить MTP-голову",
|
||||||
|
"import.adv.o1": "O(1)-внимание",
|
||||||
|
"import.adv.o1Hint": "Потоковое внимание с постоянной памятью: KV-кэш освобождается после prefill, длинный контекст не раздувает память. Веса не меняются — записывается лишь подсказка на загрузку; вернуться можно всегда (запуск с --o1 off). Проверено на Qwen3-0.6B: все слои ≈ ×1.18 перплексии.",
|
||||||
|
"import.adv.o1Off": "выкл — точное внимание (по умолчанию)",
|
||||||
|
"import.adv.o1All": "all — конвертировать все softmax-слои",
|
||||||
|
"import.adv.o1Deep": "deep N — только N самых глубоких слоёв",
|
||||||
|
"import.adv.o1Custom": "вручную — явный список слоёв",
|
||||||
|
"import.adv.o1DeepN": "N глубоких слоёв",
|
||||||
|
"import.adv.o1Layers": "Слои (через запятую)",
|
||||||
|
"import.adv.o1mHint": "число опорных точек (по умолчанию 32)",
|
||||||
|
"import.adv.o1mLabel": "Опорные точки (--o1-m)",
|
||||||
|
"import.adv.o1mHelp": "Сколько опорных токенов представляют всю старую историю при аппроксимации внимания. Больше — ближе к точному вниманию, но медленнее и больше памяти; меньше — быстрее, но грубее. 32 — проверенное значение; меняйте, только если понимаете зачем.",
|
||||||
|
"import.adv.o1windowLabel": "Точное окно (--o1-window)",
|
||||||
|
"import.adv.o1windowHelp": "Последние N токенов всегда обрабатываются точным (не аппроксимированным) вниманием — это защищает качество на только что написанном/прочитанном тексте. Больше — лучше локальное качество, больше памяти. По умолчанию 128.",
|
||||||
|
"import.adv.o1sinkLabel": "Sink-ключи (--o1-sink)",
|
||||||
|
"import.adv.o1sinkHelp": "Первые N токенов контекста навсегда остаются точными («якоря внимания»). Модели опираются на эти якоря — их точность стабилизирует длинную генерацию. По умолчанию 4; менять почти никогда не нужно.",
|
||||||
|
"import.adv.o1windowHint": "ширина точного окна (по умолчанию 128)",
|
||||||
|
"import.adv.o1sinkHint": "постоянные точные sink-ключи (по умолчанию 4)",
|
||||||
};
|
};
|
||||||
|
|||||||
Vendored
+3
-1
@@ -25,7 +25,9 @@ export function clear(node) {
|
|||||||
|
|
||||||
export function mount(node, ...children) {
|
export function mount(node, ...children) {
|
||||||
clear(node);
|
clear(node);
|
||||||
for (const c of children.flat()) if (c != null && c !== false) node.appendChild(c);
|
for (const c of children.flat())
|
||||||
|
if (c != null && c !== false)
|
||||||
|
node.appendChild(typeof c === "object" ? c : document.createTextNode(String(c)));
|
||||||
return node;
|
return node;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Vendored
+19
-5
@@ -1,5 +1,5 @@
|
|||||||
// Dashboard — KPIs, time-series charts, breakdown, recent requests, health.
|
// Dashboard — KPIs, time-series charts, breakdown, recent requests, health.
|
||||||
import { h, mount, money, num, ms, pct, timeAgo } from "../ui.js";
|
import { h, mount, money, num, ms, pct, timeAgo, toast, confirmDialog } from "../ui.js";
|
||||||
import { t } from "../i18n.js";
|
import { t } from "../i18n.js";
|
||||||
import { api } from "../api.js";
|
import { api } from "../api.js";
|
||||||
import { lineChart, barList } from "../charts.js";
|
import { lineChart, barList } from "../charts.js";
|
||||||
@@ -53,10 +53,24 @@ export async function renderDashboard() {
|
|||||||
"div",
|
"div",
|
||||||
{ class: "flex" },
|
{ class: "flex" },
|
||||||
h("div", { class: "grow" }, h("h2", {}, t("dash.title")), h("p", {}, t("dash.subtitle"))),
|
h("div", { class: "grow" }, h("h2", {}, t("dash.title")), h("p", {}, t("dash.subtitle"))),
|
||||||
seg(RANGES, range, (r) => {
|
h("div", { class: "flex", style: "gap:8px;align-items:center" },
|
||||||
range = r;
|
seg(RANGES, range, (r) => {
|
||||||
load();
|
range = r;
|
||||||
}, (r) => t("dash.range." + r))
|
load();
|
||||||
|
}, (r) => t("dash.range." + r)),
|
||||||
|
h("button", {
|
||||||
|
class: "btn sm danger",
|
||||||
|
onClick: async () => {
|
||||||
|
if (!(await confirmDialog(t("dash.clearConfirm")))) return;
|
||||||
|
try {
|
||||||
|
await api.clearStats();
|
||||||
|
toast(t("dash.cleared"), "good");
|
||||||
|
await load();
|
||||||
|
} catch (e) {
|
||||||
|
toast(String(e.message || e), "bad");
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}, t("dash.clear")))
|
||||||
)
|
)
|
||||||
),
|
),
|
||||||
|
|
||||||
|
|||||||
Vendored
+50
-2
@@ -15,8 +15,14 @@ const QUANTS = [
|
|||||||
["F32", "import.q.f32"],
|
["F32", "import.q.f32"],
|
||||||
];
|
];
|
||||||
|
|
||||||
function field(label, control, hint) {
|
// A small "?" badge with a hover/focus tooltip (same style as Settings).
|
||||||
return h("label", { class: "field" }, h("span", {}, label), control,
|
function help(text) {
|
||||||
|
return text ? h("span", { class: "help", title: text, tabindex: "0", role: "img", "aria-label": text }, "?") : null;
|
||||||
|
}
|
||||||
|
function field(label, control, hint, helpText) {
|
||||||
|
return h("label", { class: "field" },
|
||||||
|
h("span", {}, label, helpText ? " " : null, help(helpText)),
|
||||||
|
control,
|
||||||
hint ? h("div", { class: "hint" }, hint) : null);
|
hint ? h("div", { class: "hint" }, hint) : null);
|
||||||
}
|
}
|
||||||
function opt(v, label, sel) {
|
function opt(v, label, sel) {
|
||||||
@@ -125,6 +131,32 @@ export function renderImport() {
|
|||||||
const meanBitsIn = h("input", { type: "number", step: "0.1", placeholder: "5.0" });
|
const meanBitsIn = h("input", { type: "number", step: "0.1", placeholder: "5.0" });
|
||||||
const shardIn = h("input", { type: "number", step: "0.5", placeholder: t("import.adv.noShard") });
|
const shardIn = h("input", { type: "number", step: "0.5", placeholder: t("import.adv.noShard") });
|
||||||
const skipMtp = h("input", { type: "checkbox" });
|
const skipMtp = h("input", { type: "checkbox" });
|
||||||
|
// O(1) attention (cortiq ≥ 0.2.0): constant-memory streaming attention hint,
|
||||||
|
// weights byte-identical — the runtime applies it at load.
|
||||||
|
const o1Sel = h("select", {},
|
||||||
|
opt("", t("import.adv.o1Off")),
|
||||||
|
opt("all", t("import.adv.o1All")),
|
||||||
|
opt("deep", t("import.adv.o1Deep")),
|
||||||
|
opt("custom", t("import.adv.o1Custom")));
|
||||||
|
const o1DeepIn = h("input", { type: "number", min: "1", placeholder: "12" });
|
||||||
|
const o1CustomIn = h("input", { placeholder: "0,4,8" });
|
||||||
|
const o1M = h("input", { type: "number", min: "1", placeholder: "32" });
|
||||||
|
const o1Window = h("input", { type: "number", min: "0", placeholder: "128" });
|
||||||
|
const o1Sink = h("input", { type: "number", min: "0", placeholder: "4" });
|
||||||
|
const o1SpecRow = h("div", { class: "row", style: "display:none" },
|
||||||
|
field(t("import.adv.o1DeepN"), o1DeepIn),
|
||||||
|
field(t("import.adv.o1Layers"), o1CustomIn));
|
||||||
|
const o1Knobs = h("div", { class: "row", style: "display:none" },
|
||||||
|
field(t("import.adv.o1mLabel"), o1M, t("import.adv.o1mHint"), t("import.adv.o1mHelp")),
|
||||||
|
field(t("import.adv.o1windowLabel"), o1Window, t("import.adv.o1windowHint"), t("import.adv.o1windowHelp")),
|
||||||
|
field(t("import.adv.o1sinkLabel"), o1Sink, t("import.adv.o1sinkHint"), t("import.adv.o1sinkHelp")));
|
||||||
|
o1Sel.addEventListener("change", () => {
|
||||||
|
const on = !!o1Sel.value;
|
||||||
|
o1Knobs.style.display = on ? "" : "none";
|
||||||
|
o1SpecRow.style.display = o1Sel.value === "deep" || o1Sel.value === "custom" ? "" : "none";
|
||||||
|
o1DeepIn.parentElement.style.display = o1Sel.value === "deep" ? "" : "none";
|
||||||
|
o1CustomIn.parentElement.style.display = o1Sel.value === "custom" ? "" : "none";
|
||||||
|
});
|
||||||
const advBody = h(
|
const advBody = h(
|
||||||
"div",
|
"div",
|
||||||
{ class: "adv-body", style: "display:none" },
|
{ class: "adv-body", style: "display:none" },
|
||||||
@@ -137,6 +169,9 @@ export function renderImport() {
|
|||||||
h("div", { class: "row" },
|
h("div", { class: "row" },
|
||||||
field(t("import.adv.shard"), shardIn),
|
field(t("import.adv.shard"), shardIn),
|
||||||
field("MTP", h("label", { class: "check" }, skipMtp, t("import.adv.skipMtp")))),
|
field("MTP", h("label", { class: "check" }, skipMtp, t("import.adv.skipMtp")))),
|
||||||
|
field(t("import.adv.o1"), o1Sel, t("import.adv.o1Hint")),
|
||||||
|
o1SpecRow,
|
||||||
|
o1Knobs,
|
||||||
);
|
);
|
||||||
const advToggle = h("button", { class: "btn ghost sm" }, "⚙ " + t("import.adv.title"));
|
const advToggle = h("button", { class: "btn ghost sm" }, "⚙ " + t("import.adv.title"));
|
||||||
advToggle.addEventListener("click", () => {
|
advToggle.addEventListener("click", () => {
|
||||||
@@ -156,6 +191,19 @@ export function renderImport() {
|
|||||||
if (vbitSel.value) params.vbit_shape = vbitSel.value;
|
if (vbitSel.value) params.vbit_shape = vbitSel.value;
|
||||||
if (meanBitsIn.value) params.mean_bits = parseFloat(meanBitsIn.value);
|
if (meanBitsIn.value) params.mean_bits = parseFloat(meanBitsIn.value);
|
||||||
if (shardIn.value) params.shard_max_gb = parseFloat(shardIn.value);
|
if (shardIn.value) params.shard_max_gb = parseFloat(shardIn.value);
|
||||||
|
if (o1Sel.value) {
|
||||||
|
// spec: all | deepN | explicit layer list
|
||||||
|
params.o1 =
|
||||||
|
o1Sel.value === "deep" ? "deep" + (parseInt(o1DeepIn.value) || 12)
|
||||||
|
: o1Sel.value === "custom" ? o1CustomIn.value.trim()
|
||||||
|
: "all";
|
||||||
|
if (!params.o1) delete params.o1;
|
||||||
|
else {
|
||||||
|
if (o1M.value) params.o1_m = parseInt(o1M.value);
|
||||||
|
if (o1Window.value) params.o1_window = parseInt(o1Window.value);
|
||||||
|
if (o1Sink.value) params.o1_sink = parseInt(o1Sink.value);
|
||||||
|
}
|
||||||
|
}
|
||||||
goBtn.disabled = true;
|
goBtn.disabled = true;
|
||||||
try {
|
try {
|
||||||
await api.startImport(params);
|
await api.startImport(params);
|
||||||
|
|||||||
Vendored
+149
-39
@@ -1,5 +1,5 @@
|
|||||||
// Models — pool table with add/edit modal, probe, and per-model secret entry.
|
// Models — pool table with add/edit modal, probe, and per-model secret entry.
|
||||||
import { h, mount, modal, toast, confirmDialog, money } from "../ui.js";
|
import { h, mount, modal, toast, money } from "../ui.js";
|
||||||
import { t } from "../i18n.js";
|
import { t } from "../i18n.js";
|
||||||
import { api } from "../api.js";
|
import { api } from "../api.js";
|
||||||
import { appState } from "../app.js";
|
import { appState } from "../app.js";
|
||||||
@@ -16,52 +16,142 @@ function keyBadge(src) {
|
|||||||
return h("span", { class: "badge " + (map[src] ?? "") }, src);
|
return h("span", { class: "badge " + (map[src] ?? "") }, src);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Verified per-provider defaults (base URL, key env, whether a key is needed).
|
||||||
|
const PROVIDER_DEFAULTS = {
|
||||||
|
openai: { base: "https://api.openai.com/v1", keyEnv: "OPENAI_API_KEY", needsKey: true, tier: "mid", caps: ["tools", "vision"] },
|
||||||
|
anthropic: { base: "https://api.anthropic.com", keyEnv: "ANTHROPIC_API_KEY", needsKey: true, tier: "expensive", caps: ["tools", "vision"] },
|
||||||
|
openrouter: { base: "https://openrouter.ai/api/v1", keyEnv: "OPENROUTER_API_KEY", needsKey: true, tier: "mid", caps: ["tools", "vision"] },
|
||||||
|
lmstudio: { base: "http://localhost:1234/v1", keyEnv: "", needsKey: false, tier: "local", caps: ["tools", "vision"] },
|
||||||
|
ollama: { base: "http://localhost:11434/v1", keyEnv: "", needsKey: false, tier: "local", caps: ["tools"] },
|
||||||
|
http: { base: "", keyEnv: "", needsKey: false, tier: "cheap", caps: ["tools"] },
|
||||||
|
};
|
||||||
|
|
||||||
|
// Derive a clean model id from a model name (last path segment, slugified).
|
||||||
|
// e.g. "qwen/qwen3.7-max" → "qwen3-7-max".
|
||||||
|
function slugId(name) {
|
||||||
|
const base = (name || "").split("/").pop() || "";
|
||||||
|
return base.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
||||||
|
}
|
||||||
|
|
||||||
function modelForm(meta, existing) {
|
function modelForm(meta, existing) {
|
||||||
const m = existing || { provider: "openai", kind: "chat", cost_tier: "cheap", caps: [], price_in: 0, price_out: 0 };
|
const m = existing || { provider: "openai", kind: "chat", cost_tier: "mid", caps: ["tools"], price_in: 0, price_out: 0 };
|
||||||
const idIn = h("input", { value: m.id || "", placeholder: "local-qwen", disabled: existing ? true : null });
|
const idIn = h("input", { value: m.id || "", placeholder: "my-model", disabled: existing ? true : null });
|
||||||
const providerSel = h("select", {}, ...(meta.providers || ["openai"]).map((p) => opt(p, p, p === m.provider)));
|
const providerSel = h("select", {}, ...(meta.providers || ["openai"]).map((p) => opt(p, p, p === m.provider)));
|
||||||
const baseUrlIn = h("input", { value: m.base_url || "", placeholder: "http://localhost:8000/v1" });
|
const baseUrlIn = h("input", { value: m.base_url || "", placeholder: "https://api.openai.com/v1" });
|
||||||
const modelIn = h("input", { value: m.model || "", placeholder: "qwen2.5-7b-instruct" });
|
|
||||||
|
// API key — typed directly, stored securely as a secret under a provider env name.
|
||||||
|
const keyIn = h("input", { type: "password", placeholder: t("models.form.apiKeyPlaceholder"), autocomplete: "new-password" });
|
||||||
|
const testBtn = h("button", { class: "btn small", type: "button" }, t("models.form.testKey"));
|
||||||
|
const keyStatus = h("span", { class: "small", style: "margin-left:8px" });
|
||||||
|
|
||||||
|
// Model — below the key (some providers list models from the key); searchable datalist.
|
||||||
|
const dlId = "model-opts-" + (m.id || "new");
|
||||||
|
const dl = h("datalist", { id: dlId });
|
||||||
|
if (m.model) dl.appendChild(h("option", { value: m.model }));
|
||||||
|
const modelIn = h("input", { value: m.model || "", placeholder: t("models.form.modelPlaceholder"), list: dlId, autocomplete: "off" });
|
||||||
|
|
||||||
const kindSel = h("select", {}, ...(meta.kinds || ["chat"]).map((k) => opt(k, k, k === m.kind)));
|
const kindSel = h("select", {}, ...(meta.kinds || ["chat"]).map((k) => opt(k, k, k === m.kind)));
|
||||||
const tierSel = h("select", {}, ...(meta.cost_tiers || ["cheap"]).map((c) => opt(c, c, c === m.cost_tier)));
|
const tierSel = h("select", {}, ...(meta.cost_tiers || ["cheap"]).map((c) => opt(c, c, c === m.cost_tier)));
|
||||||
const priceInIn = h("input", { type: "number", step: "0.01", value: m.price_in ?? 0 });
|
const priceInIn = h("input", { type: "number", step: "0.01", value: m.price_in ?? 0 });
|
||||||
const priceOutIn = h("input", { type: "number", step: "0.01", value: m.price_out ?? 0 });
|
const priceOutIn = h("input", { type: "number", step: "0.01", value: m.price_out ?? 0 });
|
||||||
const keyEnvIn = h("input", { value: m.api_key_env || "", placeholder: "OPENAI_API_KEY" });
|
|
||||||
const secretIn = h("input", { type: "password", placeholder: t("models.form.secretPlaceholder") });
|
|
||||||
const capInputs = (meta.caps || ["tools", "vision"]).map((c) => {
|
const capInputs = (meta.caps || ["tools", "vision"]).map((c) => {
|
||||||
const cb = h("input", { type: "checkbox", checked: (m.caps || []).includes(c) ? true : null });
|
const cb = h("input", { type: "checkbox", checked: (m.caps || []).includes(c) ? true : null });
|
||||||
return { c, cb };
|
return { c, cb };
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const envFor = () =>
|
||||||
|
(existing && existing.api_key_env) ||
|
||||||
|
(PROVIDER_DEFAULTS[providerSel.value] || {}).keyEnv ||
|
||||||
|
providerSel.value.toUpperCase().replace(/[^A-Z0-9]/g, "_") + "_API_KEY";
|
||||||
|
|
||||||
|
// Fill in a provider's verified defaults when it is selected.
|
||||||
|
function applyDefaults() {
|
||||||
|
const d = PROVIDER_DEFAULTS[providerSel.value];
|
||||||
|
if (!d) return;
|
||||||
|
baseUrlIn.value = d.base;
|
||||||
|
tierSel.value = d.tier;
|
||||||
|
capInputs.forEach(({ c, cb }) => { cb.checked = d.caps.includes(c); });
|
||||||
|
keyIn.placeholder = d.needsKey ? t("models.form.apiKeyPlaceholder") : t("models.form.apiKeyNotNeeded");
|
||||||
|
mount(keyStatus, "");
|
||||||
|
}
|
||||||
|
providerSel.addEventListener("change", applyDefaults);
|
||||||
|
if (!existing) applyDefaults();
|
||||||
|
|
||||||
|
// Auto-fill the id from the model name (until the user edits id by hand).
|
||||||
|
if (!existing) {
|
||||||
|
modelIn.addEventListener("input", () => {
|
||||||
|
if (!idIn.value || idIn._auto) {
|
||||||
|
idIn.value = slugId(modelIn.value);
|
||||||
|
idIn._auto = true;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
idIn.addEventListener("input", () => { idIn._auto = false; });
|
||||||
|
}
|
||||||
|
|
||||||
|
async function testKey() {
|
||||||
|
if (!baseUrlIn.value.trim()) { mount(keyStatus, h("span", { style: "color:var(--bad)" }, t("models.form.baseUrl") + "?")); return; }
|
||||||
|
testBtn.disabled = true;
|
||||||
|
mount(keyStatus, t("models.form.testing"));
|
||||||
|
try {
|
||||||
|
const r = await api.providerModels({
|
||||||
|
provider: providerSel.value,
|
||||||
|
base_url: baseUrlIn.value.trim(),
|
||||||
|
api_key: keyIn.value.trim() || null,
|
||||||
|
api_key_env: envFor(),
|
||||||
|
});
|
||||||
|
if (r.ok) {
|
||||||
|
dl.replaceChildren(...(r.models || []).map((mm) => h("option", { value: mm })));
|
||||||
|
mount(keyStatus, h("span", { style: "color:var(--good)" }, "✓ " + t("models.form.foundModels", { n: r.count })));
|
||||||
|
modelIn.focus();
|
||||||
|
} else {
|
||||||
|
mount(keyStatus, h("span", { style: "color:var(--bad)" }, "✗ " + (r.error || t("common.error"))));
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
mount(keyStatus, h("span", { style: "color:var(--bad)" }, "✗ " + String(e.message || e)));
|
||||||
|
}
|
||||||
|
testBtn.disabled = false;
|
||||||
|
}
|
||||||
|
testBtn.addEventListener("click", testKey);
|
||||||
|
|
||||||
|
const keyHint =
|
||||||
|
existing && existing.key_source === "store" ? t("models.form.keyStored")
|
||||||
|
: existing && existing.key_source === "env" ? t("models.form.keyFromEnv")
|
||||||
|
: t("models.form.apiKeyHint");
|
||||||
|
|
||||||
const node = h(
|
const node = h(
|
||||||
"div",
|
"div",
|
||||||
{},
|
{},
|
||||||
h("div", { class: "row" }, field(t("models.form.id"), idIn, existing ? null : t("models.form.idHint")), field(t("models.form.provider"), providerSel)),
|
h("div", { class: "row" }, field(t("models.form.id"), idIn, existing ? null : t("models.form.idHint")), field(t("models.form.provider"), providerSel)),
|
||||||
field(t("models.form.baseUrl"), baseUrlIn),
|
field(t("models.form.baseUrl"), baseUrlIn),
|
||||||
field(t("models.form.model"), modelIn),
|
field(t("models.form.apiKey"), h("div", { class: "flex wrap", style: "gap:8px;align-items:center" }, keyIn, testBtn, keyStatus), keyHint),
|
||||||
|
field(t("models.form.model"), modelIn, t("models.form.modelHint")),
|
||||||
|
dl,
|
||||||
h("div", { class: "row" }, field(t("models.form.kind"), kindSel), field(t("models.form.costTier"), tierSel)),
|
h("div", { class: "row" }, field(t("models.form.kind"), kindSel), field(t("models.form.costTier"), tierSel)),
|
||||||
h("div", { class: "row" }, field(t("models.form.priceIn"), priceInIn), field(t("models.form.priceOut"), priceOutIn)),
|
h("div", { class: "row" }, field(t("models.form.priceIn"), priceInIn), field(t("models.form.priceOut"), priceOutIn)),
|
||||||
field(t("models.form.caps"), h("div", { class: "flex wrap" }, ...capInputs.map(({ c, cb }) => h("label", { class: "check" }, cb, c)))),
|
field(t("models.form.caps"), h("div", { class: "flex wrap" }, ...capInputs.map(({ c, cb }) => h("label", { class: "check" }, cb, c))))
|
||||||
field(t("models.form.apiKeyEnv"), keyEnvIn, t("models.form.apiKeyEnvHint")),
|
|
||||||
field(t("models.form.secret"), secretIn)
|
|
||||||
);
|
);
|
||||||
|
|
||||||
const getValue = () => ({
|
const getValue = () => {
|
||||||
model: {
|
const typedKey = keyIn.value.trim();
|
||||||
id: existing ? m.id : idIn.value.trim(),
|
const hasKey = !!typedKey || !!(existing && existing.api_key_env);
|
||||||
provider: providerSel.value,
|
const keyEnv = envFor();
|
||||||
base_url: baseUrlIn.value.trim(),
|
return {
|
||||||
model: modelIn.value.trim(),
|
model: {
|
||||||
kind: kindSel.value,
|
id: existing ? m.id : idIn.value.trim(),
|
||||||
cost_tier: tierSel.value,
|
provider: providerSel.value,
|
||||||
price_in: parseFloat(priceInIn.value) || 0,
|
base_url: baseUrlIn.value.trim(),
|
||||||
price_out: parseFloat(priceOutIn.value) || 0,
|
model: modelIn.value.trim(),
|
||||||
api_key_env: keyEnvIn.value.trim() || null,
|
kind: kindSel.value,
|
||||||
caps: capInputs.filter(({ cb }) => cb.checked).map(({ c }) => c),
|
cost_tier: tierSel.value,
|
||||||
},
|
price_in: parseFloat(priceInIn.value) || 0,
|
||||||
secret: secretIn.value,
|
price_out: parseFloat(priceOutIn.value) || 0,
|
||||||
keyEnv: keyEnvIn.value.trim(),
|
api_key_env: hasKey ? keyEnv : null,
|
||||||
});
|
caps: capInputs.filter(({ cb }) => cb.checked).map(({ c }) => c),
|
||||||
|
},
|
||||||
|
secret: typedKey,
|
||||||
|
keyEnv,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
return { node, getValue };
|
return { node, getValue };
|
||||||
}
|
}
|
||||||
@@ -70,12 +160,14 @@ async function openModal(meta, existing, reload) {
|
|||||||
const { node, getValue } = modelForm(meta, existing);
|
const { node, getValue } = modelForm(meta, existing);
|
||||||
modal(existing ? t("models.form.editTitle") : t("models.form.addTitle"), node, async () => {
|
modal(existing ? t("models.form.editTitle") : t("models.form.addTitle"), node, async () => {
|
||||||
const { model, secret, keyEnv } = getValue();
|
const { model, secret, keyEnv } = getValue();
|
||||||
|
// Forgot the id? Derive it from the model name so saving never errors on a
|
||||||
|
// blank id.
|
||||||
|
if (!model.id) model.id = slugId(model.model);
|
||||||
if (!model.id) {
|
if (!model.id) {
|
||||||
toast(t("models.form.id") + "?", "bad");
|
toast(t("models.form.idOrModel"), "bad");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
// strip null api_key_env so serde keeps it optional
|
if (!model.api_key_env) delete model.api_key_env; // keep serde field optional
|
||||||
if (!model.api_key_env) delete model.api_key_env;
|
|
||||||
try {
|
try {
|
||||||
if (existing) await api.updateModel(existing.id, model);
|
if (existing) await api.updateModel(existing.id, model);
|
||||||
else await api.createModel(model);
|
else await api.createModel(model);
|
||||||
@@ -188,15 +280,33 @@ function row(m, meta, reload) {
|
|||||||
"button",
|
"button",
|
||||||
{
|
{
|
||||||
class: "btn sm danger",
|
class: "btn sm danger",
|
||||||
onClick: async () => {
|
onClick: () => {
|
||||||
if (!(await confirmDialog(t("models.deleteConfirm", { id: m.id })))) return;
|
// local models may have a .cmf on disk — let the user decide its fate
|
||||||
try {
|
const isLocal = m.managed || /127\.0\.0\.1|localhost/.test(m.base_url || "");
|
||||||
await api.deleteModel(m.id);
|
// unchecked by default: deleting a pool entry should not silently
|
||||||
toast(t("toast.deleted"), "good");
|
// cost the user a multi-GB re-download/conversion
|
||||||
reload();
|
const fileCb = h("input", { type: "checkbox" });
|
||||||
} catch (e) {
|
const body = h(
|
||||||
toast(String(e.message || e), "bad");
|
"div",
|
||||||
}
|
{},
|
||||||
|
h("p", {}, t("models.deleteConfirm", { id: m.id })),
|
||||||
|
isLocal
|
||||||
|
? h("label", { class: "check" }, fileCb, t("models.deleteFile"))
|
||||||
|
: null
|
||||||
|
);
|
||||||
|
modal(t("common.delete") + ": " + m.id, body, async () => {
|
||||||
|
try {
|
||||||
|
const r = await api.deleteModel(m.id, isLocal && fileCb.checked);
|
||||||
|
toast(
|
||||||
|
r.file_removed ? t("models.deletedWithFile") : t("toast.deleted"),
|
||||||
|
"good"
|
||||||
|
);
|
||||||
|
await reload();
|
||||||
|
} catch (e) {
|
||||||
|
toast(String(e.message || e), "bad");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}, t("common.delete"));
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
t("common.delete")
|
t("common.delete")
|
||||||
|
|||||||
Vendored
+43
-8
@@ -214,6 +214,9 @@ export async function renderSettings() {
|
|||||||
? [{ id: s.cmf.model_id || "cmf-local", model: s.cmf.local_model, port: s.cmf.local_port ?? 8081, threads: s.cmf.threads ?? 8, gpu: !!s.cmf.gpu }]
|
? [{ id: s.cmf.model_id || "cmf-local", model: s.cmf.local_model, port: s.cmf.local_port ?? 8081, threads: s.cmf.threads ?? 8, gpu: !!s.cmf.gpu }]
|
||||||
: [];
|
: [];
|
||||||
const initServers = (s.cmf?.servers && s.cmf.servers.length) ? s.cmf.servers : legacyServers;
|
const initServers = (s.cmf?.servers && s.cmf.servers.length) ? s.cmf.servers : legacyServers;
|
||||||
|
// available .cmf files under models_dir → offer as a dropdown, not a typed path
|
||||||
|
const cmfFiles = ((await api.cmfFiles().catch(() => ({ files: [] }))).files) || [];
|
||||||
|
const fileLabel = (f) => f.name + (f.size ? ` (${(f.size / 1073741824).toFixed(f.size >= 1073741824 ? 1 : 2)} GB)` : f.is_dir ? " (dir)" : "");
|
||||||
function autoId(path) {
|
function autoId(path) {
|
||||||
const base = (path || "").split("/").pop().replace(/\.cmf$/i, "");
|
const base = (path || "").split("/").pop().replace(/\.cmf$/i, "");
|
||||||
return base ? "cmf-" + base.replace(/[^a-zA-Z0-9_-]/g, "-") : "";
|
return base ? "cmf-" + base.replace(/[^a-zA-Z0-9_-]/g, "-") : "";
|
||||||
@@ -221,14 +224,34 @@ export async function renderSettings() {
|
|||||||
function makeServerRow(sv) {
|
function makeServerRow(sv) {
|
||||||
sv = sv || { id: "", model: "", port: 8090, threads: 8, gpu: false };
|
sv = sv || { id: "", model: "", port: 8090, threads: 8, gpu: false };
|
||||||
const idIn = h("input", { value: sv.id || "", placeholder: "cmf-id", style: "width:130px" });
|
const idIn = h("input", { value: sv.id || "", placeholder: "cmf-id", style: "width:130px" });
|
||||||
const modelIn = h("input", { value: sv.model || "", placeholder: "models/model.cmf", style: "flex:2;min-width:150px" });
|
// pick from existing .cmf files rather than typing a path
|
||||||
|
const modelSel = h("select", { style: "flex:2;min-width:150px" });
|
||||||
|
const seen = new Set();
|
||||||
|
modelSel.appendChild(h("option", { value: "" }, "— " + t("settings.cmf.pick") + " —"));
|
||||||
|
cmfFiles.forEach((f) => { modelSel.appendChild(h("option", { value: f.path }, fileLabel(f))); seen.add(f.path); });
|
||||||
|
if (sv.model && !seen.has(sv.model)) modelSel.appendChild(h("option", { value: sv.model }, sv.model));
|
||||||
|
modelSel.value = sv.model || "";
|
||||||
const portIn = h("input", { type: "number", value: sv.port ?? 8090, style: "width:82px" });
|
const portIn = h("input", { type: "number", value: sv.port ?? 8090, style: "width:82px" });
|
||||||
const threadsIn = h("input", { type: "number", min: 0, value: sv.threads ?? 8, style: "width:60px", title: t("settings.cmf.threadsHelp") });
|
const threadsIn = h("input", { type: "number", min: 0, value: sv.threads ?? 8, style: "width:60px", title: t("settings.cmf.threadsHelp") });
|
||||||
const gpuIn = h("input", { type: "checkbox", checked: sv.gpu ? true : null, title: t("settings.cmf.gpuHelp") });
|
const gpuIn = h("input", { type: "checkbox", checked: sv.gpu ? true : null, title: t("settings.cmf.gpuHelp") });
|
||||||
|
// serve-time O(1) override: off cures a too-aggressive --o1 conversion
|
||||||
|
const o1Opts = ["", "off", "all", "deep4", "deep8", "deep12"];
|
||||||
|
const o1In = h("select", { style: "width:110px", title: t("settings.cmf.o1Help") },
|
||||||
|
...o1Opts.map((v) => h("option", { value: v, selected: (sv.o1 || "") === v ? true : null },
|
||||||
|
v === "" ? t("settings.cmf.o1Auto") : v)));
|
||||||
|
if (sv.o1 && !o1Opts.includes(sv.o1)) o1In.appendChild(h("option", { value: sv.o1, selected: true }, sv.o1));
|
||||||
|
// reasoning switch for thinking models (off → enable_thinking=false)
|
||||||
|
const thinkIn = h("select", { style: "width:118px", title: t("settings.cmf.thinkHelp") },
|
||||||
|
h("option", { value: "", selected: !sv.thinking ? true : null }, t("settings.cmf.thinkAuto")),
|
||||||
|
h("option", { value: "off", selected: sv.thinking === "off" ? true : null }, t("settings.cmf.thinkOff")));
|
||||||
|
// per-model generation defaults (applied when the request doesn't set them)
|
||||||
|
const tempIn = h("input", { type: "number", step: "0.1", min: 0, max: 2, value: sv.temperature ?? "", placeholder: "T° 0.7", style: "width:74px", title: t("settings.cmf.tempHelp") });
|
||||||
|
const topPIn = h("input", { type: "number", step: "0.05", min: 0, max: 1, value: sv.top_p ?? "", placeholder: "top-p", style: "width:70px", title: t("settings.cmf.topPHelp") });
|
||||||
|
const maxTokIn = h("input", { type: "number", min: 1, value: sv.max_tokens ?? "", placeholder: "max tok", style: "width:82px", title: t("settings.cmf.maxTokHelp") });
|
||||||
const checkBtn = h("button", { class: "btn small", type: "button" }, t("settings.cmf.checkPort"));
|
const checkBtn = h("button", { class: "btn small", type: "button" }, t("settings.cmf.checkPort"));
|
||||||
const removeBtn = h("button", { class: "btn small danger", type: "button", title: t("common.delete") }, "✕");
|
const removeBtn = h("button", { class: "btn small danger", type: "button", title: t("common.delete") }, "✕");
|
||||||
const st = h("span", { class: "small" });
|
const st = h("span", { class: "small" });
|
||||||
modelIn.addEventListener("input", () => { if (!idIn.value || idIn._auto) { idIn.value = autoId(modelIn.value); idIn._auto = true; } });
|
modelSel.addEventListener("change", () => { if (!idIn.value || idIn._auto) { idIn.value = autoId(modelSel.value); idIn._auto = true; } });
|
||||||
idIn.addEventListener("input", () => { idIn._auto = false; });
|
idIn.addEventListener("input", () => { idIn._auto = false; });
|
||||||
checkBtn.addEventListener("click", async () => {
|
checkBtn.addEventListener("click", async () => {
|
||||||
const p = parseInt(portIn.value, 10); if (!p) return;
|
const p = parseInt(portIn.value, 10); if (!p) return;
|
||||||
@@ -236,7 +259,11 @@ export async function renderSettings() {
|
|||||||
try {
|
try {
|
||||||
const r = await api.cmfPortCheck(p);
|
const r = await api.cmfPortCheck(p);
|
||||||
if (r.available) mount(st, h("span", { style: "color:var(--good)" }, "✓ " + t("settings.cmf.portFree")));
|
if (r.available) mount(st, h("span", { style: "color:var(--good)" }, "✓ " + t("settings.cmf.portFree")));
|
||||||
else {
|
else if (r.managed) {
|
||||||
|
// held by this gateway's own managed server — replacing the model on
|
||||||
|
// this port is fine, it frees up on restart
|
||||||
|
mount(st, h("span", { style: "color:var(--accent)" }, "ℹ " + t("settings.cmf.portOwn")));
|
||||||
|
} else {
|
||||||
const kids = [h("span", { style: "color:var(--bad)" }, "✗ " + t("settings.cmf.portBusy"))];
|
const kids = [h("span", { style: "color:var(--bad)" }, "✗ " + t("settings.cmf.portBusy"))];
|
||||||
if (r.suggested) { const b = h("button", { class: "btn small", type: "button" }, t("settings.cmf.useSuggested", { port: r.suggested })); b.addEventListener("click", () => { portIn.value = r.suggested; mount(st, ""); }); kids.push(b); }
|
if (r.suggested) { const b = h("button", { class: "btn small", type: "button" }, t("settings.cmf.useSuggested", { port: r.suggested })); b.addEventListener("click", () => { portIn.value = r.suggested; mount(st, ""); }); kids.push(b); }
|
||||||
mount(st, h("span", {}, ...kids));
|
mount(st, h("span", {}, ...kids));
|
||||||
@@ -245,8 +272,14 @@ export async function renderSettings() {
|
|||||||
checkBtn.disabled = false;
|
checkBtn.disabled = false;
|
||||||
});
|
});
|
||||||
const row = h("div", { class: "flex wrap", style: "gap:6px;align-items:center;margin:6px 0" },
|
const row = h("div", { class: "flex wrap", style: "gap:6px;align-items:center;margin:6px 0" },
|
||||||
idIn, modelIn, portIn, checkBtn, threadsIn, h("label", { class: "flex", style: "gap:4px" }, gpuIn, "GPU"), removeBtn, st);
|
idIn, modelSel, portIn, checkBtn, threadsIn, h("label", { class: "flex", style: "gap:4px" }, gpuIn, "GPU"), o1In, thinkIn, tempIn, topPIn, maxTokIn, removeBtn, st);
|
||||||
const entry = { node: row, read: () => ({ id: idIn.value.trim() || autoId(modelIn.value), model: modelIn.value.trim(), port: parseInt(portIn.value) || 8090, threads: parseInt(threadsIn.value) || 0, gpu: gpuIn.checked }) };
|
const entry = { node: row, read: () => {
|
||||||
|
const v = { id: idIn.value.trim() || autoId(modelSel.value), model: modelSel.value.trim(), port: parseInt(portIn.value) || 8090, threads: parseInt(threadsIn.value) || 0, gpu: gpuIn.checked, o1: o1In.value, thinking: thinkIn.value };
|
||||||
|
if (tempIn.value !== "") v.temperature = parseFloat(tempIn.value);
|
||||||
|
if (topPIn.value !== "") v.top_p = parseFloat(topPIn.value);
|
||||||
|
if (maxTokIn.value !== "") v.max_tokens = parseInt(maxTokIn.value);
|
||||||
|
return v;
|
||||||
|
} };
|
||||||
removeBtn.addEventListener("click", () => { const i = serverRows.indexOf(entry); if (i >= 0) serverRows.splice(i, 1); row.remove(); });
|
removeBtn.addEventListener("click", () => { const i = serverRows.indexOf(entry); if (i >= 0) serverRows.splice(i, 1); row.remove(); });
|
||||||
serverRows.push(entry);
|
serverRows.push(entry);
|
||||||
serversWrap.appendChild(row);
|
serversWrap.appendChild(row);
|
||||||
@@ -433,9 +466,11 @@ export async function renderSettings() {
|
|||||||
"div",
|
"div",
|
||||||
{ class: "card" },
|
{ class: "card" },
|
||||||
h("div", { class: "card-head" }, h("h3", {}, t("settings.route"))),
|
h("div", { class: "card-head" }, h("h3", {}, t("settings.route"))),
|
||||||
field(t("settings.route.strategy"), strategySel),
|
field(t("settings.route.strategy"), strategySel, null, t("settings.route.strategyHelp")),
|
||||||
h("div", { class: "row" }, field(t("settings.route.maxChars"), maxChars), field(t("settings.route.cacheTtl"), cacheTtl)),
|
h("div", { class: "row" },
|
||||||
field(t("settings.route.profile"), profileSel)
|
field(t("settings.route.maxChars"), maxChars, null, t("settings.route.maxCharsHelp")),
|
||||||
|
field(t("settings.route.cacheTtl"), cacheTtl, null, t("settings.route.cacheTtlHelp"))),
|
||||||
|
field(t("settings.route.profile"), profileSel, null, t("settings.route.profileHelp"))
|
||||||
),
|
),
|
||||||
h(
|
h(
|
||||||
"div",
|
"div",
|
||||||
|
|||||||
Reference in New Issue
Block a user