feat(cmf): integrate CMF as a managed local model server
Run models in the CMF (.cmf) format locally, via the native `cortiq` CLI from crates.io — no Python and no external model server. - cmf_runtime: install/update `cortiq-cli` from crates.io, spawn and health-check `cortiq serve` as a managed local backend. - import: model-import jobs shell out to the native `cortiq convert` (HF repo -> .cmf, quant mapped Q8_2F->q8_2f / Q8_ROW->q8 / Q4->q4); the Python converter is used only for advanced/exotic options. - pipeline: local-first routing — serve from the local CMF model when the router is disabled/unreachable, with a "local only" hard switch. - registry/state: register the local CMF server as an OpenAI provider and track local candidates. - config: data-dir resolution + CmfCfg (auto-install/update, managed server, host/port/model) and relaxed validation for the managed model. - admin + settings UI (en/ru): "Local models (CMF)" settings card.
This commit is contained in:
+60
-6
@@ -8,7 +8,7 @@
|
||||
use crate::cache::SemanticCache;
|
||||
use crate::config::Config;
|
||||
use crate::registry::Registry;
|
||||
use crate::router_client::RouterClient;
|
||||
use crate::router_client::{RouterClient, RouterLastStatus};
|
||||
use crate::routing::RoutingTable;
|
||||
use crate::secrets::SecretStore;
|
||||
use crate::stats::Stats;
|
||||
@@ -27,8 +27,12 @@ pub struct Live {
|
||||
}
|
||||
|
||||
impl Live {
|
||||
pub fn build(cfg: Config, secrets: &SecretStore) -> anyhow::Result<Self> {
|
||||
let router = RouterClient::new(&cfg.router)?;
|
||||
pub fn build(
|
||||
cfg: Config,
|
||||
secrets: &SecretStore,
|
||||
router_status: RouterLastStatus,
|
||||
) -> anyhow::Result<Self> {
|
||||
let router = RouterClient::new(&cfg.router, secrets, router_status)?;
|
||||
let registry = Registry::from_config(&cfg, secrets)?;
|
||||
let routing = RoutingTable::from_config(&cfg);
|
||||
Ok(Self {
|
||||
@@ -38,15 +42,57 @@ impl Live {
|
||||
router,
|
||||
})
|
||||
}
|
||||
|
||||
/// Local-first fallback targets, in order: the managed local CMF model (if
|
||||
/// configured), then the routing default, then any registered model — only
|
||||
/// registered ids are returned. Used whenever the router is bypassed
|
||||
/// (local-only / disabled) or unavailable, so CMF models serve with no router.
|
||||
pub fn local_candidates(&self) -> Vec<String> {
|
||||
let mut v = Vec::new();
|
||||
if self.cfg.cmf.manage_server && !self.cfg.cmf.local_model.trim().is_empty() {
|
||||
v.push(self.cfg.cmf.model_id.clone());
|
||||
}
|
||||
let def = self.routing.default_model().to_string();
|
||||
if !def.is_empty() && !v.contains(&def) {
|
||||
v.push(def);
|
||||
}
|
||||
v.retain(|id| self.registry.get(id).is_some());
|
||||
if v.is_empty() {
|
||||
if let Some(a) = self.registry.any_id() {
|
||||
v.push(a);
|
||||
}
|
||||
}
|
||||
v
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AppState {
|
||||
pub live: ArcSwap<Live>,
|
||||
pub stats: Arc<Stats>,
|
||||
pub promotion: Arc<crate::promotion::Promotion>,
|
||||
pub imports: Arc<crate::import::JobStore>,
|
||||
/// Managed local CMF model server (install/update/spawn lifecycle + status).
|
||||
pub cmf: Arc<crate::cmf_runtime::CmfRuntime>,
|
||||
pub cache: Arc<SemanticCache>,
|
||||
pub secrets: SecretStore,
|
||||
pub config_path: String,
|
||||
pub pipeline: crate::pipeline::Pipeline,
|
||||
/// Outcome of the most recent router call (survives config reloads) —
|
||||
/// lets the admin panel distinguish a bad/expired key from a down router.
|
||||
pub router_status: RouterLastStatus,
|
||||
}
|
||||
|
||||
/// Map the user-facing `[shadow]` config to the promotion-table tuning.
|
||||
fn promotion_cfg(s: &crate::config::ShadowCfg) -> crate::promotion::PromotionCfg {
|
||||
crate::promotion::PromotionCfg {
|
||||
enabled: s.enabled && !s.local_model_id.is_empty(),
|
||||
window: s.window,
|
||||
n_min: s.n_min,
|
||||
promote_lb: s.promote_lb,
|
||||
soak: s.soak,
|
||||
file: if s.file.is_empty() { None } else { Some(s.file.clone()) },
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::ops::Deref for SharedState {
|
||||
@@ -61,15 +107,23 @@ impl SharedState {
|
||||
let secrets_path = sibling(&config_path, "secrets.toml");
|
||||
let secrets = SecretStore::load(&secrets_path);
|
||||
let stats = Stats::new(&cfg.stats);
|
||||
let promotion = crate::promotion::Promotion::new(promotion_cfg(&cfg.shadow));
|
||||
let imports = crate::import::JobStore::new();
|
||||
let cmf = crate::cmf_runtime::CmfRuntime::new();
|
||||
let cache = SemanticCache::new(&cfg.cache);
|
||||
let live = Live::build(cfg, &secrets)?;
|
||||
let router_status = RouterLastStatus::default();
|
||||
let live = Live::build(cfg, &secrets, router_status.clone())?;
|
||||
Ok(SharedState(Arc::new(AppState {
|
||||
live: ArcSwap::from_pointee(live),
|
||||
stats,
|
||||
promotion,
|
||||
imports,
|
||||
cmf,
|
||||
cache,
|
||||
secrets,
|
||||
config_path,
|
||||
pipeline: crate::pipeline::Pipeline::new(),
|
||||
router_status,
|
||||
})))
|
||||
}
|
||||
|
||||
@@ -84,7 +138,7 @@ impl AppState {
|
||||
/// If any step fails, the active state and file are left unchanged.
|
||||
pub fn reload(&self, new_cfg: Config) -> anyhow::Result<()> {
|
||||
new_cfg.validate()?;
|
||||
let live = Live::build(new_cfg.clone(), &self.secrets)?;
|
||||
let live = Live::build(new_cfg.clone(), &self.secrets, self.router_status.clone())?;
|
||||
new_cfg.save(&self.config_path)?;
|
||||
self.live.store(Arc::new(live));
|
||||
Ok(())
|
||||
@@ -94,7 +148,7 @@ impl AppState {
|
||||
/// (e.g. after a secret change — only provider keys changed).
|
||||
pub fn rebuild(&self) -> anyhow::Result<()> {
|
||||
let cfg = self.live.load_full().cfg.clone();
|
||||
let live = Live::build(cfg, &self.secrets)?;
|
||||
let live = Live::build(cfg, &self.secrets, self.router_status.clone())?;
|
||||
self.live.store(Arc::new(live));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user