From 5759de153b7f65fb8ddbdab22edb5facf28fe632 Mon Sep 17 00:00:00 2001 From: infosave2007 Date: Tue, 30 Jun 2026 17:04:53 +0300 Subject: [PATCH] feat: semantic (embedding-based) response cache - cache.rs: cosine-similarity store (threshold, TTL, ring buffer), partitioned by routing signature; tracks hit rate and model cost saved - pipeline: embed routing text, look up before the model call (route_source=cache on hit), insert successful responses; stats record cache hits - config [cache] + admin settings card; dashboard shows hit-rate and savings - embedding model resolved from the pool (or first kind="embedding") --- CHANGELOG.md | 4 + config/gateway.example.toml | 12 +++ src/admin/mod.rs | 15 +++- src/cache.rs | 171 ++++++++++++++++++++++++++++++++++++ src/config.rs | 42 +++++++++ src/main.rs | 1 + src/pipeline.rs | 126 +++++++++++++++++++++++++- src/state.rs | 4 + web/js/i18n/de.js | 11 +++ web/js/i18n/en.js | 11 +++ web/js/i18n/es.js | 11 +++ web/js/i18n/fr.js | 11 +++ web/js/i18n/ru.js | 11 +++ web/js/i18n/tr.js | 11 +++ web/js/i18n/zh.js | 11 +++ web/js/views/dashboard.js | 16 ++++ web/js/views/settings.js | 29 ++++++ 17 files changed, 492 insertions(+), 5 deletions(-) create mode 100644 src/cache.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index b94327d..73f5509 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/). SSE), so Claude-native clients can target the gateway. - **Embeddings**: `POST /v1/embeddings` inbound + provider `embed()`; embedding models are now resolvable from the pool. +- **Semantic cache**: embedding-based cache that returns a stored answer for prompts + semantically near a previous one (cosine threshold, TTL, ring buffer), skipping the + model call. Partitioned by routing signature; hit-rate and cost saved are tracked and + shown on the dashboard; configurable from Settings (`[cache]`). ## [0.1.0] - 2026-06-30 diff --git a/config/gateway.example.toml b/config/gateway.example.toml index 776a7b0..dfaf13d 100644 --- a/config/gateway.example.toml +++ b/config/gateway.example.toml @@ -150,3 +150,15 @@ enabled = true file = "config/stats.jsonl" # empty = in-memory only ring_size = 500 # how many recent requests to keep in the feed retention = "7d" # aggregate/series depth (replayed on startup) + +# ============================================================================ +# Semantic cache — returns a cached answer for prompts semantically near a +# previous one, skipping the model call. Requires an embedding model in the pool. +# Disabled by default. (Cache settings take effect on restart.) +# ============================================================================ +[cache] +enabled = false +threshold = 0.92 # cosine similarity for a hit (0..1) +ttl = "1h" +max_entries = 1000 +# embed_model = "local-embed" # defaults to the first kind = "embedding" model diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 3e2adf5..f1ca96c 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -8,8 +8,8 @@ pub mod assets; use crate::config::{ - AdminCfg, ApiKeyCfg, BreakerCfg, Config, CortiqCfg, LogCfg, ModelCfg, ProtocolsCfg, RouteCfg, - RouterCfg, RoutingCfg, RoutingPolicy, StatsCfg, TelemetryCfg, TierTargets, + AdminCfg, ApiKeyCfg, BreakerCfg, CacheCfg, Config, CortiqCfg, LogCfg, ModelCfg, ProtocolsCfg, + RouteCfg, RouterCfg, RoutingCfg, RoutingPolicy, StatsCfg, TelemetryCfg, TierTargets, }; use crate::model::{ChatRequest, GenParams, Message, RequestMeta, RoutingDirective}; use crate::state::SharedState; @@ -477,6 +477,7 @@ async fn get_settings(State(state): State) -> ApiResult { "cortiq": c.cortiq, "stats": c.stats, "admin": c.admin, + "cache": c.cache, })) } @@ -491,6 +492,7 @@ struct SettingsBody { cortiq: Option, stats: Option, admin: Option, + cache: Option, } async fn put_settings(State(state): State, Json(b): Json) -> ApiResult { @@ -529,6 +531,11 @@ async fn put_settings(State(state): State, Json(b): Json, Query(q): Query .and_then(parse_duration_secs) .unwrap_or(24 * 3600); let groupby = q.groupby.as_deref().unwrap_or("model"); - ok(state.stats.snapshot(range_secs, groupby)) + let mut snap = state.stats.snapshot(range_secs, groupby); + snap["cache"] = state.cache.snapshot(); + ok(snap) } #[derive(Deserialize)] diff --git a/src/cache.rs b/src/cache.rs new file mode 100644 index 0000000..ce444d1 --- /dev/null +++ b/src/cache.rs @@ -0,0 +1,171 @@ +//! Semantic (embedding-based) response cache. +//! +//! Each entry stores an L2-normalized embedding of the request plus the response. +//! A lookup returns a cached answer when the cosine similarity to a previous request +//! (with the same routing signature, within TTL) meets the configured threshold — +//! skipping the expensive model call. In-memory ring buffer; counters track hit +//! rate and the model cost saved. + +use crate::config::CacheCfg; +use crate::stats::parse_duration_secs; +use std::collections::VecDeque; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{SystemTime, UNIX_EPOCH}; + +fn now_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +struct Entry { + sig: String, + vec: Vec, // L2-normalized + ts: u64, + model_used: String, + content: String, + prompt_tokens: u32, + completion_tokens: u32, + cost: f64, +} + +pub struct CacheHit { + pub model_used: String, + pub content: String, + pub prompt_tokens: u32, + pub completion_tokens: u32, + pub similarity: f32, +} + +pub struct SemanticCache { + enabled: bool, + threshold: f32, + ttl_secs: u64, + max_entries: usize, + embed_model: Option, + entries: Mutex>, + hits: AtomicU64, + misses: AtomicU64, + saved_micro_usd: AtomicU64, +} + +impl SemanticCache { + pub fn new(cfg: &CacheCfg) -> Arc { + Arc::new(Self { + enabled: cfg.enabled, + threshold: cfg.threshold, + ttl_secs: parse_duration_secs(&cfg.ttl).unwrap_or(3600), + max_entries: cfg.max_entries.max(1), + embed_model: cfg.embed_model.clone(), + entries: Mutex::new(VecDeque::new()), + hits: AtomicU64::new(0), + misses: AtomicU64::new(0), + saved_micro_usd: AtomicU64::new(0), + }) + } + + pub fn enabled(&self) -> bool { + self.enabled + } + pub fn embed_model(&self) -> Option<&str> { + self.embed_model.as_deref() + } + + /// Look up a cached response by routing signature + query embedding. + pub fn lookup(&self, sig: &str, query: &[f32]) -> Option { + if !self.enabled { + return None; + } + let q = normalized(query); + let cutoff = now_secs().saturating_sub(self.ttl_secs); + let inner = self.entries.lock().unwrap(); + let mut best: Option<(f32, &Entry)> = None; + for e in inner.iter() { + if e.sig != sig || e.ts < cutoff { + continue; + } + let sim = dot(&e.vec, &q); + if sim >= self.threshold && best.map(|(b, _)| sim > b).unwrap_or(true) { + best = Some((sim, e)); + } + } + match best { + Some((sim, e)) => { + self.hits.fetch_add(1, Ordering::Relaxed); + self.saved_micro_usd + .fetch_add((e.cost * 1_000_000.0) as u64, Ordering::Relaxed); + Some(CacheHit { + model_used: e.model_used.clone(), + content: e.content.clone(), + prompt_tokens: e.prompt_tokens, + completion_tokens: e.completion_tokens, + similarity: sim, + }) + } + None => { + self.misses.fetch_add(1, Ordering::Relaxed); + None + } + } + } + + /// Store a response under its routing signature + query embedding. + #[allow(clippy::too_many_arguments)] + pub fn insert( + &self, + sig: String, + query: Vec, + model_used: String, + content: String, + prompt_tokens: u32, + completion_tokens: u32, + cost: f64, + ) { + if !self.enabled || query.is_empty() { + return; + } + let vec = normalized(&query); + let mut inner = self.entries.lock().unwrap(); + inner.push_back(Entry { + sig, + vec, + ts: now_secs(), + model_used, + content, + prompt_tokens, + completion_tokens, + cost, + }); + while inner.len() > self.max_entries { + inner.pop_front(); + } + } + + pub fn snapshot(&self) -> serde_json::Value { + let h = self.hits.load(Ordering::Relaxed); + let m = self.misses.load(Ordering::Relaxed); + serde_json::json!({ + "enabled": self.enabled, + "hits": h, + "misses": m, + "hit_rate": if h + m > 0 { h as f64 / (h + m) as f64 } else { 0.0 }, + "entries": self.entries.lock().unwrap().len(), + "saved_usd": self.saved_micro_usd.load(Ordering::Relaxed) as f64 / 1_000_000.0, + }) + } +} + +fn dot(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b.iter()).map(|(x, y)| x * y).sum() +} + +fn normalized(v: &[f32]) -> Vec { + let n = v.iter().map(|x| x * x).sum::().sqrt(); + if n > 0.0 { + v.iter().map(|x| x / n).collect() + } else { + v.to_vec() + } +} diff --git a/src/config.rs b/src/config.rs index a10ff34..71397e5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -31,6 +31,8 @@ pub struct Config { pub admin: AdminCfg, #[serde(default)] pub stats: StatsCfg, + #[serde(default)] + pub cache: CacheCfg, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -320,6 +322,46 @@ fn default_stats_retention() -> String { "7d".into() } +/// Semantic (embedding-based) response cache — returns a cached answer for prompts +/// that are semantically near a previous one, skipping the (expensive) model call. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CacheCfg { + #[serde(default)] + pub enabled: bool, + /// Cosine-similarity threshold for a hit (0..1). + #[serde(default = "default_semcache_threshold")] + pub threshold: f32, + /// Entry time-to-live, e.g. `1h`. + #[serde(default = "default_semcache_ttl")] + pub ttl: String, + /// Maximum number of cached entries (ring buffer). + #[serde(default = "default_semcache_max")] + pub max_entries: usize, + /// Embedding model id (defaults to the first `kind = "embedding"` model). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub embed_model: Option, +} +impl Default for CacheCfg { + fn default() -> Self { + Self { + enabled: false, + threshold: default_semcache_threshold(), + ttl: default_semcache_ttl(), + max_entries: default_semcache_max(), + embed_model: None, + } + } +} +fn default_semcache_threshold() -> f32 { + 0.92 +} +fn default_semcache_ttl() -> String { + "1h".into() +} +fn default_semcache_max() -> usize { + 1000 +} + impl Config { /// Load and validate the config from a file. pub fn load(path: &str) -> anyhow::Result { diff --git a/src/main.rs b/src/main.rs index 9b2ceaa..3eede89 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,7 @@ #![allow(dead_code)] mod admin; +mod cache; mod config; mod error; mod model; diff --git a/src/pipeline.rs b/src/pipeline.rs index 00e757c..232efa7 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -4,9 +4,9 @@ use crate::error::{GatewayError, Result}; use crate::model::{ - ChatRequest, ChatResponse, Message, RouteDecision, RouteInfo, RoutingDirective, + ChatRequest, ChatResponse, Choice, Message, RouteDecision, RouteInfo, RoutingDirective, Usage, }; -use crate::state::SharedState; +use crate::state::{Live, SharedState}; use crate::stats::RequestRecord; use std::time::{Instant, SystemTime, UNIX_EPOCH}; @@ -28,6 +28,45 @@ impl Pipeline { } .to_string(); + // --- semantic cache lookup (short-circuit before the model call) --- + let cache = &state.cache; + let mut cache_key: Option<(String, Vec)> = None; + if cache.enabled() { + let live = state.live(); + let sig = cache_sig(&req); + let text = self.extract_text(&req.messages, "concat_all", live.cfg.route.max_chars); + let embed_model = cache + .embed_model() + .map(|s| s.to_string()) + .or_else(|| first_embedding_model(&live)); + if let Some(model_id) = embed_model { + if let Some(vec) = embed(&live, &model_id, &text).await { + if let Some(hit) = cache.lookup(&sig, &vec) { + state.stats.record(RequestRecord { + ts: now_secs(), + account, + protocol, + directive, + task_label: "cache".into(), + tier: "cache".into(), + score: 0.0, + model_id: hit.model_used.clone(), + route_source: "cache".into(), + prompt_tokens: hit.prompt_tokens, + completion_tokens: hit.completion_tokens, + cost_usd: 0.0, + latency_ms: started.elapsed().as_millis() as u64, + outcome: "ok".into(), + failover: false, + error: None, + }); + return Ok(cached_response(&hit)); + } + cache_key = Some((sig, vec)); + } + } + } + let result = self.run_inner(req, state).await; // record statistics (for both success and error) @@ -76,6 +115,26 @@ impl Pipeline { }; state.stats.record(rec); + // --- store a successful response in the semantic cache --- + if let (Ok(resp), Some((sig, vec))) = (&result, cache_key) { + let content = resp + .choices + .first() + .map(|c| c.message.content.clone()) + .unwrap_or_default(); + if !content.is_empty() { + state.cache.insert( + sig, + vec, + resp.cortiq.selected_model.clone(), + content, + resp.usage.prompt_tokens, + resp.usage.completion_tokens, + resp.cortiq.cost_usd, + ); + } + } + result } @@ -350,6 +409,69 @@ fn now_secs() -> u64 { .as_secs() } +/// Routing signature used to partition cache entries (so a pinned model's cached +/// answer is never returned for a different model/profile). +fn cache_sig(req: &ChatRequest) -> String { + match &req.routing { + RoutingDirective::Auto { profile } => { + format!("auto:{}", profile.as_deref().unwrap_or("")) + } + RoutingDirective::Pinned { model_id } => format!("pinned:{model_id}"), + } +} + +fn first_embedding_model(live: &Live) -> Option { + live.cfg + .models + .iter() + .find(|m| m.kind == "embedding") + .map(|m| m.id.clone()) +} + +/// Embed `text` with the given model; returns the embedding vector (or None on error). +async fn embed(live: &Live, model_id: &str, text: &str) -> Option> { + let provider = live.registry.get(model_id)?; + let body = provider.embed(serde_json::json!(text)).await.ok()?; + let arr = body["data"][0]["embedding"].as_array()?; + Some( + arr.iter() + .filter_map(|x| x.as_f64()) + .map(|x| x as f32) + .collect(), + ) +} + +fn cached_response(hit: &crate::cache::CacheHit) -> ChatResponse { + ChatResponse { + id: "cache".to_string(), + model_used: hit.model_used.clone(), + choices: vec![Choice { + index: 0, + message: Message { + role: "assistant".into(), + content: hit.content.clone(), + tool_calls: vec![], + }, + finish_reason: "stop".into(), + }], + usage: Usage { + prompt_tokens: hit.prompt_tokens, + completion_tokens: hit.completion_tokens, + total_tokens: hit.prompt_tokens + hit.completion_tokens, + }, + cortiq: RouteInfo { + task_label: "cache".into(), + complexity_score: 0.0, + complexity_tier: "cache".into(), + selected_model: hit.model_used.clone(), + route_source: "cache".into(), + router_request_id: None, + cost_usd: 0.0, + failover: false, + }, + } +} + fn record_error( state: &SharedState, account: &str, diff --git a/src/state.rs b/src/state.rs index 37b8efb..c76cb9b 100644 --- a/src/state.rs +++ b/src/state.rs @@ -5,6 +5,7 @@ //! a restart: the admin API validates the new config, builds a new `Live`, //! atomically writes the TOML, and swaps the pointer. +use crate::cache::SemanticCache; use crate::config::Config; use crate::registry::Registry; use crate::router_client::RouterClient; @@ -42,6 +43,7 @@ impl Live { pub struct AppState { pub live: ArcSwap, pub stats: Arc, + pub cache: Arc, pub secrets: SecretStore, pub config_path: String, pub pipeline: crate::pipeline::Pipeline, @@ -59,10 +61,12 @@ impl SharedState { let secrets_path = sibling(&config_path, "secrets.toml"); let secrets = SecretStore::load(&secrets_path); let stats = Stats::new(&cfg.stats); + let cache = SemanticCache::new(&cfg.cache); let live = Live::build(cfg, &secrets)?; Ok(SharedState(Arc::new(AppState { live: ArcSwap::from_pointee(live), stats, + cache, secrets, config_path, pipeline: crate::pipeline::Pipeline::new(), diff --git a/web/js/i18n/de.js b/web/js/i18n/de.js index 999696a..c14584e 100644 --- a/web/js/i18n/de.js +++ b/web/js/i18n/de.js @@ -254,4 +254,15 @@ export default { "toast.deleted": "Gelöscht", "toast.probing": "Prüfe…", "toast.imported": "Konfiguration importiert", + "dash.cache": "Semantischer Cache", + "dash.cache.hitRate": "Trefferquote", + "dash.cache.hits": "Treffer", + "dash.cache.saved": "Gespart", + "dash.cache.disabled": "Semantischer Cache ist deaktiviert — in den Einstellungen aktivieren.", + "settings.cache": "Semantischer Cache", + "settings.cache.enabled": "Semantischen Cache aktivieren", + "settings.cache.threshold": "Ähnlichkeitsschwelle", + "settings.cache.ttl": "Eintrags-TTL", + "settings.cache.embed": "Embedding-Modell", + "settings.cache.note": "Cache-Einstellungen werden nach einem Neustart wirksam.", }; diff --git a/web/js/i18n/en.js b/web/js/i18n/en.js index b2c9b74..05f4e57 100644 --- a/web/js/i18n/en.js +++ b/web/js/i18n/en.js @@ -255,4 +255,15 @@ export default { "toast.deleted": "Deleted", "toast.probing": "Probing…", "toast.imported": "Config imported", + "dash.cache": "Semantic cache", + "dash.cache.hitRate": "Hit rate", + "dash.cache.hits": "Hits", + "dash.cache.saved": "Saved", + "dash.cache.disabled": "Semantic cache is disabled — enable it in Settings.", + "settings.cache": "Semantic cache", + "settings.cache.enabled": "Enable semantic cache", + "settings.cache.threshold": "Similarity threshold", + "settings.cache.ttl": "Entry TTL", + "settings.cache.embed": "Embedding model", + "settings.cache.note": "Cache settings take effect after a restart.", }; diff --git a/web/js/i18n/es.js b/web/js/i18n/es.js index 952daa8..2dbaa6f 100644 --- a/web/js/i18n/es.js +++ b/web/js/i18n/es.js @@ -254,4 +254,15 @@ export default { "toast.deleted": "Eliminado", "toast.probing": "Sondeando…", "toast.imported": "Configuración importada", + "dash.cache": "Caché semántico", + "dash.cache.hitRate": "Aciertos", + "dash.cache.hits": "Aciertos", + "dash.cache.saved": "Ahorrado", + "dash.cache.disabled": "El caché semántico está desactivado — actívalo en Ajustes.", + "settings.cache": "Caché semántico", + "settings.cache.enabled": "Activar caché semántico", + "settings.cache.threshold": "Umbral de similitud", + "settings.cache.ttl": "TTL de entrada", + "settings.cache.embed": "Modelo de embeddings", + "settings.cache.note": "Los ajustes del caché se aplican tras reiniciar.", }; diff --git a/web/js/i18n/fr.js b/web/js/i18n/fr.js index d0f83df..d55f56b 100644 --- a/web/js/i18n/fr.js +++ b/web/js/i18n/fr.js @@ -254,4 +254,15 @@ export default { "toast.deleted": "Supprimé", "toast.probing": "Sondage…", "toast.imported": "Config importée", + "dash.cache": "Cache sémantique", + "dash.cache.hitRate": "Taux de succès", + "dash.cache.hits": "Succès", + "dash.cache.saved": "Économisé", + "dash.cache.disabled": "Le cache sémantique est désactivé — activez-le dans les Paramètres.", + "settings.cache": "Cache sémantique", + "settings.cache.enabled": "Activer le cache sémantique", + "settings.cache.threshold": "Seuil de similarité", + "settings.cache.ttl": "TTL d'entrée", + "settings.cache.embed": "Modèle d'embeddings", + "settings.cache.note": "Les paramètres du cache prennent effet après un redémarrage.", }; diff --git a/web/js/i18n/ru.js b/web/js/i18n/ru.js index 9abc3b7..2af538b 100644 --- a/web/js/i18n/ru.js +++ b/web/js/i18n/ru.js @@ -254,4 +254,15 @@ export default { "toast.deleted": "Удалено", "toast.probing": "Проверка…", "toast.imported": "Конфигурация импортирована", + "dash.cache": "Семантический кэш", + "dash.cache.hitRate": "Попаданий", + "dash.cache.hits": "Попадания", + "dash.cache.saved": "Сэкономлено", + "dash.cache.disabled": "Семантический кэш выключен — включите в Настройках.", + "settings.cache": "Семантический кэш", + "settings.cache.enabled": "Включить семантический кэш", + "settings.cache.threshold": "Порог сходства", + "settings.cache.ttl": "TTL записи", + "settings.cache.embed": "Модель эмбеддингов", + "settings.cache.note": "Настройки кэша применяются после перезапуска.", }; diff --git a/web/js/i18n/tr.js b/web/js/i18n/tr.js index 7d22bed..ce214d8 100644 --- a/web/js/i18n/tr.js +++ b/web/js/i18n/tr.js @@ -254,4 +254,15 @@ export default { "toast.deleted": "Silindi", "toast.probing": "Sınanıyor…", "toast.imported": "Yapılandırma içe aktarıldı", + "dash.cache": "Anlamsal önbellek", + "dash.cache.hitRate": "İsabet oranı", + "dash.cache.hits": "İsabet", + "dash.cache.saved": "Tasarruf", + "dash.cache.disabled": "Anlamsal önbellek kapalı — Ayarlar'dan etkinleştirin.", + "settings.cache": "Anlamsal önbellek", + "settings.cache.enabled": "Anlamsal önbelleği etkinleştir", + "settings.cache.threshold": "Benzerlik eşiği", + "settings.cache.ttl": "Giriş TTL", + "settings.cache.embed": "Gömme modeli", + "settings.cache.note": "Anlamsal önbellek ayarları yeniden başlatmadan sonra geçerli olur.", }; diff --git a/web/js/i18n/zh.js b/web/js/i18n/zh.js index 0491dc0..b7a8f7f 100644 --- a/web/js/i18n/zh.js +++ b/web/js/i18n/zh.js @@ -254,4 +254,15 @@ export default { "toast.deleted": "已删除", "toast.probing": "探测中…", "toast.imported": "配置已导入", + "dash.cache": "语义缓存", + "dash.cache.hitRate": "命中率", + "dash.cache.hits": "命中", + "dash.cache.saved": "已节省", + "dash.cache.disabled": "语义缓存已禁用 — 在设置中启用。", + "settings.cache": "语义缓存", + "settings.cache.enabled": "启用语义缓存", + "settings.cache.threshold": "相似度阈值", + "settings.cache.ttl": "条目 TTL", + "settings.cache.embed": "嵌入模型", + "settings.cache.note": "缓存设置将在重启后生效。", }; diff --git a/web/js/views/dashboard.js b/web/js/views/dashboard.js index 50c14ce..659420f 100644 --- a/web/js/views/dashboard.js +++ b/web/js/views/dashboard.js @@ -108,6 +108,22 @@ export async function renderDashboard() { barList(breakdown, num) ), + // semantic cache + stats.cache && stats.cache.enabled + ? h( + "div", + { class: "card" }, + h("div", { class: "card-head" }, h("h3", {}, t("dash.cache"))), + h( + "div", + { class: "kpis" }, + kpi(pct(stats.cache.hit_rate), t("dash.cache.hitRate")), + kpi(num(stats.cache.hits), t("dash.cache.hits")), + kpi(money(stats.cache.saved_usd), t("dash.cache.saved")) + ) + ) + : null, + // recent h( "div", diff --git a/web/js/views/settings.js b/web/js/views/settings.js index d0a5cc0..bb7afcf 100644 --- a/web/js/views/settings.js +++ b/web/js/views/settings.js @@ -42,6 +42,18 @@ export async function renderSettings() { const statsEnabled = check(t("settings.stats.enabled"), s.stats?.enabled); const statsFile = h("input", { value: s.stats?.file || "" }); const statsRet = h("input", { value: s.stats?.retention || "7d" }); + // cache + const modelsData = await api.listModels(); + const embedModels = (modelsData.models || []).filter((m) => m.kind === "embedding").map((m) => m.id); + const cacheEnabled = check(t("settings.cache.enabled"), s.cache?.enabled); + const cacheThresh = h("input", { type: "number", step: "0.01", value: s.cache?.threshold ?? 0.92 }); + const cacheTtlIn = h("input", { value: s.cache?.ttl || "1h" }); + const cacheEmbedSel = h( + "select", + {}, + h("option", { value: "" }, "auto"), + ...embedModels.map((id) => h("option", { value: id, selected: id === s.cache?.embed_model }, id)) + ); const saveBtn = h("button", { class: "btn primary" }, t("common.save")); saveBtn.addEventListener("click", async () => { @@ -71,11 +83,19 @@ export async function renderSettings() { retention: statsRet.value.trim() || "7d", ring_size: s.stats?.ring_size ?? 500, }, + cache: { + enabled: cacheEnabled.cb.checked, + threshold: parseFloat(cacheThresh.value) || 0.92, + ttl: cacheTtlIn.value.trim() || "1h", + max_entries: s.cache?.max_entries ?? 1000, + embed_model: cacheEmbedSel.value || null, + }, }; // serde: drop null optionals if (!patch.router.api_key_env) delete patch.router.api_key_env; if (!patch.router.taxonomy_id) delete patch.router.taxonomy_id; if (!patch.telemetry.otlp_endpoint_env) delete patch.telemetry.otlp_endpoint_env; + if (!patch.cache.embed_model) delete patch.cache.embed_model; try { const r = await api.putSettings(patch); toast(r.needs_restart ? t("settings.needsRestart") : t("toast.saved"), r.needs_restart ? "warn" : "good"); @@ -170,6 +190,15 @@ export async function renderSettings() { statsEnabled.node, field(t("settings.stats.file"), statsFile), field(t("settings.stats.retention"), statsRet) + ), + h( + "div", + { class: "card" }, + h("div", { class: "card-head" }, h("h3", {}, t("settings.cache"))), + cacheEnabled.node, + h("div", { class: "row" }, field(t("settings.cache.threshold"), cacheThresh), field(t("settings.cache.ttl"), cacheTtlIn)), + field(t("settings.cache.embed"), cacheEmbedSel), + h("div", { class: "hint" }, t("settings.cache.note")) ) ), h(