feat: semantic (embedding-based) response cache

- cache.rs: cosine-similarity store (threshold, TTL, ring buffer), partitioned by
  routing signature; tracks hit rate and model cost saved
- pipeline: embed routing text, look up before the model call (route_source=cache on
  hit), insert successful responses; stats record cache hits
- config [cache] + admin settings card; dashboard shows hit-rate and savings
- embedding model resolved from the pool (or first kind="embedding")
This commit is contained in:
infosave2007
2026-06-30 17:04:53 +03:00
parent bf8534f13d
commit 5759de153b
17 changed files with 492 additions and 5 deletions
+4
View File
@@ -5,6 +5,7 @@
//! a restart: the admin API validates the new config, builds a new `Live`,
//! atomically writes the TOML, and swaps the pointer.
use crate::cache::SemanticCache;
use crate::config::Config;
use crate::registry::Registry;
use crate::router_client::RouterClient;
@@ -42,6 +43,7 @@ impl Live {
pub struct AppState {
pub live: ArcSwap<Live>,
pub stats: Arc<Stats>,
pub cache: Arc<SemanticCache>,
pub secrets: SecretStore,
pub config_path: String,
pub pipeline: crate::pipeline::Pipeline,
@@ -59,10 +61,12 @@ impl SharedState {
let secrets_path = sibling(&config_path, "secrets.toml");
let secrets = SecretStore::load(&secrets_path);
let stats = Stats::new(&cfg.stats);
let cache = SemanticCache::new(&cfg.cache);
let live = Live::build(cfg, &secrets)?;
Ok(SharedState(Arc::new(AppState {
live: ArcSwap::from_pointee(live),
stats,
cache,
secrets,
config_path,
pipeline: crate::pipeline::Pipeline::new(),