feat: semantic (embedding-based) response cache
- cache.rs: cosine-similarity store (threshold, TTL, ring buffer), partitioned by routing signature; tracks hit rate and model cost saved - pipeline: embed routing text, look up before the model call (route_source=cache on hit), insert successful responses; stats record cache hits - config [cache] + admin settings card; dashboard shows hit-rate and savings - embedding model resolved from the pool (or first kind="embedding")
This commit is contained in:
Vendored
+16
@@ -108,6 +108,22 @@ export async function renderDashboard() {
|
||||
barList(breakdown, num)
|
||||
),
|
||||
|
||||
// semantic cache
|
||||
stats.cache && stats.cache.enabled
|
||||
? h(
|
||||
"div",
|
||||
{ class: "card" },
|
||||
h("div", { class: "card-head" }, h("h3", {}, t("dash.cache"))),
|
||||
h(
|
||||
"div",
|
||||
{ class: "kpis" },
|
||||
kpi(pct(stats.cache.hit_rate), t("dash.cache.hitRate")),
|
||||
kpi(num(stats.cache.hits), t("dash.cache.hits")),
|
||||
kpi(money(stats.cache.saved_usd), t("dash.cache.saved"))
|
||||
)
|
||||
)
|
||||
: null,
|
||||
|
||||
// recent
|
||||
h(
|
||||
"div",
|
||||
|
||||
Vendored
+29
@@ -42,6 +42,18 @@ export async function renderSettings() {
|
||||
const statsEnabled = check(t("settings.stats.enabled"), s.stats?.enabled);
|
||||
const statsFile = h("input", { value: s.stats?.file || "" });
|
||||
const statsRet = h("input", { value: s.stats?.retention || "7d" });
|
||||
// cache
|
||||
const modelsData = await api.listModels();
|
||||
const embedModels = (modelsData.models || []).filter((m) => m.kind === "embedding").map((m) => m.id);
|
||||
const cacheEnabled = check(t("settings.cache.enabled"), s.cache?.enabled);
|
||||
const cacheThresh = h("input", { type: "number", step: "0.01", value: s.cache?.threshold ?? 0.92 });
|
||||
const cacheTtlIn = h("input", { value: s.cache?.ttl || "1h" });
|
||||
const cacheEmbedSel = h(
|
||||
"select",
|
||||
{},
|
||||
h("option", { value: "" }, "auto"),
|
||||
...embedModels.map((id) => h("option", { value: id, selected: id === s.cache?.embed_model }, id))
|
||||
);
|
||||
|
||||
const saveBtn = h("button", { class: "btn primary" }, t("common.save"));
|
||||
saveBtn.addEventListener("click", async () => {
|
||||
@@ -71,11 +83,19 @@ export async function renderSettings() {
|
||||
retention: statsRet.value.trim() || "7d",
|
||||
ring_size: s.stats?.ring_size ?? 500,
|
||||
},
|
||||
cache: {
|
||||
enabled: cacheEnabled.cb.checked,
|
||||
threshold: parseFloat(cacheThresh.value) || 0.92,
|
||||
ttl: cacheTtlIn.value.trim() || "1h",
|
||||
max_entries: s.cache?.max_entries ?? 1000,
|
||||
embed_model: cacheEmbedSel.value || null,
|
||||
},
|
||||
};
|
||||
// serde: drop null optionals
|
||||
if (!patch.router.api_key_env) delete patch.router.api_key_env;
|
||||
if (!patch.router.taxonomy_id) delete patch.router.taxonomy_id;
|
||||
if (!patch.telemetry.otlp_endpoint_env) delete patch.telemetry.otlp_endpoint_env;
|
||||
if (!patch.cache.embed_model) delete patch.cache.embed_model;
|
||||
try {
|
||||
const r = await api.putSettings(patch);
|
||||
toast(r.needs_restart ? t("settings.needsRestart") : t("toast.saved"), r.needs_restart ? "warn" : "good");
|
||||
@@ -170,6 +190,15 @@ export async function renderSettings() {
|
||||
statsEnabled.node,
|
||||
field(t("settings.stats.file"), statsFile),
|
||||
field(t("settings.stats.retention"), statsRet)
|
||||
),
|
||||
h(
|
||||
"div",
|
||||
{ class: "card" },
|
||||
h("div", { class: "card-head" }, h("h3", {}, t("settings.cache"))),
|
||||
cacheEnabled.node,
|
||||
h("div", { class: "row" }, field(t("settings.cache.threshold"), cacheThresh), field(t("settings.cache.ttl"), cacheTtlIn)),
|
||||
field(t("settings.cache.embed"), cacheEmbedSel),
|
||||
h("div", { class: "hint" }, t("settings.cache.note"))
|
||||
)
|
||||
),
|
||||
h(
|
||||
|
||||
Reference in New Issue
Block a user