Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0f89cdaa07 | ||
|
|
d211eb6150 |
Generated
+1
-1
@@ -285,7 +285,7 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cortiq-gateway"
|
name = "cortiq-gateway"
|
||||||
version = "0.2.3"
|
version = "0.2.4"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "cortiq-gateway"
|
name = "cortiq-gateway"
|
||||||
version = "0.2.3"
|
version = "0.2.4"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
|
|||||||
+17
-28
@@ -28,40 +28,29 @@ cache_ttl = "60s"
|
|||||||
profile = "balanced" # cost-saver | balanced | quality-first
|
profile = "balanced" # cost-saver | balanced | quality-first
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# Model pool. This example ships ONE local CMF model, served by the managed
|
# Model pool — EMPTY by default. A fresh gateway starts with no models and an
|
||||||
# cortiq server (see the [cmf] section below). Add more models — local or cloud —
|
# empty pool; add them in the admin console (/admin → Models), or run a local
|
||||||
# by copying the [[models]] shape: provider = "openai"/"anthropic", and for a
|
# .cmf via the [cmf] section below. To add one by hand, copy this shape:
|
||||||
# hosted provider an api_key_env = "..." (never put secrets in this file).
|
#
|
||||||
|
# [[models]]
|
||||||
|
# id = "my-model"
|
||||||
|
# provider = "openai" # openai | anthropic (openai also covers
|
||||||
|
# base_url = "http://127.0.0.1:8081/v1" # OpenRouter/Together/Groq/vLLM/Ollama)
|
||||||
|
# model = "my-model"
|
||||||
|
# # api_key_env = "OPENAI_API_KEY" # for hosted providers (never inline a key)
|
||||||
|
# cost_tier = "local"
|
||||||
|
# caps = ["tools"]
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|
||||||
[[models]]
|
|
||||||
id = "cmf-local"
|
|
||||||
provider = "openai"
|
|
||||||
base_url = "http://127.0.0.1:8081/v1" # the managed cortiq server (see [cmf])
|
|
||||||
model = "cmf-local"
|
|
||||||
cost_tier = "local"
|
|
||||||
price_in = 0.0
|
|
||||||
price_out = 0.0
|
|
||||||
caps = ["tools"]
|
|
||||||
|
|
||||||
# --- embedding model (optional: /v1/embeddings + the semantic cache) ---------
|
|
||||||
[[models]]
|
|
||||||
id = "local-embed"
|
|
||||||
provider = "openai"
|
|
||||||
base_url = "http://localhost:8000/v1" # a local embeddings server (vLLM / llama.cpp / Ollama)
|
|
||||||
model = "nomic-embed-text"
|
|
||||||
cost_tier = "local"
|
|
||||||
kind = "embedding"
|
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# Routing table: complexity tier → ordered pool (priority = fallback chain).
|
# Routing table: complexity tier → ordered pool (priority = fallback chain).
|
||||||
# Add cloud models above and list them here for cost/quality-based fallback.
|
# Leave default empty for a fresh install; set it to a model id once you add one.
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
[routing]
|
[routing]
|
||||||
low = ["cmf-local"]
|
low = []
|
||||||
medium = ["cmf-local"]
|
medium = []
|
||||||
high = ["cmf-local"]
|
high = []
|
||||||
default = "cmf-local" # used when the router is unreachable
|
default = "" # set to a model id once you add one
|
||||||
|
|
||||||
[routing.policy]
|
[routing.policy]
|
||||||
mode = "fixed_table" # fixed_table | cost_aware
|
mode = "fixed_table" # fixed_table | cost_aware
|
||||||
|
|||||||
+6
-1
@@ -599,7 +599,12 @@ impl Config {
|
|||||||
// is what lets the gateway run on local models alone (no cloud, no router).
|
// is what lets the gateway run on local models alone (no cloud, no router).
|
||||||
let has_local = self.cmf.manage_server && !self.cmf.local_model.trim().is_empty();
|
let has_local = self.cmf.manage_server && !self.cmf.local_model.trim().is_empty();
|
||||||
if self.models.is_empty() && !has_local {
|
if self.models.is_empty() && !has_local {
|
||||||
anyhow::bail!("config: at least one [[models]] entry (or a managed local CMF model) is required");
|
// Fresh install: start with an EMPTY model pool so the admin console
|
||||||
|
// comes up — the user adds models in /admin → Models (or points [cmf]
|
||||||
|
// at a local .cmf). Routing simply has nothing to select until then.
|
||||||
|
tracing::warn!(
|
||||||
|
"no models configured — starting with an empty pool; add one in /admin → Models"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
// uniqueness of model ids
|
// uniqueness of model ids
|
||||||
let mut seen = std::collections::HashSet::new();
|
let mut seen = std::collections::HashSet::new();
|
||||||
|
|||||||
Reference in New Issue
Block a user