infosave2007
42377a75d4
The example config pool now shows one local CMF model (served by the managed cortiq server) instead of local-qwen/gpt-4o-mini/claude-opus placeholders, with routing pointing at it, and a documented [cmf] section (manage_server, threads, gpu, ...). Add cloud/local providers by copying the [[models]] shape.
163 lines
7.5 KiB
TOML
163 lines
7.5 KiB
TOML
# ============================================================================
|
|
# Cortiq Gateway — example configuration
|
|
# Copy to gateway.toml and edit for your own model pool.
|
|
# Do NOT store secrets here — provide them via *_env (environment variables)
|
|
# or set them from the admin console (stored in config/secrets.toml).
|
|
# ============================================================================
|
|
|
|
listen = "0.0.0.0:9000"
|
|
|
|
# --- connection to cortiq-router (decides task type / complexity) -----------
|
|
# Hosted allaigate router: url = "https://138.226.222.209", verify_tls = false
|
|
# (cert is bound to an IP), taxonomy_id = "data-assistant", a cortiq_… key in
|
|
# CORTIQ_ROUTER_KEY.
|
|
# NOTE: on hard prompts the router escalates to an oracle LLM (~10 s) — for the
|
|
# hosted router set timeout_ms = 12000+, otherwise the gateway degrades to default.
|
|
[router]
|
|
url = "http://localhost:8080"
|
|
api_key_env = "CORTIQ_ROUTER_KEY" # name of the env var holding the router key
|
|
timeout_ms = 800 # timeout for the /v1/route call (12000+ for oracle)
|
|
taxonomy_id = "general.task-type" # for allaigate: "data-assistant"
|
|
verify_tls = true # false if the router uses an IP / self-signed cert
|
|
|
|
# --- how to pick the text to classify from the conversation -----------------
|
|
[route]
|
|
text_strategy = "last_user" # last_user | last_user_plus_system | concat_all
|
|
max_chars = 4000
|
|
cache_ttl = "60s"
|
|
profile = "balanced" # cost-saver | balanced | quality-first
|
|
|
|
# ============================================================================
|
|
# Model pool. This example ships ONE local CMF model, served by the managed
|
|
# cortiq server (see the [cmf] section below). Add more models — local or cloud —
|
|
# by copying the [[models]] shape: provider = "openai"/"anthropic", and for a
|
|
# hosted provider an api_key_env = "..." (never put secrets in this file).
|
|
# ============================================================================
|
|
|
|
[[models]]
|
|
id = "cmf-local"
|
|
provider = "openai"
|
|
base_url = "http://127.0.0.1:8081/v1" # the managed cortiq server (see [cmf])
|
|
model = "cmf-local"
|
|
cost_tier = "local"
|
|
price_in = 0.0
|
|
price_out = 0.0
|
|
caps = ["tools"]
|
|
|
|
# --- embedding model (optional: /v1/embeddings + the semantic cache) ---------
|
|
[[models]]
|
|
id = "local-embed"
|
|
provider = "openai"
|
|
base_url = "http://localhost:8000/v1" # a local embeddings server (vLLM / llama.cpp / Ollama)
|
|
model = "nomic-embed-text"
|
|
cost_tier = "local"
|
|
kind = "embedding"
|
|
|
|
# ============================================================================
|
|
# Routing table: complexity tier → ordered pool (priority = fallback chain).
|
|
# Add cloud models above and list them here for cost/quality-based fallback.
|
|
# ============================================================================
|
|
[routing]
|
|
low = ["cmf-local"]
|
|
medium = ["cmf-local"]
|
|
high = ["cmf-local"]
|
|
default = "cmf-local" # used when the router is unreachable
|
|
|
|
[routing.policy]
|
|
mode = "fixed_table" # fixed_table | cost_aware
|
|
# for cost_aware:
|
|
# min_class = { low = "cheap", medium = "mid", high = "expensive" }
|
|
# max_cost_usd_per_request = 0.50
|
|
|
|
# --- resilience -------------------------------------------------------------
|
|
[breaker]
|
|
threshold = 5
|
|
cooldown = "30s"
|
|
|
|
# ============================================================================
|
|
# Inbound protocols — enable only what you need (toggles apply live in the UI)
|
|
# ============================================================================
|
|
[protocols]
|
|
openai_chat = true
|
|
openai_completions = true
|
|
openai_embeddings = true
|
|
openai_models = true
|
|
anthropic_messages = false
|
|
mcp = false
|
|
native_passthrough = true
|
|
|
|
# --- response routing metadata ----------------------------------------------
|
|
[cortiq]
|
|
echo = true # add a "cortiq" field to the response body (X-Cortiq-* headers are always sent)
|
|
|
|
# ============================================================================
|
|
# Authentication: virtual gateway keys (agents get these, not provider keys).
|
|
# If this section is empty, the gateway runs in open mode (local development).
|
|
# ============================================================================
|
|
[[api_keys]]
|
|
key = "sk-gw-team-a"
|
|
account = "team-a"
|
|
rate_per_min = 600
|
|
# allow_models = ["cortiq-auto", "cmf-local"] # restrict available models (optional)
|
|
|
|
# --- idempotency and logging ------------------------------------------------
|
|
[idempotency]
|
|
ttl = "10m"
|
|
|
|
[log]
|
|
bodies = false # do NOT log message bodies (privacy) — default
|
|
level = "info"
|
|
|
|
[telemetry]
|
|
metrics = true # GET /metrics (Prometheus)
|
|
otlp_endpoint_env = "OTEL_EXPORTER_OTLP_ENDPOINT" # optional
|
|
|
|
# ============================================================================
|
|
# Admin web console — embedded in the binary, served at /admin
|
|
# ============================================================================
|
|
[admin]
|
|
enabled = true
|
|
# Name of the env var holding the admin token. If unset (or empty), a token is
|
|
# generated at startup and printed to the log (see also the --admin-token flag).
|
|
token_env = "CORTIQ_ADMIN_TOKEN"
|
|
# listen = "127.0.0.1:9001" # optional: separate bind for admin (default: main listen)
|
|
|
|
# ============================================================================
|
|
# Request statistics (dashboard + /metrics). Kept in memory + appended to JSONL.
|
|
# ============================================================================
|
|
[stats]
|
|
enabled = true
|
|
file = "config/stats.jsonl" # empty = in-memory only
|
|
ring_size = 500 # how many recent requests to keep in the feed
|
|
retention = "7d" # aggregate/series depth (replayed on startup)
|
|
|
|
# ============================================================================
|
|
# Semantic cache — returns a cached answer for prompts semantically near a
|
|
# previous one, skipping the model call. Requires an embedding model in the pool.
|
|
# Disabled by default. (Cache settings take effect on restart.)
|
|
# ============================================================================
|
|
[cache]
|
|
enabled = false
|
|
threshold = 0.92 # cosine similarity for a hit (0..1)
|
|
ttl = "1h"
|
|
max_entries = 1000
|
|
# embed_model = "local-embed" # defaults to the first kind = "embedding" model
|
|
|
|
# ============================================================================
|
|
# Local CMF model server. The gateway can install/run a local .cmf via cortiq
|
|
# (github.com/infosave2007/cmf) and register it as the "cmf-local" model above,
|
|
# so it works even when the router and cloud providers are unavailable.
|
|
# Changes to manage_server / local_model / threads / gpu apply on restart.
|
|
# ============================================================================
|
|
[cmf]
|
|
manage_server = false # true → gateway spawns `cortiq serve` at startup
|
|
local_model = "models/model.cmf" # path to the .cmf to serve
|
|
local_host = "127.0.0.1"
|
|
local_port = 8081 # must match the cmf-local base_url above
|
|
model_id = "cmf-local" # id it registers under (matches [[models]])
|
|
threads = 8 # decode worker threads (CMF_THREADS); ~3x vs serial
|
|
gpu = false # offload matvecs to GPU (Metal/wgpu); mainly a prefill win
|
|
cortiq_bin = "cortiq" # the cortiq CLI (cargo install cortiq-cli)
|
|
auto_install = false # install cortiq-cli from crates.io if the binary is missing
|
|
local_only = false # true → route every request to the local model
|