Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9c15109f26 | ||
|
|
f65f131c41 |
@@ -3,6 +3,43 @@
|
||||
All notable changes to this project are documented here.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/).
|
||||
|
||||
## [0.2.47] - 2026-08-21
|
||||
|
||||
### Changed
|
||||
- **The hosted allaigate router is addressed by hostname:
|
||||
`https://router.allaigate.com`** is now the default `[router].url`, and
|
||||
`verify_tls` defaults to `true`. The endpoint used to be reachable only by bare IP,
|
||||
whose certificate is issued for the hostname — so every hosted setup had to run with
|
||||
certificate verification switched off. It no longer does.
|
||||
- **Existing configs are migrated on load.** A `[router].url` still pointing at the old
|
||||
`138.226.222.209` (with or without a scheme or trailing slash) is rewritten to the
|
||||
hostname, `verify_tls` is turned back on, and the file is saved — the endpoint is the
|
||||
same machine, and this is the difference between a verified connection and a blind one.
|
||||
A self-hosted router URL is never touched.
|
||||
- README, `docs/ROUTER.md` (both languages), `config/gateway.example.toml` and
|
||||
`bench/accuracy.py` updated to the hostname.
|
||||
|
||||
## [0.2.46] - 2026-08-21
|
||||
|
||||
Verified end to end against a real Open WebUI container talking to the gateway.
|
||||
|
||||
### Fixed
|
||||
- **Offered `tools` no longer make a local model unroutable.** Open WebUI attaches its ~26
|
||||
built-in tools (knowledge bases, notes, calendar, …) to *every* message, and the gateway
|
||||
treated that offer as a hard capability requirement — so a plain "hello" against a pool of
|
||||
tools-less local models failed with «No healthy model available that satisfies required
|
||||
capabilities». In the OpenAI contract `tools` is an offer: only `tool_choice: "required"`
|
||||
(or a named function) makes the call impossible without them. The gateway now still
|
||||
prefers a tool-capable model, but falls back to serving the request with the tools
|
||||
stripped (along with `tool_choice`/`parallel_tool_calls`, which a provider rejects on
|
||||
their own). `tool_choice: "required"` still fails loudly, as it must.
|
||||
- **A pinned model that merely lacks tool support is honoured** instead of being reported as
|
||||
«not available — it is not in the pool (deleted?)». An unknown pin still fails loudly.
|
||||
|
||||
### Added
|
||||
- `X-Cortiq-Tools-Dropped: true` (and `cortiq.tools_dropped` in the echo block) whenever the
|
||||
gateway answers without the tools the client offered, so the downgrade is never silent.
|
||||
|
||||
## [0.2.45] - 2026-08-21
|
||||
|
||||
### Fixed
|
||||
|
||||
Generated
+1
-1
@@ -285,7 +285,7 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||
|
||||
[[package]]
|
||||
name = "cortiq-gateway"
|
||||
version = "0.2.45"
|
||||
version = "0.2.47"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"arc-swap",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "cortiq-gateway"
|
||||
version = "0.2.45"
|
||||
version = "0.2.47"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "Apache-2.0"
|
||||
|
||||
@@ -175,9 +175,9 @@ print(resp.choices[0].message.content)
|
||||
`model: "cortiq-auto"` enables routing. Any **real** model name from the config
|
||||
(`"gpt-4o-mini"`, `"local-qwen"`) is a direct passthrough, without routing.
|
||||
|
||||
> **Using the hosted allaigate router?** Set `url = "https://138.226.222.209"`,
|
||||
> `verify_tls = false`, `taxonomy_id = "data-assistant"`, and a `cortiq_…` key in
|
||||
> `CORTIQ_ROUTER_KEY`. On hard prompts the router escalates to an oracle (~10 s), so
|
||||
> **Using the hosted allaigate router?** It is the default: `url =
|
||||
> "https://router.allaigate.com"` with `verify_tls = true`. Set `taxonomy_id =
|
||||
> "data-assistant"` and a `cortiq_…` key in `CORTIQ_ROUTER_KEY`. On hard prompts the router escalates to an oracle (~10 s), so
|
||||
> set `timeout_ms = 12000+` — otherwise the gateway gracefully degrades to the default model.
|
||||
|
||||
---
|
||||
|
||||
+3
-3
@@ -167,9 +167,9 @@ print(resp.choices[0].message.content)
|
||||
`model: "cortiq-auto"` включает роутинг. Любое **реальное** имя модели из конфига
|
||||
(`"gpt-4o-mini"`, `"local-qwen"`) — это passthrough напрямую, без роутинга.
|
||||
|
||||
> **Используете хостовый allaigate-роутер?** Укажите `url = "https://138.226.222.209"`,
|
||||
> `verify_tls = false`, `taxonomy_id = "data-assistant"` и ключ `cortiq_…` в
|
||||
> `CORTIQ_ROUTER_KEY`. На сложных запросах роутер эскалирует к oracle (~10 с), поэтому
|
||||
> **Используете хостовый allaigate-роутер?** Он стоит по умолчанию: `url =
|
||||
> "https://router.allaigate.com"`, `verify_tls = true`. Задайте `taxonomy_id =
|
||||
> "data-assistant"` и ключ `cortiq_…` в `CORTIQ_ROUTER_KEY`. На сложных запросах роутер эскалирует к oracle (~10 с), поэтому
|
||||
> ставьте `timeout_ms = 12000+` — иначе шлюз мягко деградирует на default-модель.
|
||||
|
||||
---
|
||||
|
||||
+2
-2
@@ -4,12 +4,12 @@ heuristic (what you get without a semantic classifier, e.g. with LiteLLM/Portkey
|
||||
|
||||
CORTIQ_ROUTER_KEY=cortiq_... python3 bench/accuracy.py
|
||||
|
||||
Env: CORTIQ_ROUTER_KEY (required), ROUTER_URL (default https://138.226.222.209),
|
||||
Env: CORTIQ_ROUTER_KEY (required), ROUTER_URL (default https://router.allaigate.com),
|
||||
TAXONOMY (default data-assistant).
|
||||
"""
|
||||
import json, os, ssl, sys, urllib.request
|
||||
|
||||
ROUTER_URL = os.environ.get("ROUTER_URL", "https://138.226.222.209").rstrip("/")
|
||||
ROUTER_URL = os.environ.get("ROUTER_URL", "https://router.allaigate.com").rstrip("/")
|
||||
TAXONOMY = os.environ.get("TAXONOMY", "data-assistant")
|
||||
KEY = os.environ.get("CORTIQ_ROUTER_KEY", "")
|
||||
CTX = ssl._create_unverified_context()
|
||||
|
||||
@@ -8,13 +8,13 @@
|
||||
listen = "0.0.0.0:9000"
|
||||
|
||||
# --- connection to cortiq-router (decides task type / complexity) -----------
|
||||
# Hosted allaigate router: url = "https://138.226.222.209", verify_tls = false
|
||||
# Hosted allaigate router: url = "https://router.allaigate.com" (the default)
|
||||
# (cert is bound to an IP), taxonomy_id = "data-assistant", a cortiq_… key in
|
||||
# CORTIQ_ROUTER_KEY.
|
||||
# NOTE: on hard prompts the router escalates to an oracle LLM (~10 s) — for the
|
||||
# hosted router set timeout_ms = 12000+, otherwise the gateway degrades to default.
|
||||
[router]
|
||||
url = "https://138.226.222.209" # hosted allaigate router (or your own cortiq-router URL)
|
||||
url = "https://router.allaigate.com" # hosted allaigate router (or your own cortiq-router URL)
|
||||
api_key_env = "CORTIQ_ROUTER_KEY" # name of the env var holding the router key
|
||||
timeout_ms = 12000 # hard prompts escalate to an oracle (~10 s)
|
||||
taxonomy_id = "data-assistant" # the hosted router's task taxonomy
|
||||
|
||||
@@ -65,6 +65,13 @@ concatenated, images and other non-text parts are skipped); `tools` may be `null
|
||||
cannot be read comes back as an OpenAI error envelope with HTTP 400 — never a bare 422,
|
||||
which OpenAI clients cannot parse (they show no answer at all).
|
||||
|
||||
`tools` is treated as an **offer**, not a requirement. The gateway prefers a tool-capable
|
||||
model, but when none is available it answers without the tools rather than failing, and says
|
||||
so via `X-Cortiq-Tools-Dropped: true` (chat UIs attach their built-in tools to every message,
|
||||
so the strict reading made ordinary chat impossible on a tools-less local pool).
|
||||
`tool_choice: "required"` — or a named function — is a real requirement and still fails with
|
||||
`502 no_healthy_model` when no model can honour it.
|
||||
|
||||
### Response (non-streaming)
|
||||
|
||||
```jsonc
|
||||
|
||||
@@ -65,6 +65,13 @@
|
||||
HTTP 400 — но не «голый» 422, который OpenAI-клиенты не умеют разбирать (для пользователя
|
||||
это выглядит как отсутствие ответа).
|
||||
|
||||
`tools` трактуется как **предложение**, а не требование. Шлюз предпочитает модель с
|
||||
поддержкой tools, но если такой нет — отвечает без них, а не падает, и сообщает об этом
|
||||
заголовком `X-Cortiq-Tools-Dropped: true` (чат-интерфейсы добавляют свои встроенные tools в
|
||||
каждое сообщение, поэтому строгое прочтение делало обычный чат на локальном пуле без tools
|
||||
невозможным). `tool_choice: "required"` — или именованная функция — это уже настоящее
|
||||
требование, и оно по-прежнему завершается `502 no_healthy_model`, если его некому выполнить.
|
||||
|
||||
### Ответ (без стриминга)
|
||||
|
||||
```jsonc
|
||||
|
||||
+4
-4
@@ -41,11 +41,11 @@ minted and shown to you.
|
||||
|
||||
```toml
|
||||
[router]
|
||||
url = "https://138.226.222.209" # allaigate routing API
|
||||
verify_tls = false # cert is bound to an IP
|
||||
url = "https://router.allaigate.com" # allaigate routing API (the default)
|
||||
verify_tls = true # valid certificate — keep verification on
|
||||
taxonomy_id = "data-assistant"
|
||||
api_key_env = "CORTIQ_ROUTER_KEY" # your cortiq_… key
|
||||
timeout_ms = 15000 # allow for oracle escalation on hard prompts
|
||||
api_key_env = "CORTIQ_ROUTER_KEY" # your cortiq_… key
|
||||
timeout_ms = 15000 # allow for oracle escalation on hard prompts
|
||||
```
|
||||
|
||||
```bash
|
||||
|
||||
+4
-4
@@ -40,11 +40,11 @@ Cortiq Gateway — тонкий и быстрый прокси; **интелле
|
||||
|
||||
```toml
|
||||
[router]
|
||||
url = "https://138.226.222.209" # routing API allaigate
|
||||
verify_tls = false # серт привязан к IP
|
||||
url = "https://router.allaigate.com" # routing API allaigate (значение по умолчанию)
|
||||
verify_tls = true # валидный сертификат — проверку не выключаем
|
||||
taxonomy_id = "data-assistant"
|
||||
api_key_env = "CORTIQ_ROUTER_KEY" # ваш ключ cortiq_…
|
||||
timeout_ms = 15000 # запас под oracle-эскалацию на сложных запросах
|
||||
api_key_env = "CORTIQ_ROUTER_KEY" # ваш ключ cortiq_…
|
||||
timeout_ms = 15000 # запас под oracle-эскалацию на сложных запросах
|
||||
```
|
||||
|
||||
```bash
|
||||
|
||||
+71
-3
@@ -318,6 +318,34 @@ fn default_verify_tls() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// The hosted allaigate routing API.
|
||||
pub const DEFAULT_ROUTER_URL: &str = "https://router.allaigate.com";
|
||||
|
||||
/// The bare IP the hosted router used to be addressed by. It served a
|
||||
/// certificate for `router.allaigate.com`, so every config pointing at it had to
|
||||
/// turn TLS verification off. The hostname now resolves, so configs written
|
||||
/// before it existed are moved over on load — the endpoint is the same machine,
|
||||
/// and this is the difference between a verified connection and a blind one.
|
||||
const LEGACY_ROUTER_HOSTS: &[&str] = &[
|
||||
"https://138.226.222.209",
|
||||
"http://138.226.222.209",
|
||||
"138.226.222.209",
|
||||
];
|
||||
|
||||
impl RouterCfg {
|
||||
/// Point a legacy IP-addressed router config at the hostname and restore TLS
|
||||
/// verification. Returns true when something changed.
|
||||
fn migrate_hosted_endpoint(&mut self) -> bool {
|
||||
let url = self.url.trim().trim_end_matches('/');
|
||||
if !LEGACY_ROUTER_HOSTS.contains(&url) {
|
||||
return false;
|
||||
}
|
||||
self.url = DEFAULT_ROUTER_URL.to_string();
|
||||
self.verify_tls = true;
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct RouteCfg {
|
||||
#[serde(default = "default_text_strategy")]
|
||||
@@ -882,9 +910,9 @@ impl Default for Config {
|
||||
// models and the console asks for one; nothing breaks.
|
||||
router: RouterCfg {
|
||||
enabled: true,
|
||||
url: "https://138.226.222.209".to_string(),
|
||||
url: DEFAULT_ROUTER_URL.to_string(),
|
||||
timeout_ms: 15_000, // hard prompts escalate to an oracle (~10 s)
|
||||
verify_tls: false, // the hosted endpoint is IP-bound
|
||||
verify_tls: true,
|
||||
api_key_env: Some("CORTIQ_ROUTER_KEY".to_string()),
|
||||
taxonomy_id: Some("data-assistant".to_string()),
|
||||
},
|
||||
@@ -954,7 +982,14 @@ impl Config {
|
||||
pub fn load(path: &str) -> anyhow::Result<Self> {
|
||||
let raw = std::fs::read_to_string(path)
|
||||
.map_err(|e| anyhow::anyhow!("cannot read config {path}: {e}"))?;
|
||||
let cfg: Config = toml::from_str(&raw)?;
|
||||
let mut cfg: Config = toml::from_str(&raw)?;
|
||||
if cfg.router.migrate_hosted_endpoint() {
|
||||
tracing::info!(
|
||||
url = DEFAULT_ROUTER_URL,
|
||||
"router endpoint moved from the bare IP to the hostname; TLS verification is on"
|
||||
);
|
||||
let _ = cfg.save(path);
|
||||
}
|
||||
cfg.validate()?;
|
||||
Ok(cfg)
|
||||
}
|
||||
@@ -1091,6 +1126,39 @@ impl Config {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_hosted_router_defaults_to_the_hostname_with_tls_on() {
|
||||
let cfg = Config::default();
|
||||
assert_eq!(cfg.router.url, "https://router.allaigate.com");
|
||||
assert!(cfg.router.verify_tls);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_legacy_ip_router_config_moves_to_the_hostname() {
|
||||
for legacy in [
|
||||
"https://138.226.222.209",
|
||||
"https://138.226.222.209/",
|
||||
"138.226.222.209",
|
||||
] {
|
||||
let mut r = Config::default().router;
|
||||
r.url = legacy.to_string();
|
||||
r.verify_tls = false;
|
||||
assert!(r.migrate_hosted_endpoint(), "{legacy} must migrate");
|
||||
assert_eq!(r.url, DEFAULT_ROUTER_URL);
|
||||
assert!(r.verify_tls, "TLS verification must be restored");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_self_hosted_router_url_is_left_alone() {
|
||||
let mut r = Config::default().router;
|
||||
r.url = "http://127.0.0.1:8081".to_string();
|
||||
r.verify_tls = false;
|
||||
assert!(!r.migrate_hosted_endpoint());
|
||||
assert_eq!(r.url, "http://127.0.0.1:8081");
|
||||
assert!(!r.verify_tls);
|
||||
}
|
||||
|
||||
const SAMPLE: &str = r#"
|
||||
listen = "0.0.0.0:9000"
|
||||
|
||||
|
||||
@@ -113,6 +113,11 @@ pub struct RouteInfo {
|
||||
pub cost_usd: f64,
|
||||
#[serde(default)]
|
||||
pub failover: bool,
|
||||
/// The client offered `tools` but the model that answered cannot use them,
|
||||
/// so they were dropped from the upstream call (see `Pipeline::resolve`).
|
||||
/// Surfaced as `X-Cortiq-Tools-Dropped` so a silent downgrade is debuggable.
|
||||
#[serde(default)]
|
||||
pub tools_dropped: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
|
||||
+137
-8
@@ -186,7 +186,7 @@ impl Pipeline {
|
||||
// Tool support detected via the local servers' healthz handshake —
|
||||
// treated as equivalent to declared caps when filtering candidates.
|
||||
detected_tools: &std::collections::HashSet<String>,
|
||||
) -> Result<(RouteDecision, String, Vec<String>)> {
|
||||
) -> Result<(RouteDecision, String, Vec<String>, bool)> {
|
||||
// 1. Extract text for classification
|
||||
let text = self.extract_text(
|
||||
&req.messages,
|
||||
@@ -277,6 +277,7 @@ impl Pipeline {
|
||||
|
||||
// Filter candidates by capabilities (e.g. tools support)
|
||||
let needs_tools = !req.tools.is_empty();
|
||||
let tools_mandatory = needs_tools && tool_choice_is_mandatory(&req.params.passthrough);
|
||||
let has_tools = |id: &str, prov: &std::sync::Arc<dyn crate::providers::Provider>| {
|
||||
prov.caps().tools || detected_tools.contains(id)
|
||||
};
|
||||
@@ -294,6 +295,31 @@ impl Pipeline {
|
||||
}
|
||||
}
|
||||
|
||||
// `tools` is an OFFER in the OpenAI contract, not a demand: only
|
||||
// `tool_choice: "required"` (or a named function) makes the call
|
||||
// impossible without them. Chat UIs lean on that — Open WebUI attaches
|
||||
// its ~26 built-in tools to every single message — so treating the offer
|
||||
// as a hard requirement made an ordinary "hello" unanswerable on a pool
|
||||
// of tools-less local models. Serve the request without the tools and
|
||||
// flag the downgrade (`X-Cortiq-Tools-Dropped`) instead of failing.
|
||||
let mut tools_dropped = false;
|
||||
|
||||
// A pin stays a pin: when the pinned model exists and merely lacks tool
|
||||
// support, honour the pin and drop the offer rather than reporting the
|
||||
// model as missing.
|
||||
if final_candidates.is_empty() && needs_tools && !tools_mandatory {
|
||||
if let RoutingDirective::Pinned { model_id } = &req.routing {
|
||||
if live.registry.get(model_id).is_some() {
|
||||
final_candidates.push(model_id.clone());
|
||||
tools_dropped = true;
|
||||
tracing::info!(
|
||||
model = %model_id,
|
||||
"pinned model does not support the offered tools — serving without them"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An explicit pin must never silently answer from a different model: if
|
||||
// the pinned id is unknown or was filtered out, fail loudly so the caller
|
||||
// knows their choice was not honoured (instead of getting a stray reply
|
||||
@@ -326,7 +352,7 @@ impl Pipeline {
|
||||
// (an agentic request is worth a capable model), then the rest of the
|
||||
// pool. Coding agents (OpenCode, Cline, …) always send tools; without
|
||||
// this every low-tier verdict was a hard failure.
|
||||
if final_candidates.is_empty() {
|
||||
let widened_order = || -> Vec<String> {
|
||||
let mut widened: Vec<String> = Vec::new();
|
||||
for tier in ["medium", "high"] {
|
||||
for id in live.routing.candidates(tier) {
|
||||
@@ -340,7 +366,10 @@ impl Pipeline {
|
||||
widened.push(id.clone());
|
||||
}
|
||||
}
|
||||
for id in widened {
|
||||
widened
|
||||
};
|
||||
if final_candidates.is_empty() {
|
||||
for id in widened_order() {
|
||||
if let Some(prov) = live.registry.get(&id) {
|
||||
if needs_tools && !has_tools(&id, &prov) {
|
||||
continue;
|
||||
@@ -355,20 +384,40 @@ impl Pipeline {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Same relaxation for an AUTO route once every narrower pass came up empty.
|
||||
if final_candidates.is_empty() && needs_tools && !tools_mandatory {
|
||||
for id in widened_order() {
|
||||
if live.registry.get(&id).is_some() {
|
||||
final_candidates.push(id);
|
||||
}
|
||||
}
|
||||
if !final_candidates.is_empty() {
|
||||
tools_dropped = true;
|
||||
tracing::info!(
|
||||
chosen = %final_candidates[0],
|
||||
"no model in the pool supports the offered tools — serving without them"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if final_candidates.is_empty() {
|
||||
return Err(GatewayError::UpstreamUnavailable(
|
||||
"No healthy model available that satisfies required capabilities (e.g. tools)"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
Ok((decision, route_source, final_candidates))
|
||||
Ok((decision, route_source, final_candidates, tools_dropped))
|
||||
}
|
||||
|
||||
async fn run_inner(&self, req: ChatRequest, state: &SharedState) -> Result<ChatResponse> {
|
||||
async fn run_inner(&self, mut req: ChatRequest, state: &SharedState) -> Result<ChatResponse> {
|
||||
let live = state.live();
|
||||
let detected_tools = state.cmf.detected_tools.lock().unwrap().clone();
|
||||
let (decision, route_source, mut final_candidates) =
|
||||
let (decision, route_source, mut final_candidates, tools_dropped) =
|
||||
self.resolve(&req, &live, &detected_tools).await?;
|
||||
if tools_dropped {
|
||||
strip_tools(&mut req);
|
||||
}
|
||||
|
||||
// Serving-when-promoted (gated): a promoted task-label is served by the
|
||||
// local model FIRST (failover to cloud on error). Per-request VETO uses
|
||||
@@ -478,6 +527,7 @@ impl Pipeline {
|
||||
router_request_id: decision.router_request_id.clone(),
|
||||
cost_usd: cost,
|
||||
failover: attempt > 0,
|
||||
tools_dropped,
|
||||
};
|
||||
return Ok(resp);
|
||||
}
|
||||
@@ -504,7 +554,7 @@ impl Pipeline {
|
||||
/// recorded when the stream completes (tokens taken from the final usage chunk).
|
||||
pub async fn run_stream(
|
||||
&self,
|
||||
req: ChatRequest,
|
||||
mut req: ChatRequest,
|
||||
state: &SharedState,
|
||||
) -> Result<(RouteInfo, crate::providers::ChatStream)> {
|
||||
let started = Instant::now();
|
||||
@@ -518,7 +568,7 @@ impl Pipeline {
|
||||
|
||||
let live = state.live();
|
||||
let detected_tools = state.cmf.detected_tools.lock().unwrap().clone();
|
||||
let (decision, route_source, final_candidates) =
|
||||
let (decision, route_source, final_candidates, tools_dropped) =
|
||||
match self.resolve(&req, &live, &detected_tools).await {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
@@ -526,6 +576,9 @@ impl Pipeline {
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
if tools_dropped {
|
||||
strip_tools(&mut req);
|
||||
}
|
||||
|
||||
let mut last_err = None;
|
||||
let mut attempt = 0usize;
|
||||
@@ -612,6 +665,7 @@ impl Pipeline {
|
||||
router_request_id: decision.router_request_id.clone(),
|
||||
cost_usd: 0.0,
|
||||
failover: attempt > 0,
|
||||
tools_dropped,
|
||||
};
|
||||
// for the client-facing usage estimate when the provider
|
||||
// sends none: the prompt size is known here, the reply is
|
||||
@@ -689,6 +743,26 @@ impl Pipeline {
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove the tool offer from a request the chosen model cannot honour. The
|
||||
/// companion fields must go too: an upstream given `tool_choice` with no
|
||||
/// `tools` answers 400.
|
||||
fn strip_tools(req: &mut ChatRequest) {
|
||||
req.tools.clear();
|
||||
req.params.passthrough.remove("tool_choice");
|
||||
req.params.passthrough.remove("parallel_tool_calls");
|
||||
}
|
||||
|
||||
/// Is the client's `tool_choice` a demand rather than a preference?
|
||||
/// `"required"` and `{"type":"function", …}` mean the caller cannot be served
|
||||
/// without tools; `"auto"`, `"none"` and an absent field are just an offer.
|
||||
fn tool_choice_is_mandatory(passthrough: &serde_json::Map<String, serde_json::Value>) -> bool {
|
||||
match passthrough.get("tool_choice") {
|
||||
Some(serde_json::Value::String(s)) => s.eq_ignore_ascii_case("required"),
|
||||
Some(serde_json::Value::Object(_)) => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn now_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
@@ -756,6 +830,7 @@ fn cached_response(hit: &crate::cache::CacheHit) -> ChatResponse {
|
||||
router_request_id: None,
|
||||
cost_usd: 0.0,
|
||||
failover: false,
|
||||
tools_dropped: false,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1067,3 +1142,57 @@ async fn shadow_capture(
|
||||
judge_tokens,
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn map(json: serde_json::Value) -> serde_json::Map<String, serde_json::Value> {
|
||||
json.as_object().cloned().unwrap_or_default()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_offered_tool_list_is_not_a_demand() {
|
||||
// Open WebUI attaches its built-in tools with no tool_choice at all.
|
||||
assert!(!tool_choice_is_mandatory(&map(serde_json::json!({}))));
|
||||
assert!(!tool_choice_is_mandatory(&map(
|
||||
serde_json::json!({"tool_choice": "auto"})
|
||||
)));
|
||||
assert!(!tool_choice_is_mandatory(&map(
|
||||
serde_json::json!({"tool_choice": "none"})
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn required_and_named_tool_choice_are_demands() {
|
||||
assert!(tool_choice_is_mandatory(&map(
|
||||
serde_json::json!({"tool_choice": "required"})
|
||||
)));
|
||||
assert!(tool_choice_is_mandatory(&map(
|
||||
serde_json::json!({"tool_choice": {"type": "function", "function": {"name": "f"}}})
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stripping_tools_also_drops_their_companion_fields() {
|
||||
let mut req = ChatRequest {
|
||||
routing: RoutingDirective::Auto { profile: None },
|
||||
messages: vec![],
|
||||
tools: vec![serde_json::json!({"type": "function"})],
|
||||
params: GenParams {
|
||||
passthrough: map(
|
||||
serde_json::json!({"tool_choice": "auto", "parallel_tool_calls": true, "top_k": 40}),
|
||||
),
|
||||
..Default::default()
|
||||
},
|
||||
stream: false,
|
||||
meta: RequestMeta::default(),
|
||||
};
|
||||
strip_tools(&mut req);
|
||||
assert!(req.tools.is_empty());
|
||||
assert!(!req.params.passthrough.contains_key("tool_choice"));
|
||||
assert!(!req.params.passthrough.contains_key("parallel_tool_calls"));
|
||||
// unrelated passthrough must survive
|
||||
assert!(req.params.passthrough.contains_key("top_k"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,6 +42,9 @@ pub(crate) fn cortiq_headers(c: &RouteInfo) -> axum::http::HeaderMap {
|
||||
if let Some(id) = &c.router_request_id {
|
||||
put(&mut h, "X-Cortiq-Request-Id", id);
|
||||
}
|
||||
if c.tools_dropped {
|
||||
put(&mut h, "X-Cortiq-Tools-Dropped", "true");
|
||||
}
|
||||
h
|
||||
}
|
||||
|
||||
@@ -319,6 +322,9 @@ async fn handler(
|
||||
if let Some(rid) = &resp.cortiq.router_request_id {
|
||||
echo["router_request_id"] = serde_json::json!(rid);
|
||||
}
|
||||
if resp.cortiq.tools_dropped {
|
||||
echo["tools_dropped"] = serde_json::json!(true);
|
||||
}
|
||||
body["cortiq"] = echo;
|
||||
}
|
||||
|
||||
|
||||
@@ -164,6 +164,7 @@ impl Provider for AnthropicProvider {
|
||||
router_request_id: None,
|
||||
cost_usd: 0.0,
|
||||
failover: false,
|
||||
tools_dropped: false,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -167,6 +167,7 @@ impl Provider for OpenAiProvider {
|
||||
router_request_id: None,
|
||||
cost_usd: 0.0,
|
||||
failover: false,
|
||||
tools_dropped: false,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user