diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b36457..c33f43c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,27 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/). +## [0.2.46] - 2026-08-21 + +Verified end to end against a real Open WebUI container talking to the gateway. + +### Fixed +- **Offered `tools` no longer make a local model unroutable.** Open WebUI attaches its ~26 + built-in tools (knowledge bases, notes, calendar, …) to *every* message, and the gateway + treated that offer as a hard capability requirement — so a plain "hello" against a pool of + tools-less local models failed with «No healthy model available that satisfies required + capabilities». In the OpenAI contract `tools` is an offer: only `tool_choice: "required"` + (or a named function) makes the call impossible without them. The gateway now still + prefers a tool-capable model, but falls back to serving the request with the tools + stripped (along with `tool_choice`/`parallel_tool_calls`, which a provider rejects on + their own). `tool_choice: "required"` still fails loudly, as it must. +- **A pinned model that merely lacks tool support is honoured** instead of being reported as + «not available — it is not in the pool (deleted?)». An unknown pin still fails loudly. + +### Added +- `X-Cortiq-Tools-Dropped: true` (and `cortiq.tools_dropped` in the echo block) whenever the + gateway answers without the tools the client offered, so the downgrade is never silent. + ## [0.2.45] - 2026-08-21 ### Fixed diff --git a/Cargo.lock b/Cargo.lock index 86aff42..45a5bd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -285,7 +285,7 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "cortiq-gateway" -version = "0.2.45" +version = "0.2.46" dependencies = [ "anyhow", "arc-swap", diff --git a/Cargo.toml b/Cargo.toml index 2780a19..3c6ca09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cortiq-gateway" -version = "0.2.45" +version = "0.2.46" edition = "2021" rust-version = "1.75" license = "Apache-2.0" diff --git a/docs/PROTOCOLS.md b/docs/PROTOCOLS.md index 79a88b7..51c1312 100644 --- a/docs/PROTOCOLS.md +++ b/docs/PROTOCOLS.md @@ -65,6 +65,13 @@ concatenated, images and other non-text parts are skipped); `tools` may be `null cannot be read comes back as an OpenAI error envelope with HTTP 400 — never a bare 422, which OpenAI clients cannot parse (they show no answer at all). +`tools` is treated as an **offer**, not a requirement. The gateway prefers a tool-capable +model, but when none is available it answers without the tools rather than failing, and says +so via `X-Cortiq-Tools-Dropped: true` (chat UIs attach their built-in tools to every message, +so the strict reading made ordinary chat impossible on a tools-less local pool). +`tool_choice: "required"` — or a named function — is a real requirement and still fails with +`502 no_healthy_model` when no model can honour it. + ### Response (non-streaming) ```jsonc diff --git a/docs/PROTOCOLS.ru.md b/docs/PROTOCOLS.ru.md index ff0e4c8..45b688e 100644 --- a/docs/PROTOCOLS.ru.md +++ b/docs/PROTOCOLS.ru.md @@ -65,6 +65,13 @@ HTTP 400 — но не «голый» 422, который OpenAI-клиенты не умеют разбирать (для пользователя это выглядит как отсутствие ответа). +`tools` трактуется как **предложение**, а не требование. Шлюз предпочитает модель с +поддержкой tools, но если такой нет — отвечает без них, а не падает, и сообщает об этом +заголовком `X-Cortiq-Tools-Dropped: true` (чат-интерфейсы добавляют свои встроенные tools в +каждое сообщение, поэтому строгое прочтение делало обычный чат на локальном пуле без tools +невозможным). `tool_choice: "required"` — или именованная функция — это уже настоящее +требование, и оно по-прежнему завершается `502 no_healthy_model`, если его некому выполнить. + ### Ответ (без стриминга) ```jsonc diff --git a/src/model.rs b/src/model.rs index 27e0b1e..af78596 100644 --- a/src/model.rs +++ b/src/model.rs @@ -113,6 +113,11 @@ pub struct RouteInfo { pub cost_usd: f64, #[serde(default)] pub failover: bool, + /// The client offered `tools` but the model that answered cannot use them, + /// so they were dropped from the upstream call (see `Pipeline::resolve`). + /// Surfaced as `X-Cortiq-Tools-Dropped` so a silent downgrade is debuggable. + #[serde(default)] + pub tools_dropped: bool, } #[derive(Clone, Debug, Serialize)] diff --git a/src/pipeline.rs b/src/pipeline.rs index 0721156..325c498 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -186,7 +186,7 @@ impl Pipeline { // Tool support detected via the local servers' healthz handshake — // treated as equivalent to declared caps when filtering candidates. detected_tools: &std::collections::HashSet, - ) -> Result<(RouteDecision, String, Vec)> { + ) -> Result<(RouteDecision, String, Vec, bool)> { // 1. Extract text for classification let text = self.extract_text( &req.messages, @@ -277,6 +277,7 @@ impl Pipeline { // Filter candidates by capabilities (e.g. tools support) let needs_tools = !req.tools.is_empty(); + let tools_mandatory = needs_tools && tool_choice_is_mandatory(&req.params.passthrough); let has_tools = |id: &str, prov: &std::sync::Arc| { prov.caps().tools || detected_tools.contains(id) }; @@ -294,6 +295,31 @@ impl Pipeline { } } + // `tools` is an OFFER in the OpenAI contract, not a demand: only + // `tool_choice: "required"` (or a named function) makes the call + // impossible without them. Chat UIs lean on that — Open WebUI attaches + // its ~26 built-in tools to every single message — so treating the offer + // as a hard requirement made an ordinary "hello" unanswerable on a pool + // of tools-less local models. Serve the request without the tools and + // flag the downgrade (`X-Cortiq-Tools-Dropped`) instead of failing. + let mut tools_dropped = false; + + // A pin stays a pin: when the pinned model exists and merely lacks tool + // support, honour the pin and drop the offer rather than reporting the + // model as missing. + if final_candidates.is_empty() && needs_tools && !tools_mandatory { + if let RoutingDirective::Pinned { model_id } = &req.routing { + if live.registry.get(model_id).is_some() { + final_candidates.push(model_id.clone()); + tools_dropped = true; + tracing::info!( + model = %model_id, + "pinned model does not support the offered tools — serving without them" + ); + } + } + } + // An explicit pin must never silently answer from a different model: if // the pinned id is unknown or was filtered out, fail loudly so the caller // knows their choice was not honoured (instead of getting a stray reply @@ -326,7 +352,7 @@ impl Pipeline { // (an agentic request is worth a capable model), then the rest of the // pool. Coding agents (OpenCode, Cline, …) always send tools; without // this every low-tier verdict was a hard failure. - if final_candidates.is_empty() { + let widened_order = || -> Vec { let mut widened: Vec = Vec::new(); for tier in ["medium", "high"] { for id in live.routing.candidates(tier) { @@ -340,7 +366,10 @@ impl Pipeline { widened.push(id.clone()); } } - for id in widened { + widened + }; + if final_candidates.is_empty() { + for id in widened_order() { if let Some(prov) = live.registry.get(&id) { if needs_tools && !has_tools(&id, &prov) { continue; @@ -355,20 +384,40 @@ impl Pipeline { ); } } + + // Same relaxation for an AUTO route once every narrower pass came up empty. + if final_candidates.is_empty() && needs_tools && !tools_mandatory { + for id in widened_order() { + if live.registry.get(&id).is_some() { + final_candidates.push(id); + } + } + if !final_candidates.is_empty() { + tools_dropped = true; + tracing::info!( + chosen = %final_candidates[0], + "no model in the pool supports the offered tools — serving without them" + ); + } + } + if final_candidates.is_empty() { return Err(GatewayError::UpstreamUnavailable( "No healthy model available that satisfies required capabilities (e.g. tools)" .to_string(), )); } - Ok((decision, route_source, final_candidates)) + Ok((decision, route_source, final_candidates, tools_dropped)) } - async fn run_inner(&self, req: ChatRequest, state: &SharedState) -> Result { + async fn run_inner(&self, mut req: ChatRequest, state: &SharedState) -> Result { let live = state.live(); let detected_tools = state.cmf.detected_tools.lock().unwrap().clone(); - let (decision, route_source, mut final_candidates) = + let (decision, route_source, mut final_candidates, tools_dropped) = self.resolve(&req, &live, &detected_tools).await?; + if tools_dropped { + strip_tools(&mut req); + } // Serving-when-promoted (gated): a promoted task-label is served by the // local model FIRST (failover to cloud on error). Per-request VETO uses @@ -478,6 +527,7 @@ impl Pipeline { router_request_id: decision.router_request_id.clone(), cost_usd: cost, failover: attempt > 0, + tools_dropped, }; return Ok(resp); } @@ -504,7 +554,7 @@ impl Pipeline { /// recorded when the stream completes (tokens taken from the final usage chunk). pub async fn run_stream( &self, - req: ChatRequest, + mut req: ChatRequest, state: &SharedState, ) -> Result<(RouteInfo, crate::providers::ChatStream)> { let started = Instant::now(); @@ -518,7 +568,7 @@ impl Pipeline { let live = state.live(); let detected_tools = state.cmf.detected_tools.lock().unwrap().clone(); - let (decision, route_source, final_candidates) = + let (decision, route_source, final_candidates, tools_dropped) = match self.resolve(&req, &live, &detected_tools).await { Ok(t) => t, Err(e) => { @@ -526,6 +576,9 @@ impl Pipeline { return Err(e); } }; + if tools_dropped { + strip_tools(&mut req); + } let mut last_err = None; let mut attempt = 0usize; @@ -612,6 +665,7 @@ impl Pipeline { router_request_id: decision.router_request_id.clone(), cost_usd: 0.0, failover: attempt > 0, + tools_dropped, }; // for the client-facing usage estimate when the provider // sends none: the prompt size is known here, the reply is @@ -689,6 +743,26 @@ impl Pipeline { } } +/// Remove the tool offer from a request the chosen model cannot honour. The +/// companion fields must go too: an upstream given `tool_choice` with no +/// `tools` answers 400. +fn strip_tools(req: &mut ChatRequest) { + req.tools.clear(); + req.params.passthrough.remove("tool_choice"); + req.params.passthrough.remove("parallel_tool_calls"); +} + +/// Is the client's `tool_choice` a demand rather than a preference? +/// `"required"` and `{"type":"function", …}` mean the caller cannot be served +/// without tools; `"auto"`, `"none"` and an absent field are just an offer. +fn tool_choice_is_mandatory(passthrough: &serde_json::Map) -> bool { + match passthrough.get("tool_choice") { + Some(serde_json::Value::String(s)) => s.eq_ignore_ascii_case("required"), + Some(serde_json::Value::Object(_)) => true, + _ => false, + } +} + fn now_secs() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -756,6 +830,7 @@ fn cached_response(hit: &crate::cache::CacheHit) -> ChatResponse { router_request_id: None, cost_usd: 0.0, failover: false, + tools_dropped: false, }, } } @@ -1067,3 +1142,57 @@ async fn shadow_capture( judge_tokens, }); } + +#[cfg(test)] +mod tests { + use super::*; + + fn map(json: serde_json::Value) -> serde_json::Map { + json.as_object().cloned().unwrap_or_default() + } + + #[test] + fn an_offered_tool_list_is_not_a_demand() { + // Open WebUI attaches its built-in tools with no tool_choice at all. + assert!(!tool_choice_is_mandatory(&map(serde_json::json!({})))); + assert!(!tool_choice_is_mandatory(&map( + serde_json::json!({"tool_choice": "auto"}) + ))); + assert!(!tool_choice_is_mandatory(&map( + serde_json::json!({"tool_choice": "none"}) + ))); + } + + #[test] + fn required_and_named_tool_choice_are_demands() { + assert!(tool_choice_is_mandatory(&map( + serde_json::json!({"tool_choice": "required"}) + ))); + assert!(tool_choice_is_mandatory(&map( + serde_json::json!({"tool_choice": {"type": "function", "function": {"name": "f"}}}) + ))); + } + + #[test] + fn stripping_tools_also_drops_their_companion_fields() { + let mut req = ChatRequest { + routing: RoutingDirective::Auto { profile: None }, + messages: vec![], + tools: vec![serde_json::json!({"type": "function"})], + params: GenParams { + passthrough: map( + serde_json::json!({"tool_choice": "auto", "parallel_tool_calls": true, "top_k": 40}), + ), + ..Default::default() + }, + stream: false, + meta: RequestMeta::default(), + }; + strip_tools(&mut req); + assert!(req.tools.is_empty()); + assert!(!req.params.passthrough.contains_key("tool_choice")); + assert!(!req.params.passthrough.contains_key("parallel_tool_calls")); + // unrelated passthrough must survive + assert!(req.params.passthrough.contains_key("top_k")); + } +} diff --git a/src/protocols/openai_chat.rs b/src/protocols/openai_chat.rs index 6b29f70..5eb4cdd 100644 --- a/src/protocols/openai_chat.rs +++ b/src/protocols/openai_chat.rs @@ -42,6 +42,9 @@ pub(crate) fn cortiq_headers(c: &RouteInfo) -> axum::http::HeaderMap { if let Some(id) = &c.router_request_id { put(&mut h, "X-Cortiq-Request-Id", id); } + if c.tools_dropped { + put(&mut h, "X-Cortiq-Tools-Dropped", "true"); + } h } @@ -319,6 +322,9 @@ async fn handler( if let Some(rid) = &resp.cortiq.router_request_id { echo["router_request_id"] = serde_json::json!(rid); } + if resp.cortiq.tools_dropped { + echo["tools_dropped"] = serde_json::json!(true); + } body["cortiq"] = echo; } diff --git a/src/providers/anthropic.rs b/src/providers/anthropic.rs index bad2299..d7c0c66 100644 --- a/src/providers/anthropic.rs +++ b/src/providers/anthropic.rs @@ -164,6 +164,7 @@ impl Provider for AnthropicProvider { router_request_id: None, cost_usd: 0.0, failover: false, + tools_dropped: false, }, }) } diff --git a/src/providers/openai.rs b/src/providers/openai.rs index c0d1123..edbd2be 100644 --- a/src/providers/openai.rs +++ b/src/providers/openai.rs @@ -167,6 +167,7 @@ impl Provider for OpenAiProvider { router_request_id: None, cost_usd: 0.0, failover: false, + tools_dropped: false, }, }) }