github/cortiq-gateway
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cmf): auto-detect tool support of managed local servers

Browse Source 
cortiq serve >=0.5.62 advertises the loaded model's capabilities in
/healthz ({"capabilities":{"tools":true}} when the chat template has a
tools branch). The gateway now reads that handshake when the managed
server becomes healthy and treats detected tool support as equivalent
to declared caps in the candidate filter — so a low-tier chain made of
a local model no longer widens agent (tools) traffic to cloud models
once the local server can actually drive tools.

Also adds an optional manual "caps" override to [[cmf.servers]] for
older serve builds that do not advertise capabilities.
This commit is contained in:
infosave2007
2026-08-08 15:52:20 +03:00
parent dbcd9827f6
commit 19f666e7b1
5 changed files with 49 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pipeline.rs
+15 -5
View File
@@ -183,6 +183,9 @@ impl Pipeline {
&self,
req: &ChatRequest,
live: &crate::state::Live,
// Tool support detected via the local servers' healthz handshake —
// treated as equivalent to declared caps when filtering candidates.
detected_tools: &std::collections::HashSet<String>,
) -> Result<(RouteDecision, String, Vec<String>)> {
// 1. Extract text for classification
let text = self.extract_text(
@@ -274,10 +277,13 @@ impl Pipeline {
// Filter candidates by capabilities (e.g. tools support)
let needs_tools = !req.tools.is_empty();
let has_tools = |id: &str, prov: &std::sync::Arc<dyn crate::providers::Provider>| {
prov.caps().tools || detected_tools.contains(id)
};
let mut final_candidates = Vec::new();
for model_id in candidates {
if let Some(prov) = live.registry.get(&model_id) {
if needs_tools && !prov.caps().tools {
if needs_tools && !has_tools(&model_id, &prov) {
tracing::debug!(
"Candidate '{}' skipped: lacks support for requested tools",
model_id
@@ -306,7 +312,7 @@ impl Pipeline {
if final_candidates.is_empty() && live.cfg.cmf.manage_server {
for id in live.local_candidates() {
if let Some(prov) = live.registry.get(&id) {
if needs_tools && !prov.caps().tools {
if needs_tools && !has_tools(&id, &prov) {
continue;
}
final_candidates.push(id);
@@ -336,7 +342,7 @@ impl Pipeline {
}
for id in widened {
if let Some(prov) = live.registry.get(&id) {
if needs_tools && !prov.caps().tools {
if needs_tools && !has_tools(&id, &prov) {
continue;
}
final_candidates.push(id);
@@ -360,7 +366,9 @@ impl Pipeline {
async fn run_inner(&self, req: ChatRequest, state: &SharedState) -> Result<ChatResponse> {
let live = state.live();
let (decision, route_source, mut final_candidates) = self.resolve(&req, &live).await?;
let detected_tools = state.cmf.detected_tools.lock().unwrap().clone();
let (decision, route_source, mut final_candidates) =
self.resolve(&req, &live, &detected_tools).await?;
// Serving-when-promoted (gated): a promoted task-label is served by the
// local model FIRST (failover to cloud on error). Per-request VETO uses
@@ -507,7 +515,9 @@ impl Pipeline {
.to_string();
let live = state.live();
let (decision, route_source, final_candidates) = match self.resolve(&req, &live).await {
let detected_tools = state.cmf.detected_tools.lock().unwrap().clone();
let (decision, route_source, final_candidates) =
match self.resolve(&req, &live, &detected_tools).await {
Ok(t) => t,
Err(e) => {
record_error(state, &account, &protocol, &directive, started, &e);