infosave2007
9fd4bdaa85
Docker / publish (push) Canceled after 0s
Release / crates-io (push) Canceled after 0s
Release / release (macos-14, aarch64-apple-darwin) (push) Canceled after 0s
Release / release (ubuntu-latest, x86_64-unknown-linux-gnu) (push) Canceled after 0s
Release / release (windows-latest, x86_64-pc-windows-msvc) (push) Canceled after 0s
Qwen-family templates write a function call into the generated text as
<tool_call>{...}</tool_call>. Runtimes disagree on what to do with it: some
parse it into structured tool_calls AND still stream the raw markup as
content - the client then gets the same call twice, once as prose it must not
show and once as the real thing - while others never parse it at all, so a
perfectly good call arrives as text no tool loop will run.
Settle it in the gateway. A streaming-aware filter strips the markup from
content (handling a marker split across SSE chunks, leaving ordinary text
containing "<" alone, and handing an unterminated block back as text), and
when nothing structured came with it the markup is promoted into a real
tool_calls entry with a generated call_... id. Markup that is not a JSON
object with a name is ignored so a stray marker cannot become a bogus call.
Streamed tool_calls are also brought to the OpenAI shape: index on every
entry, id and type exactly once per call so continuation deltas carrying only
an argument fragment stay legal, and function.arguments always a string. A
reply that carries tool calls now always ends with finish_reason "tool_calls",
whatever the upstream called it. The same cleanup runs on the non-streaming
path, so every protocol adapter benefits.
Verified end to end: Open WebUI 0.11.0 -> gateway -> managed CMF/Qwen with
native function calling. The model returns get_weather({"city":"Paris"}), Open
WebUI executes the tool, feeds the result back, and the assistant answers from
it - no markup in content, finish_reason tool_calls, stream closed with [DONE].
80 lines
2.2 KiB
TOML
80 lines
2.2 KiB
TOML
[package]
|
|
name = "cortiq-gateway"
|
|
version = "0.2.48"
|
|
edition = "2021"
|
|
rust-version = "1.75"
|
|
license = "Apache-2.0"
|
|
description = "Universal LLM gateway with intelligent routing and an embedded multilingual admin console"
|
|
repository = "https://github.com/infosave2007/cortiq-gateway"
|
|
homepage = "https://api.allaigate.com"
|
|
readme = "README.md"
|
|
keywords = ["llm", "gateway", "proxy", "openai", "routing"]
|
|
categories = ["web-programming::http-server", "command-line-utilities"]
|
|
# Files shipped in the published crate. The embedded admin SPA MUST be included
|
|
# because rust-embed compiles it in; docs/screenshots are excluded to keep the
|
|
# crate lean. The web/ subfolders are listed one by one rather than as web/**/*
|
|
# so stray scratch files dropped next to the SPA never end up published.
|
|
include = [
|
|
"src/**/*",
|
|
"web/index.html",
|
|
"web/favicon.svg",
|
|
"web/css/**/*",
|
|
"web/js/**/*",
|
|
"config/gateway.example.toml",
|
|
"README.md",
|
|
"CHANGELOG.md",
|
|
"LICENSE",
|
|
]
|
|
|
|
[dependencies]
|
|
# async runtime + HTTP server
|
|
tokio = { version = "1", features = ["full"] }
|
|
axum = { version = "0.7", features = ["macros"] }
|
|
tower = "0.5"
|
|
tower-http = { version = "0.6", features = ["limit", "trace", "cors"] }
|
|
|
|
# HTTP client to providers and to cortiq-router (rustls, no OpenSSL)
|
|
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls"] }
|
|
|
|
# serialization
|
|
serde = { version = "1", features = ["derive"] }
|
|
serde_json = "1"
|
|
toml = "0.8"
|
|
|
|
# hot-reloadable shared state + embedded admin SPA
|
|
arc-swap = "1"
|
|
rust-embed = "8"
|
|
mime_guess = "2"
|
|
|
|
# cross-platform CSPRNG for admin/key token generation (Linux/macOS/Windows)
|
|
getrandom = "0.2"
|
|
|
|
# streaming / async utils
|
|
futures = "0.3"
|
|
async-trait = "0.1"
|
|
async-stream = "0.3"
|
|
bytes = "1"
|
|
tokio-stream = "0.1"
|
|
|
|
# CLI / config
|
|
clap = { version = "4", features = ["derive"] }
|
|
# open browser on first run (optional)
|
|
open = { version = "5", optional = true }
|
|
|
|
# host metrics for the admin dashboard (CPU / RAM / disk)
|
|
sysinfo = "0.39"
|
|
|
|
# errors / logging / metrics
|
|
thiserror = "1"
|
|
anyhow = "1"
|
|
tracing = "0.1"
|
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
|
|
|
[features]
|
|
default = ["open-browser"]
|
|
open-browser = ["open"]
|
|
|
|
[profile.release]
|
|
lto = true
|
|
codegen-units = 1
|