infosave2007
0b071279ac
CI / lint (push) Canceled after 0s
CI / test (macos-14) (push) Canceled after 0s
CI / test (ubuntu-latest) (push) Canceled after 0s
CI / test (windows-latest) (push) Canceled after 0s
Release / crates-io (push) Canceled after 0s
Release / release (macos-14, aarch64-apple-darwin) (push) Canceled after 0s
Release / release (ubuntu-latest, x86_64-unknown-linux-gnu) (push) Canceled after 0s
Release / release (windows-latest, x86_64-pc-windows-msvc) (push) Canceled after 0s
Docker / publish (push) Canceled after 0s
Every stream used to carry two chunks with an empty choices array: usage,
which the upstream emits BEFORE the stop reason, and a cortiq_cost_usd chunk
the gateway added itself. OpenAI ships the usage chunk only when the request
sets stream_options: {"include_usage": true}, and chat UIs do not set it -
which is why the same tool, in the same Open WebUI, sees neither chunk from
Groq and two from us.
Honour the flag. With no stream_options a stream is now exactly
role -> content... -> tool_calls -> finish_reason -> [DONE], with no
choice-less chunks at all. When usage is requested it ships once, after the
stop reason, with cortiq_cost_usd folded into the same chunk rather than
trailing behind it.
Token accounting, cost and the X-Cortiq-* headers are untouched: those numbers
were never read back off the wire.
Verified: the default stream matches the reference shape byte for byte, the
opt-in stream matches OpenAI's, the official OpenAI SDK reassembles the
streamed tool call, and the full Open WebUI 0.11.0 tool loop still runs the
tool and answers from its result.
80 lines
2.2 KiB
TOML
80 lines
2.2 KiB
TOML
[package]
|
|
name = "cortiq-gateway"
|
|
version = "0.2.49"
|
|
edition = "2021"
|
|
rust-version = "1.75"
|
|
license = "Apache-2.0"
|
|
description = "Universal LLM gateway with intelligent routing and an embedded multilingual admin console"
|
|
repository = "https://github.com/infosave2007/cortiq-gateway"
|
|
homepage = "https://api.allaigate.com"
|
|
readme = "README.md"
|
|
keywords = ["llm", "gateway", "proxy", "openai", "routing"]
|
|
categories = ["web-programming::http-server", "command-line-utilities"]
|
|
# Files shipped in the published crate. The embedded admin SPA MUST be included
|
|
# because rust-embed compiles it in; docs/screenshots are excluded to keep the
|
|
# crate lean. The web/ subfolders are listed one by one rather than as web/**/*
|
|
# so stray scratch files dropped next to the SPA never end up published.
|
|
include = [
|
|
"src/**/*",
|
|
"web/index.html",
|
|
"web/favicon.svg",
|
|
"web/css/**/*",
|
|
"web/js/**/*",
|
|
"config/gateway.example.toml",
|
|
"README.md",
|
|
"CHANGELOG.md",
|
|
"LICENSE",
|
|
]
|
|
|
|
[dependencies]
|
|
# async runtime + HTTP server
|
|
tokio = { version = "1", features = ["full"] }
|
|
axum = { version = "0.7", features = ["macros"] }
|
|
tower = "0.5"
|
|
tower-http = { version = "0.6", features = ["limit", "trace", "cors"] }
|
|
|
|
# HTTP client to providers and to cortiq-router (rustls, no OpenSSL)
|
|
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls"] }
|
|
|
|
# serialization
|
|
serde = { version = "1", features = ["derive"] }
|
|
serde_json = "1"
|
|
toml = "0.8"
|
|
|
|
# hot-reloadable shared state + embedded admin SPA
|
|
arc-swap = "1"
|
|
rust-embed = "8"
|
|
mime_guess = "2"
|
|
|
|
# cross-platform CSPRNG for admin/key token generation (Linux/macOS/Windows)
|
|
getrandom = "0.2"
|
|
|
|
# streaming / async utils
|
|
futures = "0.3"
|
|
async-trait = "0.1"
|
|
async-stream = "0.3"
|
|
bytes = "1"
|
|
tokio-stream = "0.1"
|
|
|
|
# CLI / config
|
|
clap = { version = "4", features = ["derive"] }
|
|
# open browser on first run (optional)
|
|
open = { version = "5", optional = true }
|
|
|
|
# host metrics for the admin dashboard (CPU / RAM / disk)
|
|
sysinfo = "0.39"
|
|
|
|
# errors / logging / metrics
|
|
thiserror = "1"
|
|
anyhow = "1"
|
|
tracing = "0.1"
|
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
|
|
|
[features]
|
|
default = ["open-browser"]
|
|
open-browser = ["open"]
|
|
|
|
[profile.release]
|
|
lto = true
|
|
codegen-units = 1
|