infosave2007
6466006de6
- bench/ reproducible harness: instant mock backend + Apache Bench (ab -k -r) - measures gateway overhead in passthrough mode (apples-to-apples) - BENCHMARKS.md: methodology, caveats, results (cortiq ~57k rps p99 1ms vs Portkey ~5.8k p99 9ms vs LiteLLM 4-workers ~1.2k p99 59ms) - README (EN+RU): Performance section + highlight
27 lines
512 B
TOML
27 lines
512 B
TOML
# Benchmark config: one model pinned to the mock upstream. Requests use
|
|
# model="bench" (a real id) → direct passthrough, no router call — so we measure
|
|
# pure proxy overhead, comparable to LiteLLM/Portkey passthrough.
|
|
listen = "0.0.0.0:9100"
|
|
|
|
[router]
|
|
url = "http://127.0.0.1:8100"
|
|
|
|
[[models]]
|
|
id = "bench"
|
|
provider = "openai"
|
|
base_url = "http://127.0.0.1:8100/v1"
|
|
model = "bench-model"
|
|
|
|
[routing]
|
|
default = "bench"
|
|
|
|
[protocols]
|
|
openai_chat = true
|
|
|
|
[admin]
|
|
enabled = false
|
|
|
|
[stats]
|
|
enabled = true
|
|
file = ""
|