fix: honest costs in streams, oracle-sized timeouts, history & log cleanup
Cost showed $0 for every streamed request: the X-Cortiq-Cost-USD header is written before generation, when the cost cannot be known. The gateway now ships the final number in a closing SSE event (`cortiq_cost_usd`, alongside the token estimate when the provider sent no usage), and the console prefers it over the header. Money renders with 4 decimals, the last two in a smaller font — $0.0006 stays readable instead of collapsing to $0. Timeouts, both ends: the router default was 800 ms, silently degrading every prompt the hosted router escalates to its oracle (~10 s) — now 15 s. Provider clients had NO timeout at all (a dead OpenRouter host hung requests forever) — now a 10 s connect timeout, with no overall cap so long generations and streams are never cut. Benchmark history: per-run × and a Clear-all button. Logs view: a clear button backed by DELETE /admin/api/logs.
This commit is contained in:
+6
-1
@@ -185,7 +185,7 @@ pub fn api_routes(admin_token: String) -> Router<SharedState> {
|
||||
.route("/admin/api/requests", get(get_requests))
|
||||
.route("/admin/api/test", post(run_test))
|
||||
.route("/admin/api/test/stream", post(run_test_stream))
|
||||
.route("/admin/api/logs", get(get_logs))
|
||||
.route("/admin/api/logs", get(get_logs).delete(clear_logs))
|
||||
.route("/admin/api/system", get(get_system))
|
||||
.route("/admin/api/restart", post(restart_gateway))
|
||||
.route("/admin/api/bench", get(bench_list).post(bench_start))
|
||||
@@ -2113,6 +2113,11 @@ async fn get_logs(Query(q): Query<RecentQuery>) -> ApiResult {
|
||||
ok(json!({ "lines": crate::logbuf::snapshot(limit) }))
|
||||
}
|
||||
|
||||
async fn clear_logs() -> ApiResult {
|
||||
crate::logbuf::clear();
|
||||
ok(json!({ "ok": true }))
|
||||
}
|
||||
|
||||
/// Host metrics snapshot: CPU, RAM, free disk at the models dir, gateway RSS.
|
||||
async fn get_system(State(state): State<SharedState>) -> ApiResult {
|
||||
let snap = state.sysmon.snapshot();
|
||||
|
||||
+4
-2
@@ -305,7 +305,9 @@ fn default_router_enabled() -> bool {
|
||||
true
|
||||
}
|
||||
fn default_router_timeout() -> u64 {
|
||||
800
|
||||
// the hosted router escalates hard prompts to an oracle (~10 s);
|
||||
// an 800 ms budget silently degraded every one of those to the fallback
|
||||
15_000
|
||||
}
|
||||
fn default_verify_tls() -> bool {
|
||||
true
|
||||
@@ -736,7 +738,7 @@ impl Default for Config {
|
||||
router: RouterCfg {
|
||||
enabled: true,
|
||||
url: "https://138.226.222.209".to_string(),
|
||||
timeout_ms: 12_000, // hard prompts escalate to an oracle (~10 s)
|
||||
timeout_ms: 15_000, // hard prompts escalate to an oracle (~10 s)
|
||||
verify_tls: false, // the hosted endpoint is IP-bound
|
||||
api_key_env: Some("CORTIQ_ROUTER_KEY".to_string()),
|
||||
taxonomy_id: Some("data-assistant".to_string()),
|
||||
|
||||
@@ -23,6 +23,11 @@ pub fn snapshot(limit: usize) -> Vec<String> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Drop everything in the ring (the admin console's "clear" button).
|
||||
pub fn clear() {
|
||||
ring().lock().unwrap().clear();
|
||||
}
|
||||
|
||||
pub fn push_line(line: &str) {
|
||||
let line = line.trim_end();
|
||||
if line.is_empty() {
|
||||
|
||||
+24
-17
@@ -798,28 +798,35 @@ fn tap_stream(
|
||||
// inject one final OpenAI-shaped usage event with the gateway's
|
||||
// estimate (~4 chars/token), flagged `cortiq_estimated` so nobody
|
||||
// mistakes it for the tokenizer's truth.
|
||||
if done_marker && !got_usage {
|
||||
let est_completion = if delta_chunks > 0 {
|
||||
delta_chunks
|
||||
} else {
|
||||
(delta_chars / 4) as u32
|
||||
};
|
||||
let est_prompt = (prompt_chars / 4) as u32;
|
||||
let evt = serde_json::json!({
|
||||
if done_marker {
|
||||
// The initial X-Cortiq-Cost-USD header is always 0 — cost is
|
||||
// only known after generation. Ship the final number (and the
|
||||
// token estimate when the provider sent none) ahead of [DONE].
|
||||
if !got_usage {
|
||||
prompt_tokens = (prompt_chars / 4) as u32;
|
||||
completion_tokens = if delta_chunks > 0 {
|
||||
delta_chunks
|
||||
} else {
|
||||
(delta_chars / 4) as u32
|
||||
};
|
||||
}
|
||||
let cost = provider.price(prompt_tokens, completion_tokens);
|
||||
let mut evt = serde_json::json!({
|
||||
"object": "chat.completion.chunk",
|
||||
"model": info.selected_model,
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": est_prompt,
|
||||
"completion_tokens": est_completion,
|
||||
"total_tokens": est_prompt + est_completion,
|
||||
},
|
||||
"cortiq_estimated": true,
|
||||
"cortiq_cost_usd": cost,
|
||||
});
|
||||
if !got_usage {
|
||||
evt["usage"] = serde_json::json!({
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"completion_tokens": completion_tokens,
|
||||
"total_tokens": prompt_tokens + completion_tokens,
|
||||
});
|
||||
evt["cortiq_estimated"] = serde_json::json!(true);
|
||||
got_usage = true;
|
||||
}
|
||||
yield Ok(bytes::Bytes::from(format!("data: {evt}\n\n")));
|
||||
prompt_tokens = est_prompt;
|
||||
completion_tokens = est_completion;
|
||||
got_usage = true;
|
||||
}
|
||||
yield item;
|
||||
}
|
||||
|
||||
@@ -32,7 +32,13 @@ impl AnthropicProvider {
|
||||
caps: Caps::from_list(&cfg.caps),
|
||||
price_in: cfg.price_in,
|
||||
price_out: cfg.price_out,
|
||||
http: reqwest::Client::new(),
|
||||
// connect timeout: a dead host fails over in seconds instead of
|
||||
// hanging the request; NO overall timeout — that would cut long
|
||||
// generations and streams (OpenRouter can be slow legitimately)
|
||||
http: reqwest::Client::builder()
|
||||
.connect_timeout(std::time::Duration::from_secs(10))
|
||||
.build()
|
||||
.unwrap_or_else(|_| reqwest::Client::new()),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,13 @@ impl OpenAiProvider {
|
||||
caps: Caps::from_list(&cfg.caps),
|
||||
price_in: cfg.price_in,
|
||||
price_out: cfg.price_out,
|
||||
http: reqwest::Client::new(),
|
||||
// connect timeout: a dead host fails over in seconds instead of
|
||||
// hanging the request; NO overall timeout — that would cut long
|
||||
// generations and streams (OpenRouter can be slow legitimately)
|
||||
http: reqwest::Client::builder()
|
||||
.connect_timeout(std::time::Duration::from_secs(10))
|
||||
.build()
|
||||
.unwrap_or_else(|_| reqwest::Client::new()),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user