feat: streaming (SSE) end-to-end
- Provider::chat_stream; OpenAI provider streams via stream_options.include_usage - pipeline.run_stream: shared route/select, failover before first byte - /v1/chat/completions forwards SSE verbatim with X-Cortiq-* headers; usage tapped for stats - admin /test/stream + live streaming in Playground (stream toggle, 7 langs) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
220df29888
commit
b6fdf23aa6
@@ -165,6 +165,7 @@ pub fn api_routes(admin_token: String) -> Router<SharedState> {
|
||||
.route("/admin/api/stats", get(get_stats))
|
||||
.route("/admin/api/requests", get(get_requests))
|
||||
.route("/admin/api/test", post(run_test))
|
||||
.route("/admin/api/test/stream", post(run_test_stream))
|
||||
.route_layer(middleware::from_fn(move |req: Request, next: Next| {
|
||||
let expected = admin_token.clone();
|
||||
async move { auth(expected, req, next).await }
|
||||
@@ -753,3 +754,38 @@ async fn run_test(State(state): State<SharedState>, Json(b): Json<TestBody>) ->
|
||||
Err(e) => ok(json!({ "ok": false, "latency_ms": latency_ms, "error": e.to_string() })),
|
||||
}
|
||||
}
|
||||
|
||||
/// Streaming playground: forwards provider SSE chunks; routing decision is exposed
|
||||
/// via X-Cortiq-* response headers (the SPA reads them after the stream ends).
|
||||
async fn run_test_stream(State(state): State<SharedState>, Json(b): Json<TestBody>) -> Response {
|
||||
if b.messages.is_empty() {
|
||||
return ApiError::bad("messages must not be empty").into_response();
|
||||
}
|
||||
let req = ChatRequest {
|
||||
routing: parse_routing(&b.model),
|
||||
messages: b.messages,
|
||||
tools: vec![],
|
||||
params: GenParams {
|
||||
temperature: b.temperature,
|
||||
max_tokens: b.max_tokens,
|
||||
..Default::default()
|
||||
},
|
||||
stream: true,
|
||||
meta: RequestMeta {
|
||||
account: "playground".into(),
|
||||
protocol: "playground".into(),
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
match state.pipeline.run_stream(req, &state).await {
|
||||
Ok((info, stream)) => {
|
||||
let mut headers = crate::protocols::openai_chat::cortiq_headers(&info);
|
||||
headers.insert(
|
||||
header::CONTENT_TYPE,
|
||||
axum::http::HeaderValue::from_static("text/event-stream"),
|
||||
);
|
||||
(headers, axum::body::Body::from_stream(stream)).into_response()
|
||||
}
|
||||
Err(e) => ApiError::bad(e.to_string()).into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user