feat: streaming (SSE) end-to-end

- Provider::chat_stream; OpenAI provider streams via stream_options.include_usage
- pipeline.run_stream: shared route/select, failover before first byte
- /v1/chat/completions forwards SSE verbatim with X-Cortiq-* headers; usage tapped for stats
- admin /test/stream + live streaming in Playground (stream toggle, 7 langs)
This commit is contained in:
infosave2007
2026-06-30 16:37:35 +03:00
parent 3d71b03788
commit 70d8cde258
17 changed files with 483 additions and 64 deletions
+17 -2
View File
@@ -7,8 +7,16 @@
pub mod anthropic;
pub mod openai;
use crate::error::GatewayError;
use crate::model::{ChatRequest, ChatResponse};
use async_trait::async_trait;
use bytes::Bytes;
use futures::stream::BoxStream;
/// A streaming chat response: raw Server-Sent-Events byte chunks in OpenAI wire
/// format (`data: {json}\n\n` … `data: [DONE]\n\n`). The gateway forwards these
/// to the client verbatim while tapping the final `usage` for statistics.
pub type ChatStream = BoxStream<'static, crate::error::Result<Bytes>>;
/// Model capabilities — checked at selection time (whether tools/vision are needed).
#[derive(Clone, Debug, Default)]
@@ -41,6 +49,13 @@ pub trait Provider: Send + Sync {
/// Non-streaming chat call.
async fn chat(&self, req: ChatRequest) -> crate::error::Result<ChatResponse>;
// TODO(v0.1): async fn chat_stream(...) -> Result<ChatStream> (SSE).
// TODO(v0.1): async fn embed(...) -> Result<EmbedResponse>.
/// Streaming chat call (SSE). Default: not supported (override per provider).
async fn chat_stream(&self, _req: ChatRequest) -> crate::error::Result<ChatStream> {
Err(GatewayError::UpstreamUnavailable(format!(
"streaming is not supported by provider '{}'",
self.id()
)))
}
// TODO(v0.2): async fn embed(...) -> Result<EmbedResponse>.
}