feat: streaming (SSE) end-to-end
- Provider::chat_stream; OpenAI provider streams via stream_options.include_usage - pipeline.run_stream: shared route/select, failover before first byte - /v1/chat/completions forwards SSE verbatim with X-Cortiq-* headers; usage tapped for stats - admin /test/stream + live streaming in Playground (stream toggle, 7 langs)
This commit is contained in:
Vendored
+48
@@ -91,3 +91,51 @@ export const api = {
|
||||
// playground
|
||||
test: (b) => j("POST", "/test", b),
|
||||
};
|
||||
|
||||
// Streaming playground: POST /test/stream, calls onDelta(text) per content chunk,
|
||||
// returns the routing decision read from X-Cortiq-* response headers.
|
||||
export async function testStream(body, onDelta) {
|
||||
const r = await fetch(BASE + "/test/stream", {
|
||||
method: "POST",
|
||||
headers: { Authorization: "Bearer " + getToken(), "content-type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
if (r.status === 401) {
|
||||
if (onUnauthorized) onUnauthorized();
|
||||
throw new Error("unauthorized");
|
||||
}
|
||||
if (!r.ok || !r.body) {
|
||||
throw new Error((await r.text()) || r.statusText);
|
||||
}
|
||||
const reader = r.body.getReader();
|
||||
const dec = new TextDecoder();
|
||||
let buf = "";
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buf += dec.decode(value, { stream: true });
|
||||
let idx;
|
||||
while ((idx = buf.indexOf("\n\n")) >= 0) {
|
||||
const line = buf.slice(0, idx).split("\n").find((l) => l.startsWith("data:"));
|
||||
buf = buf.slice(idx + 2);
|
||||
if (!line) continue;
|
||||
const data = line.slice(5).trim();
|
||||
if (!data || data === "[DONE]") continue;
|
||||
try {
|
||||
const d = JSON.parse(data).choices?.[0]?.delta?.content;
|
||||
if (d) onDelta(d);
|
||||
} catch {
|
||||
/* ignore malformed chunk */
|
||||
}
|
||||
}
|
||||
}
|
||||
const h = r.headers;
|
||||
return {
|
||||
task_label: h.get("x-cortiq-task-label") || "",
|
||||
tier: h.get("x-cortiq-complexity-tier") || "",
|
||||
score: parseFloat(h.get("x-cortiq-complexity-score") || "0"),
|
||||
selected_model: h.get("x-cortiq-selected-model") || "",
|
||||
route_source: h.get("x-cortiq-route-source") || "",
|
||||
cost_usd: parseFloat(h.get("x-cortiq-cost-usd") || "0"),
|
||||
};
|
||||
}
|
||||
|
||||
Vendored
+1
@@ -200,6 +200,7 @@ export default {
|
||||
"pg.temp": "Temperatur",
|
||||
"pg.maxTokens": "Max. Tokens",
|
||||
"pg.send": "Senden",
|
||||
"pg.stream": "Streamen",
|
||||
"pg.sending": "Wird gesendet…",
|
||||
"pg.answer": "Antwort",
|
||||
"pg.empty": "Antwort und Routing-Entscheidung erscheinen hier.",
|
||||
|
||||
Vendored
+1
@@ -201,6 +201,7 @@ export default {
|
||||
"pg.temp": "Temperature",
|
||||
"pg.maxTokens": "Max tokens",
|
||||
"pg.send": "Send",
|
||||
"pg.stream": "Stream",
|
||||
"pg.sending": "Sending…",
|
||||
"pg.answer": "Answer",
|
||||
"pg.empty": "The answer and routing decision will appear here.",
|
||||
|
||||
Vendored
+1
@@ -200,6 +200,7 @@ export default {
|
||||
"pg.temp": "Temperatura",
|
||||
"pg.maxTokens": "Tokens máx.",
|
||||
"pg.send": "Enviar",
|
||||
"pg.stream": "Streaming",
|
||||
"pg.sending": "Enviando…",
|
||||
"pg.answer": "Respuesta",
|
||||
"pg.empty": "La respuesta y la decisión de enrutamiento aparecerán aquí.",
|
||||
|
||||
Vendored
+1
@@ -200,6 +200,7 @@ export default {
|
||||
"pg.temp": "Température",
|
||||
"pg.maxTokens": "Tokens max",
|
||||
"pg.send": "Envoyer",
|
||||
"pg.stream": "Streaming",
|
||||
"pg.sending": "Envoi…",
|
||||
"pg.answer": "Réponse",
|
||||
"pg.empty": "La réponse et la décision de routage s'afficheront ici.",
|
||||
|
||||
Vendored
+1
@@ -200,6 +200,7 @@ export default {
|
||||
"pg.temp": "Температура",
|
||||
"pg.maxTokens": "Макс. токены",
|
||||
"pg.send": "Отправить",
|
||||
"pg.stream": "Стриминг",
|
||||
"pg.sending": "Отправка…",
|
||||
"pg.answer": "Ответ",
|
||||
"pg.empty": "Ответ и решение роутера появятся здесь.",
|
||||
|
||||
Vendored
+1
@@ -200,6 +200,7 @@ export default {
|
||||
"pg.temp": "Sıcaklık",
|
||||
"pg.maxTokens": "Maks. token",
|
||||
"pg.send": "Gönder",
|
||||
"pg.stream": "Akış",
|
||||
"pg.sending": "Gönderiliyor…",
|
||||
"pg.answer": "Yanıt",
|
||||
"pg.empty": "Yanıt ve yönlendirme kararı burada görünecek.",
|
||||
|
||||
Vendored
+1
@@ -200,6 +200,7 @@ export default {
|
||||
"pg.temp": "温度",
|
||||
"pg.maxTokens": "最大 Token 数",
|
||||
"pg.send": "发送",
|
||||
"pg.stream": "流式",
|
||||
"pg.sending": "发送中…",
|
||||
"pg.answer": "回答",
|
||||
"pg.empty": "回答和路由决策将在此显示。",
|
||||
|
||||
Vendored
+44
-14
@@ -1,7 +1,7 @@
|
||||
// Playground — send a prompt through the live pipeline, inspect routing decision.
|
||||
import { h, mount, money, ms, num } from "../ui.js";
|
||||
import { h, mount, money, ms } from "../ui.js";
|
||||
import { t } from "../i18n.js";
|
||||
import { api } from "../api.js";
|
||||
import { api, testStream } from "../api.js";
|
||||
import { appState } from "../app.js";
|
||||
|
||||
export async function renderPlayground() {
|
||||
@@ -16,6 +16,7 @@ export async function renderPlayground() {
|
||||
const promptIn = h("textarea", { placeholder: t("pg.promptPlaceholder"), rows: 5 });
|
||||
const tempIn = h("input", { type: "number", step: "0.1", value: 0.7 });
|
||||
const maxTokIn = h("input", { type: "number", value: 512 });
|
||||
const streamCb = h("input", { type: "checkbox", checked: true });
|
||||
const profileSel = h("select", {}, ...(meta.profiles || []).map((p) => h("option", { value: p }, p)));
|
||||
const modelSel = h("select", {}, ...ids.map((id) => h("option", { value: id }, id)));
|
||||
|
||||
@@ -62,19 +63,43 @@ export async function renderPlayground() {
|
||||
sendBtn.disabled = true;
|
||||
sendBtn.textContent = t("pg.sending");
|
||||
mount(out, h("div", { class: "flex" }, h("span", { class: "spinner" }), " ", t("pg.sending")));
|
||||
const body = {
|
||||
model,
|
||||
messages,
|
||||
temperature: parseFloat(tempIn.value),
|
||||
max_tokens: parseInt(maxTokIn.value) || null,
|
||||
};
|
||||
try {
|
||||
const r = await api.test({
|
||||
model,
|
||||
messages,
|
||||
temperature: parseFloat(tempIn.value),
|
||||
max_tokens: parseInt(maxTokIn.value) || null,
|
||||
});
|
||||
if (r.ok) {
|
||||
mount(out, r.answer || h("span", { class: "muted" }, "—"));
|
||||
renderDecision(r);
|
||||
if (streamCb.checked) {
|
||||
const started = performance.now();
|
||||
let acc = "";
|
||||
out.textContent = "";
|
||||
const info = await testStream(body, (d) => {
|
||||
acc += d;
|
||||
out.textContent = acc;
|
||||
});
|
||||
out.textContent = acc || "—";
|
||||
renderDecision({
|
||||
cortiq: {
|
||||
task_label: info.task_label,
|
||||
complexity: { score: info.score, tier: info.tier },
|
||||
selected_model: info.selected_model,
|
||||
route_source: info.route_source,
|
||||
cost_usd: info.cost_usd,
|
||||
failover: false,
|
||||
},
|
||||
usage: {},
|
||||
latency_ms: Math.round(performance.now() - started),
|
||||
});
|
||||
} else {
|
||||
mount(out, h("div", { class: "callout" }, h("b", {}, t("pg.error") + ": "), r.error || ""));
|
||||
mount(decision, h("div", { class: "card-head" }, h("h3", {}, t("pg.decision"))), h("div", { class: "empty" }, "—"));
|
||||
const r = await api.test(body);
|
||||
if (r.ok) {
|
||||
mount(out, r.answer || h("span", { class: "muted" }, "—"));
|
||||
renderDecision(r);
|
||||
} else {
|
||||
mount(out, h("div", { class: "callout" }, h("b", {}, t("pg.error") + ": "), r.error || ""));
|
||||
mount(decision, h("div", { class: "card-head" }, h("h3", {}, t("pg.decision"))), h("div", { class: "empty" }, "—"));
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
mount(out, h("div", { class: "callout" }, String(e.message || e)));
|
||||
@@ -126,7 +151,12 @@ export async function renderPlayground() {
|
||||
h("label", { class: "field" }, h("span", {}, t("pg.mode")), modeSeg),
|
||||
modeExtra,
|
||||
h("div", { class: "row" }, h("label", { class: "field" }, h("span", {}, t("pg.temp")), tempIn), h("label", { class: "field" }, h("span", {}, t("pg.maxTokens")), maxTokIn)),
|
||||
h("div", { class: "flex", style: "justify-content:flex-end" }, sendBtn)
|
||||
h(
|
||||
"div",
|
||||
{ class: "flex", style: "justify-content:space-between" },
|
||||
h("label", { class: "check", style: "margin:0" }, streamCb, t("pg.stream")),
|
||||
sendBtn
|
||||
)
|
||||
),
|
||||
h("div", { class: "card" }, h("div", { class: "card-head" }, h("h3", {}, t("pg.answer"))), out)
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user