feat(playground): image & video generation tabs + real chat metrics

The infosave collection now ships media models (Lumina-Image-2.0, MiniMax-H3
Turbo) that cortiq runs via `imagine`/`animate` CLI, not `cortiq serve`. The
gateway grows a media API: local .cmf files are classified by `cortiq info`
(chat/image/video, cached), generations run as tracked jobs with step progress
parsed from stdout, one at a time, cancellable, outputs under <data>/media.

The playground becomes three tabs. Image: width/height/steps/CFG/seed, live
progress, result rendered from P6 PPM to a canvas in the browser (PNG export
via toBlob). Video: frames/steps/quality/seed plus optional first/last
keyframes (rasterized to PPM client-side); the MJPEG+PCM AVI is decoded in JS
and played on a canvas synced to the wav — no ffmpeg on either side (mp4 remux
happens opportunistically when ffmpeg exists). A metrics card shows resolution,
steps, seed, total time, time per step/frame and file size.

Chat metrics: the local cortiq server sends no usage chunk in streams, so
token counts showed 0/0. tap_stream now injects one OpenAI-shaped usage event
ahead of [DONE] with the gateway's ~4-chars/token estimate, flagged
`cortiq_estimated`; the UI renders it as "≈ N / M" and adds tok/s and
time-to-first-token. Registration skips media packs (a `cortiq serve` entry
for a diffusion model would only fail) in both Register and the wizard's
Finish.

36 new UI strings translated across all 7 languages.
This commit is contained in:
infosave2007
2026-08-07 15:55:06 +03:00
parent 32ce2f7ed7
commit a77beeabab
16 changed files with 1814 additions and 30 deletions
+6
View File
@@ -585,6 +585,12 @@ body.modal-open { overflow: hidden; }
/* Featured gallery: the list grows as new models are published, so give it more
room than the search grid before it starts scrolling. */
.featured-section .hf-grid { max-height: min(62vh, 760px); }
/* Playground media tabs: generated image / video frame */
.media-out { display: flex; flex-direction: column; gap: 8px; }
.media-canvas {
max-width: 100%; height: auto; border-radius: 8px;
border: 1px solid var(--border, #2a2f3a); background: #000;
}
/* Live download progress inside a featured card (empty until a job runs) */
.card-prog:empty { display: none; }
.card-prog { margin-top: 6px; }
+47 -2
View File
@@ -118,6 +118,14 @@ export const api = {
registerImport: (job) =>
j("POST", "/import/" + encodeURIComponent(job) + "/register"),
// media generation (cortiq imagine / animate over local .cmf)
mediaModels: () => j("GET", "/media/models"),
mediaGenerate: (b) => j("POST", "/media", b),
mediaJobs: () => j("GET", "/media"),
mediaJob: (id) => j("GET", "/media/" + encodeURIComponent(id)),
mediaCancel: (id) => j("POST", "/media/" + encodeURIComponent(id) + "/cancel"),
mediaDelete: (id) => j("DELETE", "/media/" + encodeURIComponent(id)),
// first-run onboarding
setupStatus: () => j("GET", "/setup/status"),
// Connects every finished download as a local model unless register === false.
@@ -155,6 +163,10 @@ export async function testStream(body, onDelta) {
const reader = r.body.getReader();
const dec = new TextDecoder();
let buf = "";
const t0 = performance.now();
let ttftMs = null; // time to first content token
let usage = null; // OpenAI-style final-chunk usage, when the provider sends it
let usageEstimated = false; // gateway-injected estimate rather than tokenizer truth
for (;;) {
const { done, value } = await reader.read();
if (done) break;
@@ -167,8 +179,16 @@ export async function testStream(body, onDelta) {
const data = line.slice(5).trim();
if (!data || data === "[DONE]") continue;
try {
const d = JSON.parse(data).choices?.[0]?.delta?.content;
if (d) onDelta(d);
const chunk = JSON.parse(data);
if (chunk.usage) {
usage = chunk.usage;
usageEstimated = !!chunk.cortiq_estimated;
}
const d = chunk.choices?.[0]?.delta?.content;
if (d) {
if (ttftMs == null) ttftMs = Math.round(performance.now() - t0);
onDelta(d);
}
} catch {
/* ignore malformed chunk */
}
@@ -182,5 +202,30 @@ export async function testStream(body, onDelta) {
selected_model: h.get("x-cortiq-selected-model") || "",
route_source: h.get("x-cortiq-route-source") || "",
cost_usd: parseFloat(h.get("x-cortiq-cost-usd") || "0"),
usage,
usage_estimated: usageEstimated,
ttft_ms: ttftMs,
};
}
// Fetch a media job's output with the admin token and hand back an object URL
// (plain <img src>/<video src> cannot carry the Authorization header).
export async function mediaFileUrl(job, kind) {
const tok = getToken();
const headers = {};
if (tok) headers["Authorization"] = "Bearer " + tok;
const r = await fetch(`${BASE}/media/${encodeURIComponent(job)}/file/${encodeURIComponent(kind)}`, { headers });
if (!r.ok) throw new Error((await r.text()) || r.statusText);
return { url: URL.createObjectURL(await r.blob()), size: +(r.headers.get("content-length") || 0) };
}
// Same fetch, raw bytes — for outputs the browser cannot display natively
// (P6 PPM, MJPEG AVI) that we decode in JS.
export async function mediaFileBytes(job, kind) {
const tok = getToken();
const headers = {};
if (tok) headers["Authorization"] = "Bearer " + tok;
const r = await fetch(`${BASE}/media/${encodeURIComponent(job)}/file/${encodeURIComponent(kind)}`, { headers });
if (!r.ok) throw new Error((await r.text()) || r.statusText);
return r.arrayBuffer();
}
+38
View File
@@ -468,4 +468,42 @@ export default {
"onboarding.deleteFile": "Die .cmf-Datei von der Festplatte löschen",
"onboarding.confirmDelete": "„{id}“ löschen? Die .cmf-Datei wird von der Festplatte entfernt.",
"onboarding.unregistered": "Ebenfalls abgemeldet: {ids}",
// playground: media generation + streaming metrics
"pg.tab.chat": "Chat",
"pg.tab.image": "Bild",
"pg.tab.video": "Video",
"pg.generate": "Generieren",
"pg.generating": "Wird generiert…",
"pg.genFailed": "Generierung fehlgeschlagen — siehe Log.",
"pg.result": "Ergebnis",
"pg.genMetrics": "Generierungsmetriken",
"pg.imagePromptPlaceholder": "ein Rotfuchs im Schnee bei Sonnenuntergang, fotorealistisch, detailliertes Fell",
"pg.videoPromptPlaceholder": "ein Corgi mit Kochmütze wirft einen Pfannkuchen, Brutzeln und ein fröhliches Bellen",
"pg.imageEmpty": "Das generierte Bild erscheint hier.",
"pg.videoEmpty": "Der generierte Clip (mit Ton) erscheint hier.",
"pg.width": "Breite",
"pg.height": "Höhe",
"pg.steps": "Schritte",
"pg.seed": "Seed",
"pg.randomSeed": "Zufälliger Seed",
"pg.cfgHint": "Guidance-Stärke; ≤ 1 schaltet CFG ab und halbiert die Arbeit.",
"pg.frames": "Frames",
"pg.framesHint": "Bei 24 fps; auf das Raster des Modells gerundet.",
"pg.quality": "JPEG-Qualität",
"pg.videoSizeHint": "Vielfache von 32; die trainierte kurze Kante ist 768.",
"pg.videoStepsHint": "Die Turbo-LoRA ist auf 4 Schritte trainiert.",
"pg.firstFrame": "Erstes Frame (optional)",
"pg.lastFrame": "Letztes Frame (optional)",
"pg.keyframeHint": "Macht den Lauf zu Keyframe-to-Video.",
"pg.resolution": "Auflösung",
"pg.genTime": "Generierungszeit",
"pg.timePerStep": "Pro Schritt",
"pg.timePerFrame": "Pro Frame",
"pg.fileSize": "Dateigröße",
"pg.duration": "Dauer",
"pg.play": "Abspielen",
"pg.noMediaModels": "Noch kein lokales {kind}-Modell. Lade eines (z. B. aus der infosave-Sammlung) unter Import herunter:",
"pg.tokSpeed": "Geschwindigkeit",
"pg.ttft": "Erstes Token",
};
+38
View File
@@ -461,4 +461,42 @@ export default {
"onboarding.deleteFile": "Delete the .cmf file from disk",
"onboarding.confirmDelete": "Delete '{id}'? The .cmf file will be removed from disk.",
"onboarding.unregistered": "Also unregistered: {ids}",
// playground: media generation + streaming metrics
"pg.tab.chat": "Chat",
"pg.tab.image": "Image",
"pg.tab.video": "Video",
"pg.generate": "Generate",
"pg.generating": "Generating…",
"pg.genFailed": "Generation failed — see the log.",
"pg.result": "Result",
"pg.genMetrics": "Generation metrics",
"pg.imagePromptPlaceholder": "a red fox sitting in snow at sunset, photorealistic, detailed fur",
"pg.videoPromptPlaceholder": "a corgi in a chef hat flipping a pancake, sizzling sounds and a cheerful bark",
"pg.imageEmpty": "The generated image will appear here.",
"pg.videoEmpty": "The generated clip (with audio) will appear here.",
"pg.width": "Width",
"pg.height": "Height",
"pg.steps": "Steps",
"pg.seed": "Seed",
"pg.randomSeed": "Random seed",
"pg.cfgHint": "Guidance scale; ≤ 1 disables CFG and halves the work.",
"pg.frames": "Frames",
"pg.framesHint": "At 24 fps; snapped to the model's grid.",
"pg.quality": "JPEG quality",
"pg.videoSizeHint": "Multiples of 32; the trained short edge is 768.",
"pg.videoStepsHint": "The Turbo LoRA is trained for 4 steps.",
"pg.firstFrame": "First frame (optional)",
"pg.lastFrame": "Last frame (optional)",
"pg.keyframeHint": "Turns the run into keyframe-to-video.",
"pg.resolution": "Resolution",
"pg.genTime": "Generation time",
"pg.timePerStep": "Per step",
"pg.timePerFrame": "Per frame",
"pg.fileSize": "File size",
"pg.duration": "Duration",
"pg.play": "Play",
"pg.noMediaModels": "No local {kind} model yet. Download one (e.g. from the infosave collection) in Import:",
"pg.tokSpeed": "Speed",
"pg.ttft": "First token",
};
+38
View File
@@ -468,4 +468,42 @@ export default {
"onboarding.deleteFile": "Eliminar el archivo .cmf del disco",
"onboarding.confirmDelete": "¿Eliminar «{id}»? El archivo .cmf se borrará del disco.",
"onboarding.unregistered": "También se dio de baja: {ids}",
// playground: media generation + streaming metrics
"pg.tab.chat": "Chat",
"pg.tab.image": "Imagen",
"pg.tab.video": "Vídeo",
"pg.generate": "Generar",
"pg.generating": "Generando…",
"pg.genFailed": "La generación falló — revisa el registro.",
"pg.result": "Resultado",
"pg.genMetrics": "Métricas de generación",
"pg.imagePromptPlaceholder": "un zorro rojo en la nieve al atardecer, fotorrealista, pelaje detallado",
"pg.videoPromptPlaceholder": "un corgi con gorro de chef voltea una tortita, chisporroteo y un ladrido alegre",
"pg.imageEmpty": "La imagen generada aparecerá aquí.",
"pg.videoEmpty": "El clip generado (con audio) aparecerá aquí.",
"pg.width": "Ancho",
"pg.height": "Alto",
"pg.steps": "Pasos",
"pg.seed": "Seed",
"pg.randomSeed": "Seed aleatorio",
"pg.cfgHint": "Escala de guidance; ≤ 1 desactiva el CFG y reduce el trabajo a la mitad.",
"pg.frames": "Fotogramas",
"pg.framesHint": "A 24 fps; se ajusta a la rejilla del modelo.",
"pg.quality": "Calidad JPEG",
"pg.videoSizeHint": "Múltiplos de 32; el lado corto entrenado es 768.",
"pg.videoStepsHint": "La LoRA Turbo está entrenada para 4 pasos.",
"pg.firstFrame": "Primer fotograma (opcional)",
"pg.lastFrame": "Último fotograma (opcional)",
"pg.keyframeHint": "Convierte la ejecución en keyframe-to-video.",
"pg.resolution": "Resolución",
"pg.genTime": "Tiempo de generación",
"pg.timePerStep": "Por paso",
"pg.timePerFrame": "Por fotograma",
"pg.fileSize": "Tamaño del archivo",
"pg.duration": "Duración",
"pg.play": "Reproducir",
"pg.noMediaModels": "Aún no hay un modelo {kind} local. Descarga uno (p. ej. de la colección infosave) en Importar:",
"pg.tokSpeed": "Velocidad",
"pg.ttft": "Primer token",
};
+38
View File
@@ -468,4 +468,42 @@ export default {
"onboarding.deleteFile": "Supprimer le fichier .cmf du disque",
"onboarding.confirmDelete": "Supprimer « {id} » ? Le fichier .cmf sera retiré du disque.",
"onboarding.unregistered": "Également désenregistré : {ids}",
// playground: media generation + streaming metrics
"pg.tab.chat": "Chat",
"pg.tab.image": "Image",
"pg.tab.video": "Vidéo",
"pg.generate": "Générer",
"pg.generating": "Génération…",
"pg.genFailed": "Échec de la génération — voir le journal.",
"pg.result": "Résultat",
"pg.genMetrics": "Métriques de génération",
"pg.imagePromptPlaceholder": "un renard roux dans la neige au coucher du soleil, photoréaliste, fourrure détaillée",
"pg.videoPromptPlaceholder": "un corgi en toque de chef fait sauter une crêpe, grésillements et aboiement joyeux",
"pg.imageEmpty": "L'image générée apparaîtra ici.",
"pg.videoEmpty": "Le clip généré (avec le son) apparaîtra ici.",
"pg.width": "Largeur",
"pg.height": "Hauteur",
"pg.steps": "Étapes",
"pg.seed": "Seed",
"pg.randomSeed": "Seed aléatoire",
"pg.cfgHint": "Force du guidance ; ≤ 1 désactive le CFG et divise le travail par deux.",
"pg.frames": "Images",
"pg.framesHint": "À 24 fps ; arrondi à la grille du modèle.",
"pg.quality": "Qualité JPEG",
"pg.videoSizeHint": "Multiples de 32 ; le petit côté entraîné est 768.",
"pg.videoStepsHint": "La LoRA Turbo est entraînée pour 4 étapes.",
"pg.firstFrame": "Première image (optionnel)",
"pg.lastFrame": "Dernière image (optionnel)",
"pg.keyframeHint": "Transforme la génération en keyframe-to-video.",
"pg.resolution": "Résolution",
"pg.genTime": "Temps de génération",
"pg.timePerStep": "Par étape",
"pg.timePerFrame": "Par image",
"pg.fileSize": "Taille du fichier",
"pg.duration": "Durée",
"pg.play": "Lire",
"pg.noMediaModels": "Pas encore de modèle {kind} local. Téléchargez-en un (par ex. de la collection infosave) dans Importer :",
"pg.tokSpeed": "Vitesse",
"pg.ttft": "Premier token",
};
+38
View File
@@ -460,4 +460,42 @@ export default {
"onboarding.deleteFile": "Удалить файл .cmf с диска",
"onboarding.confirmDelete": "Удалить «{id}»? Файл .cmf будет удалён с диска.",
"onboarding.unregistered": "Также отключено из моделей: {ids}",
// playground: media generation + streaming metrics
"pg.tab.chat": "Чат",
"pg.tab.image": "Изображение",
"pg.tab.video": "Видео",
"pg.generate": "Сгенерировать",
"pg.generating": "Генерация…",
"pg.genFailed": "Генерация не удалась — смотрите лог.",
"pg.result": "Результат",
"pg.genMetrics": "Метрики генерации",
"pg.imagePromptPlaceholder": "рыжая лиса в снегу на закате, фотореалистично, детальный мех",
"pg.videoPromptPlaceholder": "корги в поварском колпаке подбрасывает блин, шипение сковороды и весёлый лай",
"pg.imageEmpty": "Сгенерированное изображение появится здесь.",
"pg.videoEmpty": "Сгенерированный ролик (со звуком) появится здесь.",
"pg.width": "Ширина",
"pg.height": "Высота",
"pg.steps": "Шаги",
"pg.seed": "Seed",
"pg.randomSeed": "Случайный seed",
"pg.cfgHint": "Сила guidance; ≤ 1 отключает CFG и вдвое ускоряет генерацию.",
"pg.frames": "Кадры",
"pg.framesHint": "При 24 fps; округляется до сетки модели.",
"pg.quality": "Качество JPEG",
"pg.videoSizeHint": "Кратно 32; обученная короткая сторона — 768.",
"pg.videoStepsHint": "Turbo LoRA обучена на 4 шага.",
"pg.firstFrame": "Первый кадр (необязательно)",
"pg.lastFrame": "Последний кадр (необязательно)",
"pg.keyframeHint": "Превращает запуск в keyframe-to-video.",
"pg.resolution": "Разрешение",
"pg.genTime": "Время генерации",
"pg.timePerStep": "На шаг",
"pg.timePerFrame": "На кадр",
"pg.fileSize": "Размер файла",
"pg.duration": "Длительность",
"pg.play": "Играть",
"pg.noMediaModels": "Локальной модели ({kind}) ещё нет. Скачайте её (например, из коллекции infosave) во вкладке Импорт:",
"pg.tokSpeed": "Скорость",
"pg.ttft": "Первый токен",
};
+38
View File
@@ -468,4 +468,42 @@ export default {
"onboarding.deleteFile": ".cmf dosyasını diskten sil",
"onboarding.confirmDelete": "“{id}” silinsin mi? .cmf dosyası diskten kaldırılacak.",
"onboarding.unregistered": "Ayrıca kaydı kaldırıldı: {ids}",
// playground: media generation + streaming metrics
"pg.tab.chat": "Sohbet",
"pg.tab.image": "Görsel",
"pg.tab.video": "Video",
"pg.generate": "Oluştur",
"pg.generating": "Oluşturuluyor…",
"pg.genFailed": "Oluşturma başarısız — günlüğe bakın.",
"pg.result": "Sonuç",
"pg.genMetrics": "Oluşturma metrikleri",
"pg.imagePromptPlaceholder": "gün batımında karda oturan kızıl tilki, fotogerçekçi, ayrıntılı tüyler",
"pg.videoPromptPlaceholder": "aşçı şapkalı bir corgi krep çeviriyor, cızırtı sesleri ve neşeli bir havlama",
"pg.imageEmpty": "Oluşturulan görsel burada görünecek.",
"pg.videoEmpty": "Oluşturulan klip (sesiyle) burada görünecek.",
"pg.width": "Genişlik",
"pg.height": "Yükseklik",
"pg.steps": "Adım",
"pg.seed": "Seed",
"pg.randomSeed": "Rastgele seed",
"pg.cfgHint": "Guidance gücü; ≤ 1 CFG'yi kapatır ve işi yarıya indirir.",
"pg.frames": "Kare",
"pg.framesHint": "24 fps'te; modelin ızgarasına yuvarlanır.",
"pg.quality": "JPEG kalitesi",
"pg.videoSizeHint": "32'nin katları; eğitilen kısa kenar 768.",
"pg.videoStepsHint": "Turbo LoRA 4 adım için eğitildi.",
"pg.firstFrame": "İlk kare (isteğe bağlı)",
"pg.lastFrame": "Son kare (isteğe bağlı)",
"pg.keyframeHint": "Çalışmayı keyframe-to-video'ya çevirir.",
"pg.resolution": "Çözünürlük",
"pg.genTime": "Oluşturma süresi",
"pg.timePerStep": "Adım başına",
"pg.timePerFrame": "Kare başına",
"pg.fileSize": "Dosya boyutu",
"pg.duration": "Süre",
"pg.play": "Oynat",
"pg.noMediaModels": "Henüz yerel bir {kind} modeli yok. İçe aktarma sekmesinden bir tane indirin (ör. infosave koleksiyonundan):",
"pg.tokSpeed": "Hız",
"pg.ttft": "İlk token",
};
+38
View File
@@ -468,4 +468,42 @@ export default {
"onboarding.deleteFile": "从磁盘删除该 .cmf 文件",
"onboarding.confirmDelete": "删除 “{id}”?该 .cmf 文件将从磁盘移除。",
"onboarding.unregistered": "同时已取消注册:{ids}",
// playground: media generation + streaming metrics
"pg.tab.chat": "对话",
"pg.tab.image": "图像",
"pg.tab.video": "视频",
"pg.generate": "生成",
"pg.generating": "生成中……",
"pg.genFailed": "生成失败 — 请查看日志。",
"pg.result": "结果",
"pg.genMetrics": "生成指标",
"pg.imagePromptPlaceholder": "夕阳下雪地里的红狐狸,照片级真实感,毛发细节丰富",
"pg.videoPromptPlaceholder": "戴厨师帽的柯基翻煎饼,滋滋声和欢快的犬吠",
"pg.imageEmpty": "生成的图像将显示在这里。",
"pg.videoEmpty": "生成的短片(含音频)将显示在这里。",
"pg.width": "宽度",
"pg.height": "高度",
"pg.steps": "步数",
"pg.seed": "Seed",
"pg.randomSeed": "随机 seed",
"pg.cfgHint": "引导强度;≤ 1 关闭 CFG,工作量减半。",
"pg.frames": "帧数",
"pg.framesHint": "24 fps;按模型网格取整。",
"pg.quality": "JPEG 质量",
"pg.videoSizeHint": "32 的倍数;训练短边为 768。",
"pg.videoStepsHint": "Turbo LoRA 按 4 步训练。",
"pg.firstFrame": "首帧(可选)",
"pg.lastFrame": "末帧(可选)",
"pg.keyframeHint": "使运行变为关键帧生视频。",
"pg.resolution": "分辨率",
"pg.genTime": "生成耗时",
"pg.timePerStep": "每步",
"pg.timePerFrame": "每帧",
"pg.fileSize": "文件大小",
"pg.duration": "时长",
"pg.play": "播放",
"pg.noMediaModels": "还没有本地{kind}模型。请在“导入模型”中下载一个(例如 infosave 合集):",
"pg.tokSpeed": "速度",
"pg.ttft": "首个 token",
};
+167
View File
@@ -0,0 +1,167 @@
// Browser-side codecs for cortiq's dependency-free media outputs:
// - P6 PPM → canvas (cortiq imagine links no image library)
// - MJPEG+PCM AVI → frame list + canvas player synced to the .wav
// (cortiq animate never touches ffmpeg; neither do we)
/// Parse binary P6 PPM into {width, height, rgb: Uint8Array}.
export function parsePpm(buf) {
const b = new Uint8Array(buf);
// header: "P6" <ws> width <ws> height <ws> maxval <single ws> raster
let pos = 0;
const isWs = (c) => c === 32 || c === 9 || c === 10 || c === 13;
function token() {
while (pos < b.length && (isWs(b[pos]) || b[pos] === 35 /* # comment */)) {
if (b[pos] === 35) while (pos < b.length && b[pos] !== 10) pos++;
else pos++;
}
let start = pos;
while (pos < b.length && !isWs(b[pos])) pos++;
return String.fromCharCode(...b.slice(start, pos));
}
if (token() !== "P6") throw new Error("not a P6 PPM");
const width = parseInt(token());
const height = parseInt(token());
const maxval = parseInt(token());
pos++; // single whitespace after maxval
if (!width || !height || !maxval) throw new Error("bad PPM header");
return { width, height, rgb: b.subarray(pos, pos + width * height * 3) };
}
/// Draw a parsed PPM onto a (new or given) canvas. Returns the canvas.
export function ppmToCanvas(buf, canvas) {
const { width, height, rgb } = parsePpm(buf);
const c = canvas || document.createElement("canvas");
c.width = width;
c.height = height;
const ctx = c.getContext("2d");
const img = ctx.createImageData(width, height);
for (let i = 0, j = 0; i < width * height; i++, j += 3) {
img.data[i * 4] = rgb[j];
img.data[i * 4 + 1] = rgb[j + 1];
img.data[i * 4 + 2] = rgb[j + 2];
img.data[i * 4 + 3] = 255;
}
ctx.putImageData(img, 0, 0);
return c;
}
/// Rasterize an uploaded image file to WxH (stretched, as the engine does with
/// the first frame) and return it as base64-encoded binary P6 PPM.
export async function fileToPpmB64(file, width, height) {
const bmp = await createImageBitmap(file);
const c = document.createElement("canvas");
c.width = width;
c.height = height;
const ctx = c.getContext("2d");
ctx.drawImage(bmp, 0, 0, width, height);
const img = ctx.getImageData(0, 0, width, height).data;
const header = `P6\n${width} ${height}\n255\n`;
const out = new Uint8Array(header.length + width * height * 3);
for (let i = 0; i < header.length; i++) out[i] = header.charCodeAt(i);
let p = header.length;
for (let i = 0; i < width * height; i++) {
out[p++] = img[i * 4];
out[p++] = img[i * 4 + 1];
out[p++] = img[i * 4 + 2];
}
// chunked btoa — the frame can be a few MB
let bin = "";
for (let i = 0; i < out.length; i += 0x8000)
bin += String.fromCharCode.apply(null, out.subarray(i, i + 0x8000));
return btoa(bin);
}
/// Extract JPEG frames from an MJPEG AVI (RIFF): walk chunks from 'movi',
/// collect '00dc'/'00dj' payloads as Blobs.
export function aviJpegFrames(buf) {
const b = new Uint8Array(buf);
const dv = new DataView(buf);
const fourcc = (o) => String.fromCharCode(b[o], b[o + 1], b[o + 2], b[o + 3]);
if (fourcc(0) !== "RIFF" || fourcc(8) !== "AVI ") throw new Error("not an AVI");
// find the 'movi' LIST
let movi = -1;
for (let o = 12; o + 12 <= b.length; ) {
const id = fourcc(o);
const size = dv.getUint32(o + 4, true);
if (id === "LIST" && fourcc(o + 8) === "movi") {
movi = o + 12;
break;
}
o += 8 + size + (size & 1);
}
if (movi < 0) throw new Error("no movi list");
const frames = [];
for (let o = movi; o + 8 <= b.length; ) {
const id = fourcc(o);
const size = dv.getUint32(o + 4, true);
if (id === "00dc" || id === "00dj") {
if (size > 0) frames.push(new Blob([b.subarray(o + 8, o + 8 + size)], { type: "image/jpeg" }));
} else if (id === "LIST") {
o += 12; // descend (rec lists)
continue;
} else if (id === "idx1") {
break;
}
o += 8 + size + (size & 1);
}
return frames;
}
/// Canvas player for extracted MJPEG frames at a fixed fps, synced to an
/// optional <audio> element (the .wav). Returns {canvas, play, pause, seek0, playing}.
export function mjpegPlayer(frames, fps, audio) {
const canvas = document.createElement("canvas");
const ctx = canvas.getContext("2d");
let bitmaps = null; // decoded lazily on first play
let raf = null;
let t0 = 0;
let startFrame = 0;
const state = { playing: false };
async function decodeAll() {
if (bitmaps) return;
bitmaps = await Promise.all(frames.map((f) => createImageBitmap(f)));
if (bitmaps.length) {
canvas.width = bitmaps[0].width;
canvas.height = bitmaps[0].height;
ctx.drawImage(bitmaps[0], 0, 0);
}
}
function drawAt(i) {
if (bitmaps && bitmaps[i]) ctx.drawImage(bitmaps[i], 0, 0);
}
function tick() {
// audio clock when available (keeps A/V in sync), wall clock otherwise
const t = audio && !audio.paused ? audio.currentTime : (performance.now() - t0) / 1000;
const i = Math.min(frames.length - 1, startFrame + Math.floor(t * fps));
drawAt(i);
if (i >= frames.length - 1) {
pause();
return;
}
raf = requestAnimationFrame(tick);
}
async function play() {
await decodeAll();
state.playing = true;
startFrame = 0;
t0 = performance.now();
if (audio) {
audio.currentTime = 0;
audio.play().catch(() => {});
}
raf = requestAnimationFrame(tick);
}
function pause() {
state.playing = false;
if (raf) cancelAnimationFrame(raf);
raf = null;
if (audio) audio.pause();
}
async function seek0() {
pause();
await decodeAll();
drawAt(0);
}
return { canvas, play, pause, seek0, state, decodeAll };
}
+406 -27
View File
@@ -1,17 +1,64 @@
// Playground — send a prompt through the live pipeline, inspect routing decision.
import { h, mount, money, ms } from "../ui.js";
// Playground — three tabs over the live gateway:
// chat → the routing pipeline (with streaming, token counts, tok/s, TTFT)
// image → `cortiq imagine` over a local text-to-image .cmf (Lumina)
// video → `cortiq animate` over a local text-to-video .cmf (MiniMax-H3)
// Media results are decoded in the browser (PPM → canvas, MJPEG AVI → canvas
// player synced to the wav) — the same no-ffmpeg stance as the engine itself.
import { h, mount, toast, money, ms } from "../ui.js";
import { t } from "../i18n.js";
import { api, testStream } from "../api.js";
import { api, testStream, mediaFileUrl, mediaFileBytes } from "../api.js";
import { appState } from "../app.js";
import { ppmToCanvas, fileToPpmB64, aviJpegFrames, mjpegPlayer } from "../media-utils.js";
function fmtBytes(n) {
if (!n && n !== 0) return "—";
if (n >= 1e9) return (n / 1e9).toFixed(2) + " GB";
if (n >= 1e6) return (n / 1e6).toFixed(1) + " MB";
if (n >= 1e3) return Math.round(n / 1e3) + " KB";
return n + " B";
}
function kv(k, v) {
return h("div", { class: "kv" }, h("span", { class: "k" }, k), h("span", { class: "v" }, v));
}
function field(label, control, hint) {
return h("label", { class: "field" }, h("span", {}, label), control, hint ? h("div", { class: "hint" }, hint) : null);
}
export async function renderPlayground() {
const meta = appState.meta || (await api.meta());
const modelsData = await api.listModels();
const ids = (modelsData.models || []).map((m) => m.id);
let mediaModels = [];
try {
mediaModels = (await api.mediaModels()).models || [];
} catch {
/* media API unavailable — tabs will show the hint */
}
const imageModels = mediaModels.filter((m) => m.modality === "image");
const videoModels = mediaModels.filter((m) => m.modality === "video");
let mode = "auto";
const root = h("div");
let tab = "chat";
// ── tabs ──
const tabsSeg = h(
"div",
{ class: "seg" },
h("button", { class: "active", "data-tab": "chat" }, "💬 " + t("pg.tab.chat")),
h("button", { "data-tab": "image" }, "🖼 " + t("pg.tab.image")),
h("button", { "data-tab": "video" }, "🎬 " + t("pg.tab.video"))
);
const panelHost = h("div", {});
tabsSeg.querySelectorAll("button").forEach((b) =>
b.addEventListener("click", () => {
tab = b.dataset.tab;
tabsSeg.querySelectorAll("button").forEach((x) => x.classList.toggle("active", x === b));
mount(panelHost, panels[tab]);
})
);
// ════════════════════════════ chat panel ════════════════════════════
let mode = "auto";
const sysIn = h("textarea", { placeholder: t("pg.systemPlaceholder"), rows: 2 });
const promptIn = h("textarea", { placeholder: t("pg.promptPlaceholder"), rows: 5 });
const tempIn = h("input", { type: "number", step: "0.1", value: 0.7 });
@@ -79,6 +126,7 @@ export async function renderPlayground() {
out.textContent = acc;
});
out.textContent = acc || "—";
const latency = Math.round(performance.now() - started);
renderDecision({
cortiq: {
task_label: info.task_label,
@@ -88,8 +136,10 @@ export async function renderPlayground() {
cost_usd: info.cost_usd,
failover: false,
},
usage: {},
latency_ms: Math.round(performance.now() - started),
usage: info.usage || {},
usage_estimated: info.usage_estimated,
latency_ms: latency,
ttft_ms: info.ttft_ms,
});
} else {
const r = await api.test(body);
@@ -108,12 +158,17 @@ export async function renderPlayground() {
sendBtn.textContent = t("pg.send");
});
function kv(k, v) {
return h("div", { class: "kv" }, h("span", { class: "k" }, k), h("span", { class: "v" }, v));
}
function renderDecision(r) {
const c = r.cortiq || {};
const score = c.complexity?.score ?? 0;
const u = r.usage || {};
const haveUsage = u.completion_tokens != null;
// generation speed over the produce phase (after first token when known)
let tokS = null;
if (haveUsage && r.latency_ms) {
const genMs = r.ttft_ms != null ? Math.max(1, r.latency_ms - r.ttft_ms) : r.latency_ms;
tokS = (u.completion_tokens / (genMs / 1000)).toFixed(1);
}
mount(
decision,
h("div", { class: "card-head" }, h("h3", {}, t("pg.decision"))),
@@ -127,17 +182,188 @@ export async function renderPlayground() {
),
kv(t("pg.selected"), h("span", { class: "mono" }, c.selected_model || "—")),
kv(t("pg.source"), c.route_source || "—"),
kv(t("pg.tokens"), `${r.usage?.prompt_tokens ?? 0}/${r.usage?.completion_tokens ?? 0}`),
kv(t("pg.tokens"), haveUsage ? `${r.usage_estimated ? "≈ " : ""}${u.prompt_tokens ?? 0} / ${u.completion_tokens ?? 0}` : "—"),
kv(t("pg.tokSpeed"), tokS != null ? tokS + " tok/s" : "—"),
kv(t("pg.ttft"), r.ttft_ms != null ? ms(r.ttft_ms) : "—"),
kv(t("pg.cost"), money(c.cost_usd)),
kv(t("pg.latency"), ms(r.latency_ms)),
kv(t("pg.failover"), c.failover ? "yes" : "no")
);
}
mount(
root,
h("div", { class: "page-head" }, h("h2", {}, t("pg.title")), h("p", {}, t("pg.subtitle"))),
const chatPanel = h(
"div",
{ class: "pg" },
h(
"div",
{},
h(
"div",
{ class: "card" },
field(t("pg.system"), sysIn),
field(t("pg.prompt"), promptIn),
field(t("pg.mode"), modeSeg),
modeExtra,
h("div", { class: "row" }, field(t("pg.temp"), tempIn), field(t("pg.maxTokens"), maxTokIn)),
h(
"div",
{ class: "flex", style: "justify-content:space-between" },
h("label", { class: "check", style: "margin:0" }, streamCb, t("pg.stream")),
sendBtn
)
),
h("div", { class: "card" }, h("div", { class: "card-head" }, h("h3", {}, t("pg.answer"))), out)
),
decision
);
// ═══════════════════════ shared media machinery ═══════════════════════
let mediaPoll = null;
function stopMediaPoll() {
if (mediaPoll) clearInterval(mediaPoll);
mediaPoll = null;
}
// one generation at a time server-side → one active job client-side
function watchJob(id, ui, onDone) {
stopMediaPoll();
const started = performance.now();
mediaPoll = setInterval(async () => {
if (!root.isConnected) return stopMediaPoll();
let j;
try {
j = await api.mediaJob(id);
} catch {
return;
}
const secs = ((performance.now() - started) / 1000).toFixed(0);
const pct = j.progress != null ? Math.round(j.progress * 100) : null;
mount(
ui.bar,
h("div", { class: "pbar" },
j.state === "running"
? (pct != null
? h("div", { class: "pbar-fill det", style: `width:${Math.max(pct, 1)}%` })
: h("div", { class: "pbar-fill run" }))
: h("div", { class: "pbar-fill " + (j.state === "done" ? "ok" : "err"), style: "width:100%" })
),
h("div", { class: "muted small" },
(j.phase || j.state) + (pct != null && j.state === "running" ? ` · ${pct}%` : "") + ` · ${secs}s`),
(j.log || []).length ? h("pre", { class: "conv-log", style: "max-height:90px" }, j.log.slice(-5).join("\n")) : null
);
if (j.state !== "running") {
stopMediaPoll();
ui.goBtn.disabled = false;
ui.cancelBtn.style.display = "none";
if (j.state === "done") onDone(j);
else if (j.state === "error") toast(t("pg.genFailed"), "err");
}
}, 1000);
}
function mediaControls(goBtn, onCancelId) {
const cancelBtn = h("button", { class: "btn ghost", style: "display:none" }, "✕ " + t("import.cancel"));
cancelBtn.addEventListener("click", async () => {
try {
await api.mediaCancel(onCancelId.id);
} catch (e) {
toast(e.message, "err");
}
});
return cancelBtn;
}
function noModelsHint(kindLabel) {
return h(
"div",
{ class: "card" },
h("div", { class: "callout" },
t("pg.noMediaModels", { kind: kindLabel }),
" ",
h("a", { href: "#/import" }, t("settings.cmf.importLink") + " ↗"))
);
}
function dice(seedIn) {
const b = h("button", { class: "btn ghost sm", title: t("pg.randomSeed") }, "🎲");
b.addEventListener("click", () => (seedIn.value = Math.floor(Math.random() * 1e9)));
return b;
}
// ════════════════════════════ image panel ════════════════════════════
let imagePanel;
if (!imageModels.length) {
imagePanel = noModelsHint(t("pg.tab.image"));
} else {
const imModel = h("select", {}, ...imageModels.map((m) => h("option", { value: m.name }, `${m.name} · ${fmtBytes(m.size_bytes)}`)));
const imPrompt = h("textarea", { placeholder: t("pg.imagePromptPlaceholder"), rows: 3 });
const imW = h("select", {}, ...[256, 384, 512, 640, 768, 1024].map((v) => h("option", { value: v, selected: v === 512 ? true : null }, v)));
const imH = h("select", {}, ...[256, 384, 512, 640, 768, 1024].map((v) => h("option", { value: v, selected: v === 512 ? true : null }, v)));
const imSteps = h("input", { type: "number", min: 1, max: 100, value: 30 });
const imCfg = h("input", { type: "number", step: "0.5", value: 4 });
const imSeed = h("input", { type: "number", value: 42 });
const imBar = h("div", {});
const imOut = h("div", { class: "media-out" }, h("div", { class: "empty" }, t("pg.imageEmpty")));
const imMetrics = h("div", { class: "card routecard" }, h("div", { class: "card-head" }, h("h3", {}, t("pg.genMetrics"))), h("div", { class: "empty" }, "—"));
const imGo = h("button", { class: "btn primary" }, "🖼 " + t("pg.generate"));
const imCancelRef = { id: null };
const imCancel = mediaControls(imGo, imCancelRef);
imGo.addEventListener("click", async () => {
const prompt = imPrompt.value.trim();
if (!prompt) return;
imGo.disabled = true;
imCancel.style.display = "";
mount(imOut, h("div", { class: "flex" }, h("span", { class: "spinner" }), " ", t("pg.generating")));
const params = {
kind: "image",
model: imModel.value,
prompt,
width: +imW.value,
height: +imH.value,
steps: +imSteps.value || 30,
cfg: parseFloat(imCfg.value) || 4,
seed: +imSeed.value || 42,
};
try {
const r = await api.mediaGenerate(params);
imCancelRef.id = r.job;
const t0 = performance.now();
watchJob(r.job, { bar: imBar, goBtn: imGo, cancelBtn: imCancel }, async (j) => {
mount(imBar);
const secs = (performance.now() - t0) / 1000;
try {
const buf = await mediaFileBytes(j.id, "image");
const canvas = ppmToCanvas(buf);
canvas.className = "media-canvas";
const dl = h("button", { class: "btn sm" }, "⬇ PNG");
dl.addEventListener("click", () =>
canvas.toBlob((b) => {
const a = h("a", { href: URL.createObjectURL(b), download: `cortiq-${j.id}.png` });
a.click();
}, "image/png")
);
mount(imOut, canvas, h("div", { class: "actions" }, dl));
mount(
imMetrics,
h("div", { class: "card-head" }, h("h3", {}, t("pg.genMetrics"))),
kv(t("pg.model"), h("span", { class: "mono" }, j.model)),
kv(t("pg.resolution"), `${j.width}×${j.height}`),
kv(t("pg.steps"), String(j.steps)),
kv("CFG", String(params.cfg)),
kv(t("pg.seed"), String(j.seed)),
kv(t("pg.genTime"), secs.toFixed(1) + " s"),
kv(t("pg.timePerStep"), (secs / j.steps).toFixed(2) + " s"),
kv(t("pg.fileSize"), fmtBytes(j.out_bytes))
);
} catch (e) {
mount(imOut, h("div", { class: "callout" }, String(e.message || e)));
}
});
} catch (e) {
imGo.disabled = false;
imCancel.style.display = "none";
mount(imOut, h("div", { class: "callout" }, String(e.message || e)));
}
});
imagePanel = h(
"div",
{ class: "pg" },
h(
@@ -146,22 +372,175 @@ export async function renderPlayground() {
h(
"div",
{ class: "card" },
h("label", { class: "field" }, h("span", {}, t("pg.system")), sysIn),
h("label", { class: "field" }, h("span", {}, t("pg.prompt")), promptIn),
h("label", { class: "field" }, h("span", {}, t("pg.mode")), modeSeg),
modeExtra,
h("div", { class: "row" }, h("label", { class: "field" }, h("span", {}, t("pg.temp")), tempIn), h("label", { class: "field" }, h("span", {}, t("pg.maxTokens")), maxTokIn)),
h(
"div",
{ class: "flex", style: "justify-content:space-between" },
h("label", { class: "check", style: "margin:0" }, streamCb, t("pg.stream")),
sendBtn
)
field(t("pg.model"), imModel),
field(t("pg.prompt"), imPrompt),
h("div", { class: "row" }, field(t("pg.width"), imW), field(t("pg.height"), imH)),
h("div", { class: "row" },
field(t("pg.steps"), imSteps),
field("CFG", imCfg, t("pg.cfgHint"))),
h("div", { class: "row" },
field(t("pg.seed"), h("div", { class: "flex", style: "gap:6px" }, imSeed, dice(imSeed)))),
h("div", { class: "actions" }, imCancel, imGo),
imBar
),
h("div", { class: "card" }, h("div", { class: "card-head" }, h("h3", {}, t("pg.answer"))), out)
h("div", { class: "card" }, h("div", { class: "card-head" }, h("h3", {}, t("pg.result"))), imOut)
),
decision
)
imMetrics
);
}
// ════════════════════════════ video panel ════════════════════════════
let videoPanel;
if (!videoModels.length) {
videoPanel = noModelsHint(t("pg.tab.video"));
} else {
const vModel = h("select", {}, ...videoModels.map((m) => h("option", { value: m.name }, `${m.name} · ${fmtBytes(m.size_bytes)}`)));
const vPrompt = h("textarea", { placeholder: t("pg.videoPromptPlaceholder"), rows: 3 });
const vW = h("input", { type: "number", step: 32, value: 512 });
const vH = h("input", { type: "number", step: 32, value: 288 });
const vFrames = h("input", { type: "number", value: 39 });
const vSteps = h("input", { type: "number", value: 4 });
const vSeed = h("input", { type: "number", value: 42 });
const vQuality = h("input", { type: "number", min: 10, max: 100, value: 92 });
const vFirst = h("input", { type: "file", accept: "image/*" });
const vLast = h("input", { type: "file", accept: "image/*" });
const vBar = h("div", {});
const vOut = h("div", { class: "media-out" }, h("div", { class: "empty" }, t("pg.videoEmpty")));
const vMetrics = h("div", { class: "card routecard" }, h("div", { class: "card-head" }, h("h3", {}, t("pg.genMetrics"))), h("div", { class: "empty" }, "—"));
const vGo = h("button", { class: "btn primary" }, "🎬 " + t("pg.generate"));
const vCancelRef = { id: null };
const vCancel = mediaControls(vGo, vCancelRef);
vGo.addEventListener("click", async () => {
const prompt = vPrompt.value.trim();
if (!prompt) return;
vGo.disabled = true;
vCancel.style.display = "";
mount(vOut, h("div", { class: "flex" }, h("span", { class: "spinner" }), " ", t("pg.generating")));
const params = {
kind: "video",
model: vModel.value,
prompt,
width: +vW.value || 512,
height: +vH.value || 288,
frames: +vFrames.value || 39,
steps: +vSteps.value || 4,
seed: +vSeed.value || 42,
quality: +vQuality.value || 92,
};
try {
if (vFirst.files[0]) params.first_frame_b64 = await fileToPpmB64(vFirst.files[0], params.width, params.height);
if (vLast.files[0]) params.last_frame_b64 = await fileToPpmB64(vLast.files[0], params.width, params.height);
const r = await api.mediaGenerate(params);
vCancelRef.id = r.job;
const t0 = performance.now();
watchJob(r.job, { bar: vBar, goBtn: vGo, cancelBtn: vCancel }, async (j) => {
mount(vBar);
const secs = (performance.now() - t0) / 1000;
try {
await showVideoResult(j, vOut);
const durS = (j.frames || 0) / 24;
mount(
vMetrics,
h("div", { class: "card-head" }, h("h3", {}, t("pg.genMetrics"))),
kv(t("pg.model"), h("span", { class: "mono" }, j.model)),
kv(t("pg.resolution"), `${j.width}×${j.height}`),
kv(t("pg.frames"), `${j.frames} @ 24 fps`),
kv(t("pg.duration"), durS.toFixed(1) + " s"),
kv(t("pg.steps"), String(j.steps)),
kv(t("pg.seed"), String(j.seed)),
kv(t("pg.genTime"), secs.toFixed(1) + " s"),
kv(t("pg.timePerFrame"), j.frames ? (secs / j.frames).toFixed(2) + " s" : "—"),
kv(t("pg.fileSize"), fmtBytes(j.out_bytes))
);
} catch (e) {
mount(vOut, h("div", { class: "callout" }, String(e.message || e)));
}
});
} catch (e) {
vGo.disabled = false;
vCancel.style.display = "none";
mount(vOut, h("div", { class: "callout" }, String(e.message || e)));
}
});
async function showVideoResult(j, host) {
const downloads = h("div", { class: "actions" });
for (const kind of j.outputs || []) {
const label = { video: "⬇ AVI", audio: "⬇ WAV", mp4: "⬇ MP4" }[kind];
if (!label) continue;
const b = h("button", { class: "btn sm" }, label);
b.addEventListener("click", async () => {
const { url } = await mediaFileUrl(j.id, kind);
const ext = kind === "video" ? "avi" : kind === "audio" ? "wav" : "mp4";
h("a", { href: url, download: `cortiq-${j.id}.${ext}` }).click();
});
downloads.appendChild(b);
}
if ((j.outputs || []).includes("mp4")) {
// ffmpeg was present server-side → native playback
const { url } = await mediaFileUrl(j.id, "mp4");
mount(host, h("video", { src: url, controls: true, class: "media-canvas" }), downloads);
return;
}
// no ffmpeg anywhere: decode the MJPEG AVI ourselves, sync to the wav
const buf = await mediaFileBytes(j.id, "video");
const frames = aviJpegFrames(buf);
let audio = null;
if ((j.outputs || []).includes("audio")) {
const { url } = await mediaFileUrl(j.id, "audio");
audio = h("audio", { src: url });
}
const player = mjpegPlayer(frames, 24, audio);
player.canvas.className = "media-canvas";
await player.seek0();
const playBtn = h("button", { class: "btn primary sm" }, "▶ " + t("pg.play"));
playBtn.addEventListener("click", async () => {
if (player.state.playing) {
player.pause();
playBtn.textContent = "▶ " + t("pg.play");
} else {
playBtn.textContent = "⏸";
await player.play();
}
});
mount(host, player.canvas, h("div", { class: "actions" }, playBtn, ...downloads.childNodes ? [downloads] : []));
}
videoPanel = h(
"div",
{ class: "pg" },
h(
"div",
{},
h(
"div",
{ class: "card" },
field(t("pg.model"), vModel),
field(t("pg.prompt"), vPrompt),
h("div", { class: "row" }, field(t("pg.width"), vW, t("pg.videoSizeHint")), field(t("pg.height"), vH)),
h("div", { class: "row" }, field(t("pg.frames"), vFrames, t("pg.framesHint")), field(t("pg.steps"), vSteps, t("pg.videoStepsHint"))),
h("div", { class: "row" },
field(t("pg.seed"), h("div", { class: "flex", style: "gap:6px" }, vSeed, dice(vSeed))),
field(t("pg.quality"), vQuality)),
h("div", { class: "row" }, field(t("pg.firstFrame"), vFirst, t("pg.keyframeHint")), field(t("pg.lastFrame"), vLast)),
h("div", { class: "actions" }, vCancel, vGo),
vBar
),
h("div", { class: "card" }, h("div", { class: "card-head" }, h("h3", {}, t("pg.result"))), vOut)
),
vMetrics
);
}
const panels = { chat: chatPanel, image: imagePanel, video: videoPanel };
mount(
root,
h("div", { class: "page-head" }, h("h2", {}, t("pg.title")), h("p", {}, t("pg.subtitle"))),
tabsSeg,
panelHost
);
mount(panelHost, panels.chat);
return root;
}