feat(playground): image & video generation tabs + real chat metrics
The infosave collection now ships media models (Lumina-Image-2.0, MiniMax-H3 Turbo) that cortiq runs via `imagine`/`animate` CLI, not `cortiq serve`. The gateway grows a media API: local .cmf files are classified by `cortiq info` (chat/image/video, cached), generations run as tracked jobs with step progress parsed from stdout, one at a time, cancellable, outputs under <data>/media. The playground becomes three tabs. Image: width/height/steps/CFG/seed, live progress, result rendered from P6 PPM to a canvas in the browser (PNG export via toBlob). Video: frames/steps/quality/seed plus optional first/last keyframes (rasterized to PPM client-side); the MJPEG+PCM AVI is decoded in JS and played on a canvas synced to the wav — no ffmpeg on either side (mp4 remux happens opportunistically when ffmpeg exists). A metrics card shows resolution, steps, seed, total time, time per step/frame and file size. Chat metrics: the local cortiq server sends no usage chunk in streams, so token counts showed 0/0. tap_stream now injects one OpenAI-shaped usage event ahead of [DONE] with the gateway's ~4-chars/token estimate, flagged `cortiq_estimated`; the UI renders it as "≈ N / M" and adds tok/s and time-to-first-token. Registration skips media packs (a `cortiq serve` entry for a diffusion model would only fail) in both Register and the wizard's Finish. 36 new UI strings translated across all 7 languages.
This commit is contained in:
infosave2007
Vendored
+167
@@ -0,0 +1,167 @@
|
||||
// Browser-side codecs for cortiq's dependency-free media outputs:
|
||||
// - P6 PPM → canvas (cortiq imagine links no image library)
|
||||
// - MJPEG+PCM AVI → frame list + canvas player synced to the .wav
|
||||
// (cortiq animate never touches ffmpeg; neither do we)
|
||||
|
||||
/// Parse binary P6 PPM into {width, height, rgb: Uint8Array}.
|
||||
export function parsePpm(buf) {
|
||||
const b = new Uint8Array(buf);
|
||||
// header: "P6" <ws> width <ws> height <ws> maxval <single ws> raster
|
||||
let pos = 0;
|
||||
const isWs = (c) => c === 32 || c === 9 || c === 10 || c === 13;
|
||||
function token() {
|
||||
while (pos < b.length && (isWs(b[pos]) || b[pos] === 35 /* # comment */)) {
|
||||
if (b[pos] === 35) while (pos < b.length && b[pos] !== 10) pos++;
|
||||
else pos++;
|
||||
}
|
||||
let start = pos;
|
||||
while (pos < b.length && !isWs(b[pos])) pos++;
|
||||
return String.fromCharCode(...b.slice(start, pos));
|
||||
}
|
||||
if (token() !== "P6") throw new Error("not a P6 PPM");
|
||||
const width = parseInt(token());
|
||||
const height = parseInt(token());
|
||||
const maxval = parseInt(token());
|
||||
pos++; // single whitespace after maxval
|
||||
if (!width || !height || !maxval) throw new Error("bad PPM header");
|
||||
return { width, height, rgb: b.subarray(pos, pos + width * height * 3) };
|
||||
}
|
||||
|
||||
/// Draw a parsed PPM onto a (new or given) canvas. Returns the canvas.
|
||||
export function ppmToCanvas(buf, canvas) {
|
||||
const { width, height, rgb } = parsePpm(buf);
|
||||
const c = canvas || document.createElement("canvas");
|
||||
c.width = width;
|
||||
c.height = height;
|
||||
const ctx = c.getContext("2d");
|
||||
const img = ctx.createImageData(width, height);
|
||||
for (let i = 0, j = 0; i < width * height; i++, j += 3) {
|
||||
img.data[i * 4] = rgb[j];
|
||||
img.data[i * 4 + 1] = rgb[j + 1];
|
||||
img.data[i * 4 + 2] = rgb[j + 2];
|
||||
img.data[i * 4 + 3] = 255;
|
||||
}
|
||||
ctx.putImageData(img, 0, 0);
|
||||
return c;
|
||||
}
|
||||
|
||||
/// Rasterize an uploaded image file to WxH (stretched, as the engine does with
|
||||
/// the first frame) and return it as base64-encoded binary P6 PPM.
|
||||
export async function fileToPpmB64(file, width, height) {
|
||||
const bmp = await createImageBitmap(file);
|
||||
const c = document.createElement("canvas");
|
||||
c.width = width;
|
||||
c.height = height;
|
||||
const ctx = c.getContext("2d");
|
||||
ctx.drawImage(bmp, 0, 0, width, height);
|
||||
const img = ctx.getImageData(0, 0, width, height).data;
|
||||
const header = `P6\n${width} ${height}\n255\n`;
|
||||
const out = new Uint8Array(header.length + width * height * 3);
|
||||
for (let i = 0; i < header.length; i++) out[i] = header.charCodeAt(i);
|
||||
let p = header.length;
|
||||
for (let i = 0; i < width * height; i++) {
|
||||
out[p++] = img[i * 4];
|
||||
out[p++] = img[i * 4 + 1];
|
||||
out[p++] = img[i * 4 + 2];
|
||||
}
|
||||
// chunked btoa — the frame can be a few MB
|
||||
let bin = "";
|
||||
for (let i = 0; i < out.length; i += 0x8000)
|
||||
bin += String.fromCharCode.apply(null, out.subarray(i, i + 0x8000));
|
||||
return btoa(bin);
|
||||
}
|
||||
|
||||
/// Extract JPEG frames from an MJPEG AVI (RIFF): walk chunks from 'movi',
|
||||
/// collect '00dc'/'00dj' payloads as Blobs.
|
||||
export function aviJpegFrames(buf) {
|
||||
const b = new Uint8Array(buf);
|
||||
const dv = new DataView(buf);
|
||||
const fourcc = (o) => String.fromCharCode(b[o], b[o + 1], b[o + 2], b[o + 3]);
|
||||
if (fourcc(0) !== "RIFF" || fourcc(8) !== "AVI ") throw new Error("not an AVI");
|
||||
// find the 'movi' LIST
|
||||
let movi = -1;
|
||||
for (let o = 12; o + 12 <= b.length; ) {
|
||||
const id = fourcc(o);
|
||||
const size = dv.getUint32(o + 4, true);
|
||||
if (id === "LIST" && fourcc(o + 8) === "movi") {
|
||||
movi = o + 12;
|
||||
break;
|
||||
}
|
||||
o += 8 + size + (size & 1);
|
||||
}
|
||||
if (movi < 0) throw new Error("no movi list");
|
||||
const frames = [];
|
||||
for (let o = movi; o + 8 <= b.length; ) {
|
||||
const id = fourcc(o);
|
||||
const size = dv.getUint32(o + 4, true);
|
||||
if (id === "00dc" || id === "00dj") {
|
||||
if (size > 0) frames.push(new Blob([b.subarray(o + 8, o + 8 + size)], { type: "image/jpeg" }));
|
||||
} else if (id === "LIST") {
|
||||
o += 12; // descend (rec lists)
|
||||
continue;
|
||||
} else if (id === "idx1") {
|
||||
break;
|
||||
}
|
||||
o += 8 + size + (size & 1);
|
||||
}
|
||||
return frames;
|
||||
}
|
||||
|
||||
/// Canvas player for extracted MJPEG frames at a fixed fps, synced to an
|
||||
/// optional <audio> element (the .wav). Returns {canvas, play, pause, seek0, playing}.
|
||||
export function mjpegPlayer(frames, fps, audio) {
|
||||
const canvas = document.createElement("canvas");
|
||||
const ctx = canvas.getContext("2d");
|
||||
let bitmaps = null; // decoded lazily on first play
|
||||
let raf = null;
|
||||
let t0 = 0;
|
||||
let startFrame = 0;
|
||||
const state = { playing: false };
|
||||
|
||||
async function decodeAll() {
|
||||
if (bitmaps) return;
|
||||
bitmaps = await Promise.all(frames.map((f) => createImageBitmap(f)));
|
||||
if (bitmaps.length) {
|
||||
canvas.width = bitmaps[0].width;
|
||||
canvas.height = bitmaps[0].height;
|
||||
ctx.drawImage(bitmaps[0], 0, 0);
|
||||
}
|
||||
}
|
||||
function drawAt(i) {
|
||||
if (bitmaps && bitmaps[i]) ctx.drawImage(bitmaps[i], 0, 0);
|
||||
}
|
||||
function tick() {
|
||||
// audio clock when available (keeps A/V in sync), wall clock otherwise
|
||||
const t = audio && !audio.paused ? audio.currentTime : (performance.now() - t0) / 1000;
|
||||
const i = Math.min(frames.length - 1, startFrame + Math.floor(t * fps));
|
||||
drawAt(i);
|
||||
if (i >= frames.length - 1) {
|
||||
pause();
|
||||
return;
|
||||
}
|
||||
raf = requestAnimationFrame(tick);
|
||||
}
|
||||
async function play() {
|
||||
await decodeAll();
|
||||
state.playing = true;
|
||||
startFrame = 0;
|
||||
t0 = performance.now();
|
||||
if (audio) {
|
||||
audio.currentTime = 0;
|
||||
audio.play().catch(() => {});
|
||||
}
|
||||
raf = requestAnimationFrame(tick);
|
||||
}
|
||||
function pause() {
|
||||
state.playing = false;
|
||||
if (raf) cancelAnimationFrame(raf);
|
||||
raf = null;
|
||||
if (audio) audio.pause();
|
||||
}
|
||||
async function seek0() {
|
||||
pause();
|
||||
await decodeAll();
|
||||
drawAt(0);
|
||||
}
|
||||
return { canvas, play, pause, seek0, state, decodeAll };
|
||||
}
|
||||
Reference in New Issue
Block a user