feat(playground): image & video generation tabs + real chat metrics
The infosave collection now ships media models (Lumina-Image-2.0, MiniMax-H3 Turbo) that cortiq runs via `imagine`/`animate` CLI, not `cortiq serve`. The gateway grows a media API: local .cmf files are classified by `cortiq info` (chat/image/video, cached), generations run as tracked jobs with step progress parsed from stdout, one at a time, cancellable, outputs under <data>/media. The playground becomes three tabs. Image: width/height/steps/CFG/seed, live progress, result rendered from P6 PPM to a canvas in the browser (PNG export via toBlob). Video: frames/steps/quality/seed plus optional first/last keyframes (rasterized to PPM client-side); the MJPEG+PCM AVI is decoded in JS and played on a canvas synced to the wav — no ffmpeg on either side (mp4 remux happens opportunistically when ffmpeg exists). A metrics card shows resolution, steps, seed, total time, time per step/frame and file size. Chat metrics: the local cortiq server sends no usage chunk in streams, so token counts showed 0/0. tap_stream now injects one OpenAI-shaped usage event ahead of [DONE] with the gateway's ~4-chars/token estimate, flagged `cortiq_estimated`; the UI renders it as "≈ N / M" and adds tok/s and time-to-first-token. Registration skips media packs (a `cortiq serve` entry for a diffusion model would only fail) in both Register and the wizard's Finish. 36 new UI strings translated across all 7 languages.
This commit is contained in:
infosave2007
@@ -184,6 +184,14 @@ pub fn api_routes(admin_token: String) -> Router<SharedState> {
|
||||
.route("/admin/api/requests", get(get_requests))
|
||||
.route("/admin/api/test", post(run_test))
|
||||
.route("/admin/api/test/stream", post(run_test_stream))
|
||||
.route("/admin/api/media/models", get(media_models))
|
||||
.route("/admin/api/media", get(media_jobs).post(media_generate))
|
||||
.route(
|
||||
"/admin/api/media/:job",
|
||||
get(media_job_status).delete(media_delete),
|
||||
)
|
||||
.route("/admin/api/media/:job/cancel", post(media_cancel))
|
||||
.route("/admin/api/media/:job/file/:kind", get(media_file))
|
||||
.route("/admin/api/setup/status", get(setup_status))
|
||||
.route("/admin/api/setup/complete", post(setup_complete))
|
||||
.route("/admin/api/setup/featured", get(setup_featured))
|
||||
@@ -1420,8 +1428,16 @@ async fn setup_complete(
|
||||
.map(|j| j.output)
|
||||
.collect();
|
||||
if !finished.is_empty() {
|
||||
let cortiq_bin = state.live().cfg.cmf.cortiq_bin.clone();
|
||||
let mut cfg = current_cfg(&state);
|
||||
for output in &finished {
|
||||
// media packs (image/video) are used by the media API directly,
|
||||
// not served — connecting them as chat servers would only fail
|
||||
let modality =
|
||||
crate::media::classify_file(std::path::Path::new(output), &cortiq_bin).await;
|
||||
if modality != crate::media::Modality::Chat {
|
||||
continue;
|
||||
}
|
||||
if let Some(id) = register_cmf_output(&mut cfg, output) {
|
||||
registered.push(id);
|
||||
}
|
||||
@@ -1972,6 +1988,15 @@ async fn register_import(State(state): State<SharedState>, Path(job): Path<Strin
|
||||
if j.state != "done" {
|
||||
return Err(ApiError::bad("conversion not finished"));
|
||||
}
|
||||
let live = state.live();
|
||||
// Image/video packs are not served by `cortiq serve` — they are used
|
||||
// directly by the media API. Nothing to register; tell the UI what it is.
|
||||
let modality =
|
||||
crate::media::classify_file(std::path::Path::new(&j.output), &live.cfg.cmf.cortiq_bin)
|
||||
.await;
|
||||
if modality != crate::media::Modality::Chat {
|
||||
return ok(json!({ "ok": true, "media_kind": modality, "model_id": null }));
|
||||
}
|
||||
let mut cfg = current_cfg(&state);
|
||||
let id = register_cmf_output(&mut cfg, &j.output).ok_or_else(|| {
|
||||
ApiError::bad(format!(
|
||||
@@ -2004,6 +2029,92 @@ async fn get_requests(State(state): State<SharedState>, Query(q): Query<RecentQu
|
||||
ok(json!({ "requests": items }))
|
||||
}
|
||||
|
||||
// ───────────────────────────── media generation ─────────────────────────────
|
||||
|
||||
/// Local `.cmf` files with their modality (chat / image / video) — the
|
||||
/// playground's media tabs read this to know what can generate what.
|
||||
async fn media_models(State(state): State<SharedState>) -> ApiResult {
|
||||
let live = state.live();
|
||||
let models =
|
||||
crate::media::list_media_models(&live.cfg.cmf.models_dir, &live.cfg.cmf.cortiq_bin).await;
|
||||
ok(json!({ "models": models }))
|
||||
}
|
||||
|
||||
async fn media_jobs(State(state): State<SharedState>) -> ApiResult {
|
||||
ok(json!({ "jobs": state.media.list() }))
|
||||
}
|
||||
|
||||
async fn media_generate(
|
||||
State(state): State<SharedState>,
|
||||
Json(p): Json<crate::media::MediaParams>,
|
||||
) -> ApiResult {
|
||||
let live = state.live();
|
||||
let id = crate::media::start_generation(state.media.clone(), &live.cfg.cmf, p)
|
||||
.map_err(ApiError::bad)?;
|
||||
ok(json!({ "job": id }))
|
||||
}
|
||||
|
||||
async fn media_job_status(State(state): State<SharedState>, Path(job): Path<String>) -> ApiResult {
|
||||
match state.media.get(&job) {
|
||||
Some(j) => ok(serde_json::to_value(j).unwrap_or_else(|_| json!({}))),
|
||||
None => Err(ApiError::not_found(format!("media job '{job}' not found"))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn media_cancel(State(state): State<SharedState>, Path(job): Path<String>) -> ApiResult {
|
||||
if state.media.get(&job).is_none() {
|
||||
return Err(ApiError::not_found(format!("media job '{job}' not found")));
|
||||
}
|
||||
ok(json!({ "ok": state.media.cancel(&job) }))
|
||||
}
|
||||
|
||||
async fn media_delete(State(state): State<SharedState>, Path(job): Path<String>) -> ApiResult {
|
||||
if state.media.get(&job).is_none() {
|
||||
return Err(ApiError::not_found(format!("media job '{job}' not found")));
|
||||
}
|
||||
let removed = state
|
||||
.media
|
||||
.delete(&job, &crate::media::media_dir())
|
||||
.map_err(ApiError::bad)?;
|
||||
ok(json!({ "ok": removed }))
|
||||
}
|
||||
|
||||
/// Serve a finished job's output bytes (ppm / avi / wav / mp4). The job id is
|
||||
/// server-generated hex and the file name is fixed per kind, so no path from the
|
||||
/// client ever reaches the filesystem.
|
||||
async fn media_file(
|
||||
State(state): State<SharedState>,
|
||||
Path((job, kind)): Path<(String, String)>,
|
||||
) -> Response {
|
||||
let Some(j) = state.media.get(&job) else {
|
||||
return ApiError::not_found(format!("media job '{job}' not found")).into_response();
|
||||
};
|
||||
if j.state != "done" {
|
||||
return ApiError::bad("generation not finished").into_response();
|
||||
}
|
||||
let Some((file, mime)) = crate::media::output_file(&kind) else {
|
||||
return ApiError::bad(format!("unknown output kind '{kind}'")).into_response();
|
||||
};
|
||||
let path = crate::media::media_dir().join(&job).join(file);
|
||||
match tokio::fs::read(&path).await {
|
||||
Ok(bytes) => (
|
||||
[
|
||||
(header::CONTENT_TYPE, mime.to_string()),
|
||||
(
|
||||
header::CONTENT_DISPOSITION,
|
||||
format!(
|
||||
"inline; filename=\"cortiq-{job}.{}\"",
|
||||
file.rsplit('.').next().unwrap_or("bin")
|
||||
),
|
||||
),
|
||||
],
|
||||
bytes,
|
||||
)
|
||||
.into_response(),
|
||||
Err(_) => ApiError::not_found("output file missing on disk").into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
// ───────────────────────────── playground (test) ───────────────────────────
|
||||
|
||||
#[derive(Deserialize)]
|
||||
|
||||
@@ -14,6 +14,7 @@ mod cmf_runtime;
|
||||
mod config;
|
||||
mod error;
|
||||
mod import;
|
||||
mod media;
|
||||
mod model;
|
||||
mod pipeline;
|
||||
mod promotion;
|
||||
|
||||
+771
@@ -0,0 +1,771 @@
|
||||
//! Media generation: run `cortiq imagine` (text→image, Lumina) and
|
||||
//! `cortiq animate` (text→video+audio, MiniMax-H3) over local `.cmf` files as
|
||||
//! tracked background jobs, the same shape as import jobs so the admin UI can
|
||||
//! poll progress and fetch the result.
|
||||
//!
|
||||
//! These models are NOT served by `cortiq serve` — each generation is one CLI
|
||||
//! run. The gateway owns the process, parses its progress, and keeps the output
|
||||
//! under `<data>/media/<job>/`.
|
||||
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Stdio;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
fn now() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
/// Random job id — media jobs are never deduplicated (every run is a new work).
|
||||
fn gen_media_id() -> String {
|
||||
let mut b = [0u8; 8];
|
||||
let _ = getrandom::getrandom(&mut b);
|
||||
b.iter().map(|x| format!("{x:02x}")).collect()
|
||||
}
|
||||
|
||||
/// What a `.cmf` under models_dir can do, decided by `cortiq info`.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Modality {
|
||||
Chat,
|
||||
Image,
|
||||
Video,
|
||||
}
|
||||
|
||||
/// `cortiq info` output → modality. The arch line names the family; the media
|
||||
/// packs also carry their pipeline components (DiT/VAE/vocoder) in the text.
|
||||
fn classify_info(info: &str) -> Modality {
|
||||
let lower = info.to_ascii_lowercase();
|
||||
let arch = lower
|
||||
.lines()
|
||||
.find_map(|l| l.trim().strip_prefix("arch:"))
|
||||
.unwrap_or("")
|
||||
.trim()
|
||||
.to_string();
|
||||
// generic arch suffixes first (e.g. "lumina2-image", "mmh3-video"), then
|
||||
// known family names as a fallback for packs without the suffix
|
||||
if arch.contains("video") || arch.contains("mmh3") || arch.contains("minimax") {
|
||||
return Modality::Video;
|
||||
}
|
||||
if arch.contains("image") || arch.contains("lumina") || arch.contains("diffusion") {
|
||||
return Modality::Image;
|
||||
}
|
||||
if lower.contains("text-to-video") {
|
||||
return Modality::Video;
|
||||
}
|
||||
if lower.contains("text-to-image") {
|
||||
return Modality::Image;
|
||||
}
|
||||
Modality::Chat
|
||||
}
|
||||
|
||||
/// One local `.cmf` with its detected modality.
|
||||
#[derive(Clone, Serialize)]
|
||||
pub struct MediaModel {
|
||||
pub name: String, // file stem
|
||||
pub path: String,
|
||||
pub size_bytes: u64,
|
||||
pub modality: Modality,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct InfoCache(Mutex<HashMap<String, (u64, u64, Modality)>>); // path → (mtime, size, modality)
|
||||
|
||||
/// Modality of one `.cmf`, via `cortiq info` — cached by (path, mtime, size),
|
||||
/// so the process runs once per file version.
|
||||
pub async fn classify_file(path: &Path, cortiq_bin: &str) -> Modality {
|
||||
static CACHE: std::sync::OnceLock<InfoCache> = std::sync::OnceLock::new();
|
||||
let cache = CACHE.get_or_init(InfoCache::default);
|
||||
|
||||
let size = crate::import::path_size(path);
|
||||
let mtime = std::fs::metadata(path)
|
||||
.ok()
|
||||
.and_then(|m| m.modified().ok())
|
||||
.and_then(|t| t.duration_since(UNIX_EPOCH).ok())
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0);
|
||||
let key = path.to_string_lossy().to_string();
|
||||
if let Some(k) = cache
|
||||
.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(&key)
|
||||
.filter(|(m, s, _)| *m == mtime && *s == size)
|
||||
.map(|(_, _, k)| *k)
|
||||
{
|
||||
return k;
|
||||
}
|
||||
let output = tokio::process::Command::new(cortiq_bin)
|
||||
.arg("info")
|
||||
.arg(path)
|
||||
.output()
|
||||
.await;
|
||||
let text = output
|
||||
.map(|o| {
|
||||
format!(
|
||||
"{}{}",
|
||||
String::from_utf8_lossy(&o.stdout),
|
||||
String::from_utf8_lossy(&o.stderr)
|
||||
)
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let k = classify_info(&text);
|
||||
cache.0.lock().unwrap().insert(key, (mtime, size, k));
|
||||
k
|
||||
}
|
||||
|
||||
/// Scan models_dir and classify every `.cmf` (file or sharded dir).
|
||||
pub async fn list_media_models(models_dir: &str, cortiq_bin: &str) -> Vec<MediaModel> {
|
||||
let mut out = Vec::new();
|
||||
let Ok(rd) = std::fs::read_dir(models_dir) else {
|
||||
return out;
|
||||
};
|
||||
for entry in rd.filter_map(|e| e.ok()) {
|
||||
let path = entry.path();
|
||||
let name = path
|
||||
.file_name()
|
||||
.unwrap_or_default()
|
||||
.to_string_lossy()
|
||||
.to_string();
|
||||
if !name.to_ascii_lowercase().ends_with(".cmf") {
|
||||
continue;
|
||||
}
|
||||
let size = crate::import::path_size(&path);
|
||||
if size == 0 {
|
||||
continue; // empty or still being written
|
||||
}
|
||||
let modality = classify_file(&path, cortiq_bin).await;
|
||||
out.push(MediaModel {
|
||||
name: name
|
||||
.trim_end_matches(".cmf")
|
||||
.trim_end_matches(".CMF")
|
||||
.to_string(),
|
||||
path: path.to_string_lossy().to_string(),
|
||||
size_bytes: size,
|
||||
modality,
|
||||
});
|
||||
}
|
||||
out.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
out
|
||||
}
|
||||
|
||||
/// Parameters for one generation, straight from the playground form.
|
||||
#[derive(Clone, Debug, serde::Deserialize)]
|
||||
pub struct MediaParams {
|
||||
pub model: String, // file stem under models_dir
|
||||
pub kind: String, // image | video
|
||||
pub prompt: String,
|
||||
#[serde(default)]
|
||||
pub width: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub height: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub steps: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub cfg: Option<f32>, // image only
|
||||
#[serde(default)]
|
||||
pub seed: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub frames: Option<u32>, // video only
|
||||
#[serde(default)]
|
||||
pub quality: Option<u32>, // video only: AVI JPEG quality
|
||||
/// Optional first/last keyframe for video, base64-encoded binary P6 PPM.
|
||||
#[serde(default)]
|
||||
pub first_frame_b64: Option<String>,
|
||||
#[serde(default)]
|
||||
pub last_frame_b64: Option<String>,
|
||||
}
|
||||
|
||||
/// Live state of one generation.
|
||||
#[derive(Clone, Serialize)]
|
||||
pub struct MediaJob {
|
||||
pub id: String,
|
||||
pub kind: String, // image | video
|
||||
pub model: String,
|
||||
pub prompt: String,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub steps: u32,
|
||||
pub seed: u64,
|
||||
pub frames: Option<u32>,
|
||||
pub state: String, // running | done | error | cancelled
|
||||
pub progress: Option<f32>,
|
||||
pub phase: Option<String>,
|
||||
pub log: Vec<String>,
|
||||
pub started: u64,
|
||||
pub finished: Option<u64>,
|
||||
/// Files this job produced: name → relative kind ("image", "video", "audio", "mp4").
|
||||
pub outputs: Vec<String>,
|
||||
pub out_bytes: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct MediaStore {
|
||||
jobs: Mutex<HashMap<String, MediaJob>>,
|
||||
cancels: Mutex<HashMap<String, Arc<Notify>>>,
|
||||
/// One generation at a time — these models take the whole GPU/RAM budget.
|
||||
busy: Mutex<Option<String>>,
|
||||
}
|
||||
|
||||
impl MediaStore {
|
||||
pub fn new() -> Arc<Self> {
|
||||
Arc::new(Self::default())
|
||||
}
|
||||
pub fn list(&self) -> Vec<MediaJob> {
|
||||
let mut v: Vec<MediaJob> = self.jobs.lock().unwrap().values().cloned().collect();
|
||||
v.sort_by_key(|j| std::cmp::Reverse(j.started));
|
||||
v.truncate(20);
|
||||
v
|
||||
}
|
||||
pub fn get(&self, id: &str) -> Option<MediaJob> {
|
||||
self.jobs.lock().unwrap().get(id).cloned()
|
||||
}
|
||||
pub fn cancel(&self, id: &str) -> bool {
|
||||
let running = matches!(
|
||||
self.jobs.lock().unwrap().get(id).map(|j| j.state.as_str()),
|
||||
Some("running")
|
||||
);
|
||||
if !running {
|
||||
return false;
|
||||
}
|
||||
if let Some(n) = self.cancels.lock().unwrap().get(id) {
|
||||
n.notify_one();
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
/// Remove a settled job and its output directory. Err = still running.
|
||||
pub fn delete(&self, id: &str, media_dir: &Path) -> Result<bool, String> {
|
||||
let job = self.jobs.lock().unwrap().get(id).cloned();
|
||||
let Some(job) = job else { return Ok(false) };
|
||||
if job.state == "running" {
|
||||
return Err("cancel the running generation first".into());
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(media_dir.join(id));
|
||||
self.jobs.lock().unwrap().remove(id);
|
||||
Ok(true)
|
||||
}
|
||||
fn push_line(&self, id: &str, line: String) {
|
||||
if let Some(j) = self.jobs.lock().unwrap().get_mut(id) {
|
||||
j.log.push(line);
|
||||
let n = j.log.len();
|
||||
if n > 120 {
|
||||
j.log.drain(0..n - 120);
|
||||
}
|
||||
}
|
||||
}
|
||||
fn set_progress(&self, id: &str, frac: f32, phase: Option<String>) {
|
||||
if let Some(j) = self.jobs.lock().unwrap().get_mut(id) {
|
||||
let f = frac.clamp(0.0, 1.0);
|
||||
if j.progress.is_none() || f > j.progress.unwrap() {
|
||||
j.progress = Some(f);
|
||||
}
|
||||
if let Some(p) = phase {
|
||||
j.phase = Some(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
fn finish(&self, id: &str, ok: bool, outputs: Vec<String>, out_bytes: u64) {
|
||||
let mut g = self.jobs.lock().unwrap();
|
||||
if let Some(j) = g.get_mut(id) {
|
||||
if j.state == "cancelled" {
|
||||
return;
|
||||
}
|
||||
j.finished = Some(now());
|
||||
j.state = if ok { "done" } else { "error" }.into();
|
||||
if ok {
|
||||
j.progress = Some(1.0);
|
||||
j.outputs = outputs;
|
||||
j.out_bytes = Some(out_bytes);
|
||||
}
|
||||
}
|
||||
drop(g);
|
||||
self.clear_busy(id);
|
||||
self.cancels.lock().unwrap().remove(id);
|
||||
}
|
||||
fn set_cancelled(&self, id: &str) {
|
||||
if let Some(j) = self.jobs.lock().unwrap().get_mut(id) {
|
||||
j.state = "cancelled".into();
|
||||
j.finished = Some(now());
|
||||
}
|
||||
self.clear_busy(id);
|
||||
self.cancels.lock().unwrap().remove(id);
|
||||
}
|
||||
fn clear_busy(&self, id: &str) {
|
||||
let mut b = self.busy.lock().unwrap();
|
||||
if b.as_deref() == Some(id) {
|
||||
*b = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Progress markers in `cortiq imagine`/`animate` stdout. The exact wording has
|
||||
/// varied between releases, so accept anything that carries an X/Y pair near a
|
||||
/// step/frame word, a bare percentage, or the converter-style `@PROGRESS`.
|
||||
fn parse_media_progress(line: &str) -> Option<(f32, Option<String>)> {
|
||||
let l = line.trim();
|
||||
if let Some(rest) = l.strip_prefix("@PROGRESS ") {
|
||||
let mut it = rest.splitn(2, ' ');
|
||||
let frac: f32 = it.next()?.trim().parse().ok()?;
|
||||
let phase = it
|
||||
.next()
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty());
|
||||
return Some((frac.clamp(0.0, 1.0), phase));
|
||||
}
|
||||
let lower = l.to_ascii_lowercase();
|
||||
// "step 12/30", "denoise 3/4", "frame 7/39" — any word followed by x/y
|
||||
for word in ["step", "frame", "sigma", "denois"] {
|
||||
if let Some(pos) = lower.find(word) {
|
||||
let tail = &lower[pos..];
|
||||
if let Some((x, y)) = extract_ratio(tail) {
|
||||
return Some((x / y, Some(l.to_string())));
|
||||
}
|
||||
}
|
||||
}
|
||||
// bare "42%" (avoid matching sizes like "42%faster")
|
||||
if let Some(idx) = lower.find('%') {
|
||||
let head: String = lower[..idx]
|
||||
.chars()
|
||||
.rev()
|
||||
.take_while(|c| c.is_ascii_digit() || *c == '.')
|
||||
.collect();
|
||||
let num: String = head.chars().rev().collect();
|
||||
if let Ok(p) = num.parse::<f32>() {
|
||||
if (0.0..=100.0).contains(&p) {
|
||||
return Some((p / 100.0, None));
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// First "N/M" pair in the string with M > 0.
|
||||
fn extract_ratio(s: &str) -> Option<(f32, f32)> {
|
||||
let bytes = s.as_bytes();
|
||||
let mut i = 0;
|
||||
while i < bytes.len() {
|
||||
if bytes[i].is_ascii_digit() {
|
||||
let start = i;
|
||||
while i < bytes.len() && bytes[i].is_ascii_digit() {
|
||||
i += 1;
|
||||
}
|
||||
if i < bytes.len() && bytes[i] == b'/' {
|
||||
let x: f32 = s[start..i].parse().ok()?;
|
||||
let ystart = i + 1;
|
||||
let mut j = ystart;
|
||||
while j < bytes.len() && bytes[j].is_ascii_digit() {
|
||||
j += 1;
|
||||
}
|
||||
if j > ystart {
|
||||
if let Ok(y) = s[ystart..j].parse::<f32>() {
|
||||
if y > 0.0 && x <= y {
|
||||
return Some((x, y));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Root for generated files: `<data>/media`.
|
||||
pub fn media_dir() -> PathBuf {
|
||||
crate::config::data_dir().join("media")
|
||||
}
|
||||
|
||||
const B64: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
/// Minimal base64 decode (standard alphabet, padding optional) — the keyframes
|
||||
/// are the only base64 input the gateway takes; not worth a dependency.
|
||||
fn b64_decode(s: &str) -> Option<Vec<u8>> {
|
||||
let mut lut = [255u8; 256];
|
||||
for (i, c) in B64.iter().enumerate() {
|
||||
lut[*c as usize] = i as u8;
|
||||
}
|
||||
let mut out = Vec::with_capacity(s.len() * 3 / 4);
|
||||
let mut buf = 0u32;
|
||||
let mut bits = 0u32;
|
||||
for c in s.bytes() {
|
||||
if c == b'=' || c == b'\n' || c == b'\r' {
|
||||
continue;
|
||||
}
|
||||
let v = lut[c as usize];
|
||||
if v == 255 {
|
||||
return None;
|
||||
}
|
||||
buf = (buf << 6) | v as u32;
|
||||
bits += 6;
|
||||
if bits >= 8 {
|
||||
bits -= 8;
|
||||
out.push((buf >> bits) as u8);
|
||||
}
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Kick off a generation. Returns the job id, or an error when another one is
|
||||
/// already running / the model file is missing.
|
||||
pub fn start_generation(
|
||||
store: Arc<MediaStore>,
|
||||
cmf: &crate::config::CmfCfg,
|
||||
p: MediaParams,
|
||||
) -> Result<String, String> {
|
||||
if p.prompt.trim().is_empty() {
|
||||
return Err("empty prompt".into());
|
||||
}
|
||||
let kind = p.kind.as_str();
|
||||
if kind != "image" && kind != "video" {
|
||||
return Err(format!("unknown kind '{kind}'"));
|
||||
}
|
||||
let model_path = Path::new(&cmf.models_dir).join(format!("{}.cmf", p.model));
|
||||
if !model_path.exists() {
|
||||
return Err(format!("model file not found: {}", model_path.display()));
|
||||
}
|
||||
{
|
||||
let mut busy = store.busy.lock().unwrap();
|
||||
if let Some(running) = busy.as_ref() {
|
||||
return Err(format!(
|
||||
"a generation is already running (job {running}) — these models need the whole machine; wait or cancel it"
|
||||
));
|
||||
}
|
||||
*busy = Some(String::new()); // reserved; real id set below
|
||||
}
|
||||
|
||||
let id = gen_media_id();
|
||||
*store.busy.lock().unwrap() = Some(id.clone());
|
||||
|
||||
let dir = media_dir().join(&id);
|
||||
if let Err(e) = std::fs::create_dir_all(&dir) {
|
||||
store.clear_busy(&id);
|
||||
return Err(format!("cannot create media dir {}: {e}", dir.display()));
|
||||
}
|
||||
|
||||
// Optional keyframes (video): decode to PPM files next to the output.
|
||||
let mut first_frame: Option<PathBuf> = None;
|
||||
let mut last_frame: Option<PathBuf> = None;
|
||||
for (b64, slot, name) in [
|
||||
(&p.first_frame_b64, &mut first_frame, "first.ppm"),
|
||||
(&p.last_frame_b64, &mut last_frame, "last.ppm"),
|
||||
] {
|
||||
if let Some(data) = b64.as_deref().filter(|s| !s.is_empty()) {
|
||||
let bytes = b64_decode(data).ok_or("invalid keyframe base64")?;
|
||||
if !bytes.starts_with(b"P6") {
|
||||
return Err("keyframe must be a binary P6 PPM".into());
|
||||
}
|
||||
let path = dir.join(name);
|
||||
std::fs::write(&path, bytes).map_err(|e| format!("write keyframe: {e}"))?;
|
||||
*slot = Some(path);
|
||||
}
|
||||
}
|
||||
|
||||
let width = p.width.unwrap_or(512).clamp(64, 2048);
|
||||
let height = p
|
||||
.height
|
||||
.unwrap_or(if kind == "video" { 288 } else { 512 })
|
||||
.clamp(64, 2048);
|
||||
let steps = p
|
||||
.steps
|
||||
.unwrap_or(if kind == "video" { 4 } else { 30 })
|
||||
.clamp(1, 200);
|
||||
let seed = p.seed.unwrap_or(42);
|
||||
let frames = p.frames.unwrap_or(39).clamp(1, 1000);
|
||||
|
||||
let out_name = if kind == "image" {
|
||||
"out.ppm"
|
||||
} else {
|
||||
"out.avi"
|
||||
};
|
||||
let out_path = dir.join(out_name);
|
||||
|
||||
let mut args: Vec<String> = if kind == "image" {
|
||||
vec![
|
||||
"imagine".into(),
|
||||
model_path.to_string_lossy().to_string(),
|
||||
"--prompt".into(),
|
||||
p.prompt.clone(),
|
||||
"--width".into(),
|
||||
width.to_string(),
|
||||
"--height".into(),
|
||||
height.to_string(),
|
||||
"--steps".into(),
|
||||
steps.to_string(),
|
||||
"--cfg".into(),
|
||||
format!("{}", p.cfg.unwrap_or(4.0)),
|
||||
"--seed".into(),
|
||||
seed.to_string(),
|
||||
"--out".into(),
|
||||
out_path.to_string_lossy().to_string(),
|
||||
]
|
||||
} else {
|
||||
let mut a = vec![
|
||||
"animate".into(),
|
||||
model_path.to_string_lossy().to_string(),
|
||||
"--prompt".into(),
|
||||
p.prompt.clone(),
|
||||
"--width".into(),
|
||||
width.to_string(),
|
||||
"--height".into(),
|
||||
height.to_string(),
|
||||
"--frames".into(),
|
||||
frames.to_string(),
|
||||
"--steps".into(),
|
||||
steps.to_string(),
|
||||
"--seed".into(),
|
||||
seed.to_string(),
|
||||
"--quality".into(),
|
||||
p.quality.unwrap_or(92).clamp(10, 100).to_string(),
|
||||
"--out".into(),
|
||||
out_path.to_string_lossy().to_string(),
|
||||
];
|
||||
if let Some(f) = &first_frame {
|
||||
a.push("--first-frame".into());
|
||||
a.push(f.to_string_lossy().to_string());
|
||||
}
|
||||
if let Some(f) = &last_frame {
|
||||
a.push("--last-frame".into());
|
||||
a.push(f.to_string_lossy().to_string());
|
||||
}
|
||||
a
|
||||
};
|
||||
// keep the model path first for readability in logs
|
||||
let _ = &mut args;
|
||||
|
||||
store.jobs.lock().unwrap().insert(
|
||||
id.clone(),
|
||||
MediaJob {
|
||||
id: id.clone(),
|
||||
kind: kind.to_string(),
|
||||
model: p.model.clone(),
|
||||
prompt: p.prompt.clone(),
|
||||
width,
|
||||
height,
|
||||
steps,
|
||||
seed,
|
||||
frames: (kind == "video").then_some(frames),
|
||||
state: "running".into(),
|
||||
progress: None,
|
||||
phase: Some("starting".into()),
|
||||
log: vec![format!("→ cortiq {}", args.join(" "))],
|
||||
started: now(),
|
||||
finished: None,
|
||||
outputs: Vec::new(),
|
||||
out_bytes: None,
|
||||
},
|
||||
);
|
||||
let cancel = Arc::new(Notify::new());
|
||||
store
|
||||
.cancels
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(id.clone(), cancel.clone());
|
||||
|
||||
let bin = cmf.cortiq_bin.clone();
|
||||
let gpu = cmf.gpu;
|
||||
let ret = id.clone();
|
||||
let dir_clone = dir.clone();
|
||||
let kind_owned = kind.to_string();
|
||||
tokio::spawn(async move {
|
||||
run_generation(
|
||||
store, id, bin, args, gpu, dir_clone, out_path, kind_owned, cancel,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn run_generation(
|
||||
store: Arc<MediaStore>,
|
||||
id: String,
|
||||
bin: String,
|
||||
args: Vec<String>,
|
||||
gpu: bool,
|
||||
dir: PathBuf,
|
||||
out_path: PathBuf,
|
||||
kind: String,
|
||||
cancel: Arc<Notify>,
|
||||
) {
|
||||
let mut cmd = tokio::process::Command::new(&bin);
|
||||
cmd.args(&args)
|
||||
.env("CMF_GPU", if gpu { "1" } else { "0" })
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
let mut child = match cmd.spawn() {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
store.push_line(&id, format!("✗ failed to start cortiq: {e}"));
|
||||
store.finish(&id, false, Vec::new(), 0);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let mut readers = Vec::new();
|
||||
if let Some(o) = child.stdout.take() {
|
||||
readers.push(tokio::spawn(read_lines(store.clone(), id.clone(), o)));
|
||||
}
|
||||
if let Some(e) = child.stderr.take() {
|
||||
readers.push(tokio::spawn(read_lines(store.clone(), id.clone(), e)));
|
||||
}
|
||||
|
||||
let status = tokio::select! {
|
||||
s = child.wait() => s,
|
||||
_ = cancel.notified() => {
|
||||
let _ = child.start_kill();
|
||||
let _ = child.wait().await;
|
||||
for r in readers { r.abort(); }
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
store.push_line(&id, "✗ cancelled".into());
|
||||
store.set_cancelled(&id);
|
||||
return;
|
||||
}
|
||||
};
|
||||
for r in readers {
|
||||
let _ = r.await;
|
||||
}
|
||||
|
||||
let ok = status.map(|s| s.success()).unwrap_or(false);
|
||||
if !ok || !out_path.exists() {
|
||||
store.push_line(&id, "✗ generation failed — see the log above".into());
|
||||
store.finish(&id, false, Vec::new(), 0);
|
||||
return;
|
||||
}
|
||||
|
||||
let mut outputs = vec![if kind == "image" { "image" } else { "video" }.to_string()];
|
||||
if kind == "video" {
|
||||
// `animate` writes a .wav next to the .avi
|
||||
if dir.join("out.wav").exists() {
|
||||
outputs.push("audio".into());
|
||||
}
|
||||
// opportunistic browser-friendly remux when ffmpeg is present
|
||||
if let Ok(p) = which_ffmpeg() {
|
||||
store.push_line(&id, "→ remuxing to mp4 (ffmpeg found)".into());
|
||||
let mp4 = dir.join("out.mp4");
|
||||
let mut f = tokio::process::Command::new(p);
|
||||
f.arg("-y").arg("-i").arg(dir.join("out.avi"));
|
||||
if dir.join("out.wav").exists() {
|
||||
f.arg("-i")
|
||||
.arg(dir.join("out.wav"))
|
||||
.args(["-map", "0:v", "-map", "1:a", "-c:a", "aac"]);
|
||||
}
|
||||
f.args(["-c:v", "libx264", "-pix_fmt", "yuv420p"]).arg(&mp4);
|
||||
f.stdout(Stdio::null()).stderr(Stdio::null());
|
||||
if matches!(f.status().await, Ok(s) if s.success()) && mp4.exists() {
|
||||
outputs.push("mp4".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
let bytes = crate::import::path_size(&out_path);
|
||||
store.push_line(
|
||||
&id,
|
||||
format!("✓ done — {}", crate::import::format_bytes(bytes)),
|
||||
);
|
||||
store.finish(&id, true, outputs, bytes);
|
||||
}
|
||||
|
||||
/// Strip ANSI color escapes that `tracing` writes even into a pipe.
|
||||
fn strip_ansi(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
let mut chars = s.chars();
|
||||
while let Some(c) = chars.next() {
|
||||
if c == '\u{1b}' {
|
||||
for d in chars.by_ref() {
|
||||
if d.is_ascii_alphabetic() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
out.push(c);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
async fn read_lines(store: Arc<MediaStore>, id: String, stream: impl tokio::io::AsyncRead + Unpin) {
|
||||
let mut lines = BufReader::new(stream).lines();
|
||||
while let Ok(Some(l)) = lines.next_line().await {
|
||||
let l = strip_ansi(l.trim());
|
||||
if l.is_empty() {
|
||||
continue;
|
||||
}
|
||||
// the Metal backend logs one line per pipeline — dozens per run, all identical
|
||||
if l.contains("Metal GPU path") {
|
||||
continue;
|
||||
}
|
||||
if let Some((frac, phase)) = parse_media_progress(&l) {
|
||||
store.set_progress(&id, frac, phase);
|
||||
} else {
|
||||
store.push_line(&id, l);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn which_ffmpeg() -> Result<String, ()> {
|
||||
for p in [
|
||||
"ffmpeg",
|
||||
"/usr/local/bin/ffmpeg",
|
||||
"/opt/homebrew/bin/ffmpeg",
|
||||
] {
|
||||
if std::process::Command::new(p)
|
||||
.arg("-version")
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return Ok(p.to_string());
|
||||
}
|
||||
}
|
||||
Err(())
|
||||
}
|
||||
|
||||
/// (relative file, content-type) for an output kind of a finished job.
|
||||
pub fn output_file(kind: &str) -> Option<(&'static str, &'static str)> {
|
||||
match kind {
|
||||
"image" => Some(("out.ppm", "image/x-portable-pixmap")),
|
||||
"video" => Some(("out.avi", "video/x-msvideo")),
|
||||
"audio" => Some(("out.wav", "audio/wav")),
|
||||
"mp4" => Some(("out.mp4", "video/mp4")),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn progress_parses_common_shapes() {
|
||||
assert_eq!(parse_media_progress("step 15/30").unwrap().0, 0.5);
|
||||
assert_eq!(
|
||||
parse_media_progress("Denoising step 3/4 ...").unwrap().0,
|
||||
0.75
|
||||
);
|
||||
assert_eq!(parse_media_progress("frame 39/39 decoded").unwrap().0, 1.0);
|
||||
assert_eq!(parse_media_progress("@PROGRESS 0.25 vae").unwrap().0, 0.25);
|
||||
assert!(parse_media_progress("loaded 2361 tensors").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_by_arch() {
|
||||
assert_eq!(classify_info(" Arch: lumina2\n"), Modality::Image);
|
||||
assert_eq!(classify_info(" Arch: mmh3\n"), Modality::Video);
|
||||
assert_eq!(classify_info(" Arch: qwen3\n"), Modality::Chat);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn b64_roundtrip() {
|
||||
assert_eq!(b64_decode("UDYgd2g=").unwrap(), b"P6 wh");
|
||||
assert!(b64_decode("!!").is_none());
|
||||
}
|
||||
}
|
||||
+35
-1
@@ -542,6 +542,11 @@ impl Pipeline {
|
||||
cost_usd: 0.0,
|
||||
failover: attempt > 0,
|
||||
};
|
||||
// for the client-facing usage estimate when the provider
|
||||
// sends none: the prompt size is known here, the reply is
|
||||
// measured inside the tap
|
||||
let prompt_chars: usize =
|
||||
req.messages.iter().map(|m| m.content.chars().count()).sum();
|
||||
let tapped = tap_stream(
|
||||
stream,
|
||||
state.stats.clone(),
|
||||
@@ -551,6 +556,7 @@ impl Pipeline {
|
||||
protocol,
|
||||
directive,
|
||||
started,
|
||||
prompt_chars,
|
||||
);
|
||||
return Ok((info, Box::pin(tapped)));
|
||||
}
|
||||
@@ -722,6 +728,7 @@ fn tap_stream(
|
||||
protocol: String,
|
||||
directive: String,
|
||||
started: Instant,
|
||||
prompt_chars: usize,
|
||||
) -> impl futures::Stream<Item = Result<bytes::Bytes>> {
|
||||
use futures::StreamExt;
|
||||
async_stream::stream! {
|
||||
@@ -734,9 +741,11 @@ fn tap_stream(
|
||||
let mut errored: Option<String> = None;
|
||||
|
||||
while let Some(item) = inner.next().await {
|
||||
let mut done_marker = false;
|
||||
match &item {
|
||||
Ok(bytes) => {
|
||||
if let Ok(s) = std::str::from_utf8(bytes) {
|
||||
done_marker = s.contains("data: [DONE]");
|
||||
buf.push_str(s);
|
||||
while let Some(idx) = buf.find("\n\n") {
|
||||
let evt: String = buf.drain(..idx + 2).collect();
|
||||
@@ -774,12 +783,37 @@ fn tap_stream(
|
||||
}
|
||||
Err(e) => errored = Some(e.to_string()),
|
||||
}
|
||||
// The local cortiq server (and some providers) never send a usage
|
||||
// chunk. Clients still deserve token counts, so ahead of [DONE] we
|
||||
// inject one final OpenAI-shaped usage event with the gateway's
|
||||
// estimate (~4 chars/token), flagged `cortiq_estimated` so nobody
|
||||
// mistakes it for the tokenizer's truth.
|
||||
if done_marker && !got_usage {
|
||||
let est_completion = (delta_chars / 4) as u32;
|
||||
let est_prompt = (prompt_chars / 4) as u32;
|
||||
let evt = serde_json::json!({
|
||||
"object": "chat.completion.chunk",
|
||||
"model": info.selected_model,
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": est_prompt,
|
||||
"completion_tokens": est_completion,
|
||||
"total_tokens": est_prompt + est_completion,
|
||||
},
|
||||
"cortiq_estimated": true,
|
||||
});
|
||||
yield Ok(bytes::Bytes::from(format!("data: {evt}\n\n")));
|
||||
prompt_tokens = est_prompt;
|
||||
completion_tokens = est_completion;
|
||||
got_usage = true;
|
||||
}
|
||||
yield item;
|
||||
}
|
||||
|
||||
if !got_usage && completion_tokens == 0 {
|
||||
// estimate when the server didn't return usage (~4 chars per token)
|
||||
// stream ended without a [DONE] marker — estimate for the stats
|
||||
completion_tokens = (delta_chars / 4) as u32;
|
||||
prompt_tokens = (prompt_chars / 4) as u32;
|
||||
}
|
||||
let cost = provider.price(prompt_tokens, completion_tokens);
|
||||
stats.record(RequestRecord {
|
||||
|
||||
@@ -71,6 +71,8 @@ pub struct AppState {
|
||||
pub stats: Arc<Stats>,
|
||||
pub promotion: Arc<crate::promotion::Promotion>,
|
||||
pub imports: Arc<crate::import::JobStore>,
|
||||
/// Image/video generation jobs (cortiq imagine / animate).
|
||||
pub media: Arc<crate::media::MediaStore>,
|
||||
/// Managed local CMF model server (install/update/spawn lifecycle + status).
|
||||
pub cmf: Arc<crate::cmf_runtime::CmfRuntime>,
|
||||
pub cache: Arc<SemanticCache>,
|
||||
@@ -113,6 +115,7 @@ impl SharedState {
|
||||
let stats = Stats::new(&cfg.stats);
|
||||
let promotion = crate::promotion::Promotion::new(promotion_cfg(&cfg.shadow));
|
||||
let imports = crate::import::JobStore::new();
|
||||
let media = crate::media::MediaStore::new();
|
||||
let cmf = crate::cmf_runtime::CmfRuntime::new();
|
||||
let cache = SemanticCache::new(&cfg.cache);
|
||||
let router_status = RouterLastStatus::default();
|
||||
@@ -122,6 +125,7 @@ impl SharedState {
|
||||
stats,
|
||||
promotion,
|
||||
imports,
|
||||
media,
|
||||
cmf,
|
||||
cache,
|
||||
secrets,
|
||||
|
||||
Vendored
+6
@@ -585,6 +585,12 @@ body.modal-open { overflow: hidden; }
|
||||
/* Featured gallery: the list grows as new models are published, so give it more
|
||||
room than the search grid before it starts scrolling. */
|
||||
.featured-section .hf-grid { max-height: min(62vh, 760px); }
|
||||
/* Playground media tabs: generated image / video frame */
|
||||
.media-out { display: flex; flex-direction: column; gap: 8px; }
|
||||
.media-canvas {
|
||||
max-width: 100%; height: auto; border-radius: 8px;
|
||||
border: 1px solid var(--border, #2a2f3a); background: #000;
|
||||
}
|
||||
/* Live download progress inside a featured card (empty until a job runs) */
|
||||
.card-prog:empty { display: none; }
|
||||
.card-prog { margin-top: 6px; }
|
||||
|
||||
Vendored
+47
-2
@@ -118,6 +118,14 @@ export const api = {
|
||||
registerImport: (job) =>
|
||||
j("POST", "/import/" + encodeURIComponent(job) + "/register"),
|
||||
|
||||
// media generation (cortiq imagine / animate over local .cmf)
|
||||
mediaModels: () => j("GET", "/media/models"),
|
||||
mediaGenerate: (b) => j("POST", "/media", b),
|
||||
mediaJobs: () => j("GET", "/media"),
|
||||
mediaJob: (id) => j("GET", "/media/" + encodeURIComponent(id)),
|
||||
mediaCancel: (id) => j("POST", "/media/" + encodeURIComponent(id) + "/cancel"),
|
||||
mediaDelete: (id) => j("DELETE", "/media/" + encodeURIComponent(id)),
|
||||
|
||||
// first-run onboarding
|
||||
setupStatus: () => j("GET", "/setup/status"),
|
||||
// Connects every finished download as a local model unless register === false.
|
||||
@@ -155,6 +163,10 @@ export async function testStream(body, onDelta) {
|
||||
const reader = r.body.getReader();
|
||||
const dec = new TextDecoder();
|
||||
let buf = "";
|
||||
const t0 = performance.now();
|
||||
let ttftMs = null; // time to first content token
|
||||
let usage = null; // OpenAI-style final-chunk usage, when the provider sends it
|
||||
let usageEstimated = false; // gateway-injected estimate rather than tokenizer truth
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
@@ -167,8 +179,16 @@ export async function testStream(body, onDelta) {
|
||||
const data = line.slice(5).trim();
|
||||
if (!data || data === "[DONE]") continue;
|
||||
try {
|
||||
const d = JSON.parse(data).choices?.[0]?.delta?.content;
|
||||
if (d) onDelta(d);
|
||||
const chunk = JSON.parse(data);
|
||||
if (chunk.usage) {
|
||||
usage = chunk.usage;
|
||||
usageEstimated = !!chunk.cortiq_estimated;
|
||||
}
|
||||
const d = chunk.choices?.[0]?.delta?.content;
|
||||
if (d) {
|
||||
if (ttftMs == null) ttftMs = Math.round(performance.now() - t0);
|
||||
onDelta(d);
|
||||
}
|
||||
} catch {
|
||||
/* ignore malformed chunk */
|
||||
}
|
||||
@@ -182,5 +202,30 @@ export async function testStream(body, onDelta) {
|
||||
selected_model: h.get("x-cortiq-selected-model") || "",
|
||||
route_source: h.get("x-cortiq-route-source") || "",
|
||||
cost_usd: parseFloat(h.get("x-cortiq-cost-usd") || "0"),
|
||||
usage,
|
||||
usage_estimated: usageEstimated,
|
||||
ttft_ms: ttftMs,
|
||||
};
|
||||
}
|
||||
|
||||
// Fetch a media job's output with the admin token and hand back an object URL
|
||||
// (plain <img src>/<video src> cannot carry the Authorization header).
|
||||
export async function mediaFileUrl(job, kind) {
|
||||
const tok = getToken();
|
||||
const headers = {};
|
||||
if (tok) headers["Authorization"] = "Bearer " + tok;
|
||||
const r = await fetch(`${BASE}/media/${encodeURIComponent(job)}/file/${encodeURIComponent(kind)}`, { headers });
|
||||
if (!r.ok) throw new Error((await r.text()) || r.statusText);
|
||||
return { url: URL.createObjectURL(await r.blob()), size: +(r.headers.get("content-length") || 0) };
|
||||
}
|
||||
|
||||
// Same fetch, raw bytes — for outputs the browser cannot display natively
|
||||
// (P6 PPM, MJPEG AVI) that we decode in JS.
|
||||
export async function mediaFileBytes(job, kind) {
|
||||
const tok = getToken();
|
||||
const headers = {};
|
||||
if (tok) headers["Authorization"] = "Bearer " + tok;
|
||||
const r = await fetch(`${BASE}/media/${encodeURIComponent(job)}/file/${encodeURIComponent(kind)}`, { headers });
|
||||
if (!r.ok) throw new Error((await r.text()) || r.statusText);
|
||||
return r.arrayBuffer();
|
||||
}
|
||||
|
||||
Vendored
+38
@@ -468,4 +468,42 @@ export default {
|
||||
"onboarding.deleteFile": "Die .cmf-Datei von der Festplatte löschen",
|
||||
"onboarding.confirmDelete": "„{id}“ löschen? Die .cmf-Datei wird von der Festplatte entfernt.",
|
||||
"onboarding.unregistered": "Ebenfalls abgemeldet: {ids}",
|
||||
|
||||
// playground: media generation + streaming metrics
|
||||
"pg.tab.chat": "Chat",
|
||||
"pg.tab.image": "Bild",
|
||||
"pg.tab.video": "Video",
|
||||
"pg.generate": "Generieren",
|
||||
"pg.generating": "Wird generiert…",
|
||||
"pg.genFailed": "Generierung fehlgeschlagen — siehe Log.",
|
||||
"pg.result": "Ergebnis",
|
||||
"pg.genMetrics": "Generierungsmetriken",
|
||||
"pg.imagePromptPlaceholder": "ein Rotfuchs im Schnee bei Sonnenuntergang, fotorealistisch, detailliertes Fell",
|
||||
"pg.videoPromptPlaceholder": "ein Corgi mit Kochmütze wirft einen Pfannkuchen, Brutzeln und ein fröhliches Bellen",
|
||||
"pg.imageEmpty": "Das generierte Bild erscheint hier.",
|
||||
"pg.videoEmpty": "Der generierte Clip (mit Ton) erscheint hier.",
|
||||
"pg.width": "Breite",
|
||||
"pg.height": "Höhe",
|
||||
"pg.steps": "Schritte",
|
||||
"pg.seed": "Seed",
|
||||
"pg.randomSeed": "Zufälliger Seed",
|
||||
"pg.cfgHint": "Guidance-Stärke; ≤ 1 schaltet CFG ab und halbiert die Arbeit.",
|
||||
"pg.frames": "Frames",
|
||||
"pg.framesHint": "Bei 24 fps; auf das Raster des Modells gerundet.",
|
||||
"pg.quality": "JPEG-Qualität",
|
||||
"pg.videoSizeHint": "Vielfache von 32; die trainierte kurze Kante ist 768.",
|
||||
"pg.videoStepsHint": "Die Turbo-LoRA ist auf 4 Schritte trainiert.",
|
||||
"pg.firstFrame": "Erstes Frame (optional)",
|
||||
"pg.lastFrame": "Letztes Frame (optional)",
|
||||
"pg.keyframeHint": "Macht den Lauf zu Keyframe-to-Video.",
|
||||
"pg.resolution": "Auflösung",
|
||||
"pg.genTime": "Generierungszeit",
|
||||
"pg.timePerStep": "Pro Schritt",
|
||||
"pg.timePerFrame": "Pro Frame",
|
||||
"pg.fileSize": "Dateigröße",
|
||||
"pg.duration": "Dauer",
|
||||
"pg.play": "Abspielen",
|
||||
"pg.noMediaModels": "Noch kein lokales {kind}-Modell. Lade eines (z. B. aus der infosave-Sammlung) unter Import herunter:",
|
||||
"pg.tokSpeed": "Geschwindigkeit",
|
||||
"pg.ttft": "Erstes Token",
|
||||
};
|
||||
|
||||
Vendored
+38
@@ -461,4 +461,42 @@ export default {
|
||||
"onboarding.deleteFile": "Delete the .cmf file from disk",
|
||||
"onboarding.confirmDelete": "Delete '{id}'? The .cmf file will be removed from disk.",
|
||||
"onboarding.unregistered": "Also unregistered: {ids}",
|
||||
|
||||
// playground: media generation + streaming metrics
|
||||
"pg.tab.chat": "Chat",
|
||||
"pg.tab.image": "Image",
|
||||
"pg.tab.video": "Video",
|
||||
"pg.generate": "Generate",
|
||||
"pg.generating": "Generating…",
|
||||
"pg.genFailed": "Generation failed — see the log.",
|
||||
"pg.result": "Result",
|
||||
"pg.genMetrics": "Generation metrics",
|
||||
"pg.imagePromptPlaceholder": "a red fox sitting in snow at sunset, photorealistic, detailed fur",
|
||||
"pg.videoPromptPlaceholder": "a corgi in a chef hat flipping a pancake, sizzling sounds and a cheerful bark",
|
||||
"pg.imageEmpty": "The generated image will appear here.",
|
||||
"pg.videoEmpty": "The generated clip (with audio) will appear here.",
|
||||
"pg.width": "Width",
|
||||
"pg.height": "Height",
|
||||
"pg.steps": "Steps",
|
||||
"pg.seed": "Seed",
|
||||
"pg.randomSeed": "Random seed",
|
||||
"pg.cfgHint": "Guidance scale; ≤ 1 disables CFG and halves the work.",
|
||||
"pg.frames": "Frames",
|
||||
"pg.framesHint": "At 24 fps; snapped to the model's grid.",
|
||||
"pg.quality": "JPEG quality",
|
||||
"pg.videoSizeHint": "Multiples of 32; the trained short edge is 768.",
|
||||
"pg.videoStepsHint": "The Turbo LoRA is trained for 4 steps.",
|
||||
"pg.firstFrame": "First frame (optional)",
|
||||
"pg.lastFrame": "Last frame (optional)",
|
||||
"pg.keyframeHint": "Turns the run into keyframe-to-video.",
|
||||
"pg.resolution": "Resolution",
|
||||
"pg.genTime": "Generation time",
|
||||
"pg.timePerStep": "Per step",
|
||||
"pg.timePerFrame": "Per frame",
|
||||
"pg.fileSize": "File size",
|
||||
"pg.duration": "Duration",
|
||||
"pg.play": "Play",
|
||||
"pg.noMediaModels": "No local {kind} model yet. Download one (e.g. from the infosave collection) in Import:",
|
||||
"pg.tokSpeed": "Speed",
|
||||
"pg.ttft": "First token",
|
||||
};
|
||||
|
||||
Vendored
+38
@@ -468,4 +468,42 @@ export default {
|
||||
"onboarding.deleteFile": "Eliminar el archivo .cmf del disco",
|
||||
"onboarding.confirmDelete": "¿Eliminar «{id}»? El archivo .cmf se borrará del disco.",
|
||||
"onboarding.unregistered": "También se dio de baja: {ids}",
|
||||
|
||||
// playground: media generation + streaming metrics
|
||||
"pg.tab.chat": "Chat",
|
||||
"pg.tab.image": "Imagen",
|
||||
"pg.tab.video": "Vídeo",
|
||||
"pg.generate": "Generar",
|
||||
"pg.generating": "Generando…",
|
||||
"pg.genFailed": "La generación falló — revisa el registro.",
|
||||
"pg.result": "Resultado",
|
||||
"pg.genMetrics": "Métricas de generación",
|
||||
"pg.imagePromptPlaceholder": "un zorro rojo en la nieve al atardecer, fotorrealista, pelaje detallado",
|
||||
"pg.videoPromptPlaceholder": "un corgi con gorro de chef voltea una tortita, chisporroteo y un ladrido alegre",
|
||||
"pg.imageEmpty": "La imagen generada aparecerá aquí.",
|
||||
"pg.videoEmpty": "El clip generado (con audio) aparecerá aquí.",
|
||||
"pg.width": "Ancho",
|
||||
"pg.height": "Alto",
|
||||
"pg.steps": "Pasos",
|
||||
"pg.seed": "Seed",
|
||||
"pg.randomSeed": "Seed aleatorio",
|
||||
"pg.cfgHint": "Escala de guidance; ≤ 1 desactiva el CFG y reduce el trabajo a la mitad.",
|
||||
"pg.frames": "Fotogramas",
|
||||
"pg.framesHint": "A 24 fps; se ajusta a la rejilla del modelo.",
|
||||
"pg.quality": "Calidad JPEG",
|
||||
"pg.videoSizeHint": "Múltiplos de 32; el lado corto entrenado es 768.",
|
||||
"pg.videoStepsHint": "La LoRA Turbo está entrenada para 4 pasos.",
|
||||
"pg.firstFrame": "Primer fotograma (opcional)",
|
||||
"pg.lastFrame": "Último fotograma (opcional)",
|
||||
"pg.keyframeHint": "Convierte la ejecución en keyframe-to-video.",
|
||||
"pg.resolution": "Resolución",
|
||||
"pg.genTime": "Tiempo de generación",
|
||||
"pg.timePerStep": "Por paso",
|
||||
"pg.timePerFrame": "Por fotograma",
|
||||
"pg.fileSize": "Tamaño del archivo",
|
||||
"pg.duration": "Duración",
|
||||
"pg.play": "Reproducir",
|
||||
"pg.noMediaModels": "Aún no hay un modelo {kind} local. Descarga uno (p. ej. de la colección infosave) en Importar:",
|
||||
"pg.tokSpeed": "Velocidad",
|
||||
"pg.ttft": "Primer token",
|
||||
};
|
||||
|
||||
Vendored
+38
@@ -468,4 +468,42 @@ export default {
|
||||
"onboarding.deleteFile": "Supprimer le fichier .cmf du disque",
|
||||
"onboarding.confirmDelete": "Supprimer « {id} » ? Le fichier .cmf sera retiré du disque.",
|
||||
"onboarding.unregistered": "Également désenregistré : {ids}",
|
||||
|
||||
// playground: media generation + streaming metrics
|
||||
"pg.tab.chat": "Chat",
|
||||
"pg.tab.image": "Image",
|
||||
"pg.tab.video": "Vidéo",
|
||||
"pg.generate": "Générer",
|
||||
"pg.generating": "Génération…",
|
||||
"pg.genFailed": "Échec de la génération — voir le journal.",
|
||||
"pg.result": "Résultat",
|
||||
"pg.genMetrics": "Métriques de génération",
|
||||
"pg.imagePromptPlaceholder": "un renard roux dans la neige au coucher du soleil, photoréaliste, fourrure détaillée",
|
||||
"pg.videoPromptPlaceholder": "un corgi en toque de chef fait sauter une crêpe, grésillements et aboiement joyeux",
|
||||
"pg.imageEmpty": "L'image générée apparaîtra ici.",
|
||||
"pg.videoEmpty": "Le clip généré (avec le son) apparaîtra ici.",
|
||||
"pg.width": "Largeur",
|
||||
"pg.height": "Hauteur",
|
||||
"pg.steps": "Étapes",
|
||||
"pg.seed": "Seed",
|
||||
"pg.randomSeed": "Seed aléatoire",
|
||||
"pg.cfgHint": "Force du guidance ; ≤ 1 désactive le CFG et divise le travail par deux.",
|
||||
"pg.frames": "Images",
|
||||
"pg.framesHint": "À 24 fps ; arrondi à la grille du modèle.",
|
||||
"pg.quality": "Qualité JPEG",
|
||||
"pg.videoSizeHint": "Multiples de 32 ; le petit côté entraîné est 768.",
|
||||
"pg.videoStepsHint": "La LoRA Turbo est entraînée pour 4 étapes.",
|
||||
"pg.firstFrame": "Première image (optionnel)",
|
||||
"pg.lastFrame": "Dernière image (optionnel)",
|
||||
"pg.keyframeHint": "Transforme la génération en keyframe-to-video.",
|
||||
"pg.resolution": "Résolution",
|
||||
"pg.genTime": "Temps de génération",
|
||||
"pg.timePerStep": "Par étape",
|
||||
"pg.timePerFrame": "Par image",
|
||||
"pg.fileSize": "Taille du fichier",
|
||||
"pg.duration": "Durée",
|
||||
"pg.play": "Lire",
|
||||
"pg.noMediaModels": "Pas encore de modèle {kind} local. Téléchargez-en un (par ex. de la collection infosave) dans Importer :",
|
||||
"pg.tokSpeed": "Vitesse",
|
||||
"pg.ttft": "Premier token",
|
||||
};
|
||||
|
||||
Vendored
+38
@@ -460,4 +460,42 @@ export default {
|
||||
"onboarding.deleteFile": "Удалить файл .cmf с диска",
|
||||
"onboarding.confirmDelete": "Удалить «{id}»? Файл .cmf будет удалён с диска.",
|
||||
"onboarding.unregistered": "Также отключено из моделей: {ids}",
|
||||
|
||||
// playground: media generation + streaming metrics
|
||||
"pg.tab.chat": "Чат",
|
||||
"pg.tab.image": "Изображение",
|
||||
"pg.tab.video": "Видео",
|
||||
"pg.generate": "Сгенерировать",
|
||||
"pg.generating": "Генерация…",
|
||||
"pg.genFailed": "Генерация не удалась — смотрите лог.",
|
||||
"pg.result": "Результат",
|
||||
"pg.genMetrics": "Метрики генерации",
|
||||
"pg.imagePromptPlaceholder": "рыжая лиса в снегу на закате, фотореалистично, детальный мех",
|
||||
"pg.videoPromptPlaceholder": "корги в поварском колпаке подбрасывает блин, шипение сковороды и весёлый лай",
|
||||
"pg.imageEmpty": "Сгенерированное изображение появится здесь.",
|
||||
"pg.videoEmpty": "Сгенерированный ролик (со звуком) появится здесь.",
|
||||
"pg.width": "Ширина",
|
||||
"pg.height": "Высота",
|
||||
"pg.steps": "Шаги",
|
||||
"pg.seed": "Seed",
|
||||
"pg.randomSeed": "Случайный seed",
|
||||
"pg.cfgHint": "Сила guidance; ≤ 1 отключает CFG и вдвое ускоряет генерацию.",
|
||||
"pg.frames": "Кадры",
|
||||
"pg.framesHint": "При 24 fps; округляется до сетки модели.",
|
||||
"pg.quality": "Качество JPEG",
|
||||
"pg.videoSizeHint": "Кратно 32; обученная короткая сторона — 768.",
|
||||
"pg.videoStepsHint": "Turbo LoRA обучена на 4 шага.",
|
||||
"pg.firstFrame": "Первый кадр (необязательно)",
|
||||
"pg.lastFrame": "Последний кадр (необязательно)",
|
||||
"pg.keyframeHint": "Превращает запуск в keyframe-to-video.",
|
||||
"pg.resolution": "Разрешение",
|
||||
"pg.genTime": "Время генерации",
|
||||
"pg.timePerStep": "На шаг",
|
||||
"pg.timePerFrame": "На кадр",
|
||||
"pg.fileSize": "Размер файла",
|
||||
"pg.duration": "Длительность",
|
||||
"pg.play": "Играть",
|
||||
"pg.noMediaModels": "Локальной модели ({kind}) ещё нет. Скачайте её (например, из коллекции infosave) во вкладке Импорт:",
|
||||
"pg.tokSpeed": "Скорость",
|
||||
"pg.ttft": "Первый токен",
|
||||
};
|
||||
|
||||
Vendored
+38
@@ -468,4 +468,42 @@ export default {
|
||||
"onboarding.deleteFile": ".cmf dosyasını diskten sil",
|
||||
"onboarding.confirmDelete": "“{id}” silinsin mi? .cmf dosyası diskten kaldırılacak.",
|
||||
"onboarding.unregistered": "Ayrıca kaydı kaldırıldı: {ids}",
|
||||
|
||||
// playground: media generation + streaming metrics
|
||||
"pg.tab.chat": "Sohbet",
|
||||
"pg.tab.image": "Görsel",
|
||||
"pg.tab.video": "Video",
|
||||
"pg.generate": "Oluştur",
|
||||
"pg.generating": "Oluşturuluyor…",
|
||||
"pg.genFailed": "Oluşturma başarısız — günlüğe bakın.",
|
||||
"pg.result": "Sonuç",
|
||||
"pg.genMetrics": "Oluşturma metrikleri",
|
||||
"pg.imagePromptPlaceholder": "gün batımında karda oturan kızıl tilki, fotogerçekçi, ayrıntılı tüyler",
|
||||
"pg.videoPromptPlaceholder": "aşçı şapkalı bir corgi krep çeviriyor, cızırtı sesleri ve neşeli bir havlama",
|
||||
"pg.imageEmpty": "Oluşturulan görsel burada görünecek.",
|
||||
"pg.videoEmpty": "Oluşturulan klip (sesiyle) burada görünecek.",
|
||||
"pg.width": "Genişlik",
|
||||
"pg.height": "Yükseklik",
|
||||
"pg.steps": "Adım",
|
||||
"pg.seed": "Seed",
|
||||
"pg.randomSeed": "Rastgele seed",
|
||||
"pg.cfgHint": "Guidance gücü; ≤ 1 CFG'yi kapatır ve işi yarıya indirir.",
|
||||
"pg.frames": "Kare",
|
||||
"pg.framesHint": "24 fps'te; modelin ızgarasına yuvarlanır.",
|
||||
"pg.quality": "JPEG kalitesi",
|
||||
"pg.videoSizeHint": "32'nin katları; eğitilen kısa kenar 768.",
|
||||
"pg.videoStepsHint": "Turbo LoRA 4 adım için eğitildi.",
|
||||
"pg.firstFrame": "İlk kare (isteğe bağlı)",
|
||||
"pg.lastFrame": "Son kare (isteğe bağlı)",
|
||||
"pg.keyframeHint": "Çalışmayı keyframe-to-video'ya çevirir.",
|
||||
"pg.resolution": "Çözünürlük",
|
||||
"pg.genTime": "Oluşturma süresi",
|
||||
"pg.timePerStep": "Adım başına",
|
||||
"pg.timePerFrame": "Kare başına",
|
||||
"pg.fileSize": "Dosya boyutu",
|
||||
"pg.duration": "Süre",
|
||||
"pg.play": "Oynat",
|
||||
"pg.noMediaModels": "Henüz yerel bir {kind} modeli yok. İçe aktarma sekmesinden bir tane indirin (ör. infosave koleksiyonundan):",
|
||||
"pg.tokSpeed": "Hız",
|
||||
"pg.ttft": "İlk token",
|
||||
};
|
||||
|
||||
Vendored
+38
@@ -468,4 +468,42 @@ export default {
|
||||
"onboarding.deleteFile": "从磁盘删除该 .cmf 文件",
|
||||
"onboarding.confirmDelete": "删除 “{id}”？该 .cmf 文件将从磁盘移除。",
|
||||
"onboarding.unregistered": "同时已取消注册：{ids}",
|
||||
|
||||
// playground: media generation + streaming metrics
|
||||
"pg.tab.chat": "对话",
|
||||
"pg.tab.image": "图像",
|
||||
"pg.tab.video": "视频",
|
||||
"pg.generate": "生成",
|
||||
"pg.generating": "生成中……",
|
||||
"pg.genFailed": "生成失败 — 请查看日志。",
|
||||
"pg.result": "结果",
|
||||
"pg.genMetrics": "生成指标",
|
||||
"pg.imagePromptPlaceholder": "夕阳下雪地里的红狐狸，照片级真实感，毛发细节丰富",
|
||||
"pg.videoPromptPlaceholder": "戴厨师帽的柯基翻煎饼，滋滋声和欢快的犬吠",
|
||||
"pg.imageEmpty": "生成的图像将显示在这里。",
|
||||
"pg.videoEmpty": "生成的短片（含音频）将显示在这里。",
|
||||
"pg.width": "宽度",
|
||||
"pg.height": "高度",
|
||||
"pg.steps": "步数",
|
||||
"pg.seed": "Seed",
|
||||
"pg.randomSeed": "随机 seed",
|
||||
"pg.cfgHint": "引导强度；≤ 1 关闭 CFG，工作量减半。",
|
||||
"pg.frames": "帧数",
|
||||
"pg.framesHint": "24 fps；按模型网格取整。",
|
||||
"pg.quality": "JPEG 质量",
|
||||
"pg.videoSizeHint": "32 的倍数；训练短边为 768。",
|
||||
"pg.videoStepsHint": "Turbo LoRA 按 4 步训练。",
|
||||
"pg.firstFrame": "首帧（可选）",
|
||||
"pg.lastFrame": "末帧（可选）",
|
||||
"pg.keyframeHint": "使运行变为关键帧生视频。",
|
||||
"pg.resolution": "分辨率",
|
||||
"pg.genTime": "生成耗时",
|
||||
"pg.timePerStep": "每步",
|
||||
"pg.timePerFrame": "每帧",
|
||||
"pg.fileSize": "文件大小",
|
||||
"pg.duration": "时长",
|
||||
"pg.play": "播放",
|
||||
"pg.noMediaModels": "还没有本地{kind}模型。请在“导入模型”中下载一个（例如 infosave 合集）：",
|
||||
"pg.tokSpeed": "速度",
|
||||
"pg.ttft": "首个 token",
|
||||
};
|
||||
|
||||
Vendored
+167
@@ -0,0 +1,167 @@
|
||||
// Browser-side codecs for cortiq's dependency-free media outputs:
|
||||
// - P6 PPM → canvas (cortiq imagine links no image library)
|
||||
// - MJPEG+PCM AVI → frame list + canvas player synced to the .wav
|
||||
// (cortiq animate never touches ffmpeg; neither do we)
|
||||
|
||||
/// Parse binary P6 PPM into {width, height, rgb: Uint8Array}.
|
||||
export function parsePpm(buf) {
|
||||
const b = new Uint8Array(buf);
|
||||
// header: "P6" <ws> width <ws> height <ws> maxval <single ws> raster
|
||||
let pos = 0;
|
||||
const isWs = (c) => c === 32 || c === 9 || c === 10 || c === 13;
|
||||
function token() {
|
||||
while (pos < b.length && (isWs(b[pos]) || b[pos] === 35 /* # comment */)) {
|
||||
if (b[pos] === 35) while (pos < b.length && b[pos] !== 10) pos++;
|
||||
else pos++;
|
||||
}
|
||||
let start = pos;
|
||||
while (pos < b.length && !isWs(b[pos])) pos++;
|
||||
return String.fromCharCode(...b.slice(start, pos));
|
||||
}
|
||||
if (token() !== "P6") throw new Error("not a P6 PPM");
|
||||
const width = parseInt(token());
|
||||
const height = parseInt(token());
|
||||
const maxval = parseInt(token());
|
||||
pos++; // single whitespace after maxval
|
||||
if (!width || !height || !maxval) throw new Error("bad PPM header");
|
||||
return { width, height, rgb: b.subarray(pos, pos + width * height * 3) };
|
||||
}
|
||||
|
||||
/// Draw a parsed PPM onto a (new or given) canvas. Returns the canvas.
|
||||
export function ppmToCanvas(buf, canvas) {
|
||||
const { width, height, rgb } = parsePpm(buf);
|
||||
const c = canvas || document.createElement("canvas");
|
||||
c.width = width;
|
||||
c.height = height;
|
||||
const ctx = c.getContext("2d");
|
||||
const img = ctx.createImageData(width, height);
|
||||
for (let i = 0, j = 0; i < width * height; i++, j += 3) {
|
||||
img.data[i * 4] = rgb[j];
|
||||
img.data[i * 4 + 1] = rgb[j + 1];
|
||||
img.data[i * 4 + 2] = rgb[j + 2];
|
||||
img.data[i * 4 + 3] = 255;
|
||||
}
|
||||
ctx.putImageData(img, 0, 0);
|
||||
return c;
|
||||
}
|
||||
|
||||
/// Rasterize an uploaded image file to WxH (stretched, as the engine does with
|
||||
/// the first frame) and return it as base64-encoded binary P6 PPM.
|
||||
export async function fileToPpmB64(file, width, height) {
|
||||
const bmp = await createImageBitmap(file);
|
||||
const c = document.createElement("canvas");
|
||||
c.width = width;
|
||||
c.height = height;
|
||||
const ctx = c.getContext("2d");
|
||||
ctx.drawImage(bmp, 0, 0, width, height);
|
||||
const img = ctx.getImageData(0, 0, width, height).data;
|
||||
const header = `P6\n${width} ${height}\n255\n`;
|
||||
const out = new Uint8Array(header.length + width * height * 3);
|
||||
for (let i = 0; i < header.length; i++) out[i] = header.charCodeAt(i);
|
||||
let p = header.length;
|
||||
for (let i = 0; i < width * height; i++) {
|
||||
out[p++] = img[i * 4];
|
||||
out[p++] = img[i * 4 + 1];
|
||||
out[p++] = img[i * 4 + 2];
|
||||
}
|
||||
// chunked btoa — the frame can be a few MB
|
||||
let bin = "";
|
||||
for (let i = 0; i < out.length; i += 0x8000)
|
||||
bin += String.fromCharCode.apply(null, out.subarray(i, i + 0x8000));
|
||||
return btoa(bin);
|
||||
}
|
||||
|
||||
/// Extract JPEG frames from an MJPEG AVI (RIFF): walk chunks from 'movi',
|
||||
/// collect '00dc'/'00dj' payloads as Blobs.
|
||||
export function aviJpegFrames(buf) {
|
||||
const b = new Uint8Array(buf);
|
||||
const dv = new DataView(buf);
|
||||
const fourcc = (o) => String.fromCharCode(b[o], b[o + 1], b[o + 2], b[o + 3]);
|
||||
if (fourcc(0) !== "RIFF" || fourcc(8) !== "AVI ") throw new Error("not an AVI");
|
||||
// find the 'movi' LIST
|
||||
let movi = -1;
|
||||
for (let o = 12; o + 12 <= b.length; ) {
|
||||
const id = fourcc(o);
|
||||
const size = dv.getUint32(o + 4, true);
|
||||
if (id === "LIST" && fourcc(o + 8) === "movi") {
|
||||
movi = o + 12;
|
||||
break;
|
||||
}
|
||||
o += 8 + size + (size & 1);
|
||||
}
|
||||
if (movi < 0) throw new Error("no movi list");
|
||||
const frames = [];
|
||||
for (let o = movi; o + 8 <= b.length; ) {
|
||||
const id = fourcc(o);
|
||||
const size = dv.getUint32(o + 4, true);
|
||||
if (id === "00dc" || id === "00dj") {
|
||||
if (size > 0) frames.push(new Blob([b.subarray(o + 8, o + 8 + size)], { type: "image/jpeg" }));
|
||||
} else if (id === "LIST") {
|
||||
o += 12; // descend (rec lists)
|
||||
continue;
|
||||
} else if (id === "idx1") {
|
||||
break;
|
||||
}
|
||||
o += 8 + size + (size & 1);
|
||||
}
|
||||
return frames;
|
||||
}
|
||||
|
||||
/// Canvas player for extracted MJPEG frames at a fixed fps, synced to an
|
||||
/// optional <audio> element (the .wav). Returns {canvas, play, pause, seek0, playing}.
|
||||
export function mjpegPlayer(frames, fps, audio) {
|
||||
const canvas = document.createElement("canvas");
|
||||
const ctx = canvas.getContext("2d");
|
||||
let bitmaps = null; // decoded lazily on first play
|
||||
let raf = null;
|
||||
let t0 = 0;
|
||||
let startFrame = 0;
|
||||
const state = { playing: false };
|
||||
|
||||
async function decodeAll() {
|
||||
if (bitmaps) return;
|
||||
bitmaps = await Promise.all(frames.map((f) => createImageBitmap(f)));
|
||||
if (bitmaps.length) {
|
||||
canvas.width = bitmaps[0].width;
|
||||
canvas.height = bitmaps[0].height;
|
||||
ctx.drawImage(bitmaps[0], 0, 0);
|
||||
}
|
||||
}
|
||||
function drawAt(i) {
|
||||
if (bitmaps && bitmaps[i]) ctx.drawImage(bitmaps[i], 0, 0);
|
||||
}
|
||||
function tick() {
|
||||
// audio clock when available (keeps A/V in sync), wall clock otherwise
|
||||
const t = audio && !audio.paused ? audio.currentTime : (performance.now() - t0) / 1000;
|
||||
const i = Math.min(frames.length - 1, startFrame + Math.floor(t * fps));
|
||||
drawAt(i);
|
||||
if (i >= frames.length - 1) {
|
||||
pause();
|
||||
return;
|
||||
}
|
||||
raf = requestAnimationFrame(tick);
|
||||
}
|
||||
async function play() {
|
||||
await decodeAll();
|
||||
state.playing = true;
|
||||
startFrame = 0;
|
||||
t0 = performance.now();
|
||||
if (audio) {
|
||||
audio.currentTime = 0;
|
||||
audio.play().catch(() => {});
|
||||
}
|
||||
raf = requestAnimationFrame(tick);
|
||||
}
|
||||
function pause() {
|
||||
state.playing = false;
|
||||
if (raf) cancelAnimationFrame(raf);
|
||||
raf = null;
|
||||
if (audio) audio.pause();
|
||||
}
|
||||
async function seek0() {
|
||||
pause();
|
||||
await decodeAll();
|
||||
drawAt(0);
|
||||
}
|
||||
return { canvas, play, pause, seek0, state, decodeAll };
|
||||
}
|
||||
Vendored
+406
-27
@@ -1,17 +1,64 @@
|
||||
// Playground — send a prompt through the live pipeline, inspect routing decision.
|
||||
import { h, mount, money, ms } from "../ui.js";
|
||||
// Playground — three tabs over the live gateway:
|
||||
// chat → the routing pipeline (with streaming, token counts, tok/s, TTFT)
|
||||
// image → `cortiq imagine` over a local text-to-image .cmf (Lumina)
|
||||
// video → `cortiq animate` over a local text-to-video .cmf (MiniMax-H3)
|
||||
// Media results are decoded in the browser (PPM → canvas, MJPEG AVI → canvas
|
||||
// player synced to the wav) — the same no-ffmpeg stance as the engine itself.
|
||||
import { h, mount, toast, money, ms } from "../ui.js";
|
||||
import { t } from "../i18n.js";
|
||||
import { api, testStream } from "../api.js";
|
||||
import { api, testStream, mediaFileUrl, mediaFileBytes } from "../api.js";
|
||||
import { appState } from "../app.js";
|
||||
import { ppmToCanvas, fileToPpmB64, aviJpegFrames, mjpegPlayer } from "../media-utils.js";
|
||||
|
||||
function fmtBytes(n) {
|
||||
if (!n && n !== 0) return "—";
|
||||
if (n >= 1e9) return (n / 1e9).toFixed(2) + " GB";
|
||||
if (n >= 1e6) return (n / 1e6).toFixed(1) + " MB";
|
||||
if (n >= 1e3) return Math.round(n / 1e3) + " KB";
|
||||
return n + " B";
|
||||
}
|
||||
function kv(k, v) {
|
||||
return h("div", { class: "kv" }, h("span", { class: "k" }, k), h("span", { class: "v" }, v));
|
||||
}
|
||||
function field(label, control, hint) {
|
||||
return h("label", { class: "field" }, h("span", {}, label), control, hint ? h("div", { class: "hint" }, hint) : null);
|
||||
}
|
||||
|
||||
export async function renderPlayground() {
|
||||
const meta = appState.meta || (await api.meta());
|
||||
const modelsData = await api.listModels();
|
||||
const ids = (modelsData.models || []).map((m) => m.id);
|
||||
let mediaModels = [];
|
||||
try {
|
||||
mediaModels = (await api.mediaModels()).models || [];
|
||||
} catch {
|
||||
/* media API unavailable — tabs will show the hint */
|
||||
}
|
||||
const imageModels = mediaModels.filter((m) => m.modality === "image");
|
||||
const videoModels = mediaModels.filter((m) => m.modality === "video");
|
||||
|
||||
let mode = "auto";
|
||||
const root = h("div");
|
||||
let tab = "chat";
|
||||
|
||||
// ── tabs ──
|
||||
const tabsSeg = h(
|
||||
"div",
|
||||
{ class: "seg" },
|
||||
h("button", { class: "active", "data-tab": "chat" }, "💬 " + t("pg.tab.chat")),
|
||||
h("button", { "data-tab": "image" }, "🖼 " + t("pg.tab.image")),
|
||||
h("button", { "data-tab": "video" }, "🎬 " + t("pg.tab.video"))
|
||||
);
|
||||
const panelHost = h("div", {});
|
||||
tabsSeg.querySelectorAll("button").forEach((b) =>
|
||||
b.addEventListener("click", () => {
|
||||
tab = b.dataset.tab;
|
||||
tabsSeg.querySelectorAll("button").forEach((x) => x.classList.toggle("active", x === b));
|
||||
mount(panelHost, panels[tab]);
|
||||
})
|
||||
);
|
||||
|
||||
// ════════════════════════════ chat panel ════════════════════════════
|
||||
let mode = "auto";
|
||||
const sysIn = h("textarea", { placeholder: t("pg.systemPlaceholder"), rows: 2 });
|
||||
const promptIn = h("textarea", { placeholder: t("pg.promptPlaceholder"), rows: 5 });
|
||||
const tempIn = h("input", { type: "number", step: "0.1", value: 0.7 });
|
||||
@@ -79,6 +126,7 @@ export async function renderPlayground() {
|
||||
out.textContent = acc;
|
||||
});
|
||||
out.textContent = acc || "—";
|
||||
const latency = Math.round(performance.now() - started);
|
||||
renderDecision({
|
||||
cortiq: {
|
||||
task_label: info.task_label,
|
||||
@@ -88,8 +136,10 @@ export async function renderPlayground() {
|
||||
cost_usd: info.cost_usd,
|
||||
failover: false,
|
||||
},
|
||||
usage: {},
|
||||
latency_ms: Math.round(performance.now() - started),
|
||||
usage: info.usage || {},
|
||||
usage_estimated: info.usage_estimated,
|
||||
latency_ms: latency,
|
||||
ttft_ms: info.ttft_ms,
|
||||
});
|
||||
} else {
|
||||
const r = await api.test(body);
|
||||
@@ -108,12 +158,17 @@ export async function renderPlayground() {
|
||||
sendBtn.textContent = t("pg.send");
|
||||
});
|
||||
|
||||
function kv(k, v) {
|
||||
return h("div", { class: "kv" }, h("span", { class: "k" }, k), h("span", { class: "v" }, v));
|
||||
}
|
||||
function renderDecision(r) {
|
||||
const c = r.cortiq || {};
|
||||
const score = c.complexity?.score ?? 0;
|
||||
const u = r.usage || {};
|
||||
const haveUsage = u.completion_tokens != null;
|
||||
// generation speed over the produce phase (after first token when known)
|
||||
let tokS = null;
|
||||
if (haveUsage && r.latency_ms) {
|
||||
const genMs = r.ttft_ms != null ? Math.max(1, r.latency_ms - r.ttft_ms) : r.latency_ms;
|
||||
tokS = (u.completion_tokens / (genMs / 1000)).toFixed(1);
|
||||
}
|
||||
mount(
|
||||
decision,
|
||||
h("div", { class: "card-head" }, h("h3", {}, t("pg.decision"))),
|
||||
@@ -127,17 +182,188 @@ export async function renderPlayground() {
|
||||
),
|
||||
kv(t("pg.selected"), h("span", { class: "mono" }, c.selected_model || "—")),
|
||||
kv(t("pg.source"), c.route_source || "—"),
|
||||
kv(t("pg.tokens"), `${r.usage?.prompt_tokens ?? 0}/${r.usage?.completion_tokens ?? 0}`),
|
||||
kv(t("pg.tokens"), haveUsage ? `${r.usage_estimated ? "≈ " : ""}${u.prompt_tokens ?? 0} / ${u.completion_tokens ?? 0}` : "—"),
|
||||
kv(t("pg.tokSpeed"), tokS != null ? tokS + " tok/s" : "—"),
|
||||
kv(t("pg.ttft"), r.ttft_ms != null ? ms(r.ttft_ms) : "—"),
|
||||
kv(t("pg.cost"), money(c.cost_usd)),
|
||||
kv(t("pg.latency"), ms(r.latency_ms)),
|
||||
kv(t("pg.failover"), c.failover ? "yes" : "no")
|
||||
);
|
||||
}
|
||||
|
||||
mount(
|
||||
root,
|
||||
h("div", { class: "page-head" }, h("h2", {}, t("pg.title")), h("p", {}, t("pg.subtitle"))),
|
||||
const chatPanel = h(
|
||||
"div",
|
||||
{ class: "pg" },
|
||||
h(
|
||||
"div",
|
||||
{},
|
||||
h(
|
||||
"div",
|
||||
{ class: "card" },
|
||||
field(t("pg.system"), sysIn),
|
||||
field(t("pg.prompt"), promptIn),
|
||||
field(t("pg.mode"), modeSeg),
|
||||
modeExtra,
|
||||
h("div", { class: "row" }, field(t("pg.temp"), tempIn), field(t("pg.maxTokens"), maxTokIn)),
|
||||
h(
|
||||
"div",
|
||||
{ class: "flex", style: "justify-content:space-between" },
|
||||
h("label", { class: "check", style: "margin:0" }, streamCb, t("pg.stream")),
|
||||
sendBtn
|
||||
)
|
||||
),
|
||||
h("div", { class: "card" }, h("div", { class: "card-head" }, h("h3", {}, t("pg.answer"))), out)
|
||||
),
|
||||
decision
|
||||
);
|
||||
|
||||
// ═══════════════════════ shared media machinery ═══════════════════════
|
||||
let mediaPoll = null;
|
||||
function stopMediaPoll() {
|
||||
if (mediaPoll) clearInterval(mediaPoll);
|
||||
mediaPoll = null;
|
||||
}
|
||||
// one generation at a time server-side → one active job client-side
|
||||
function watchJob(id, ui, onDone) {
|
||||
stopMediaPoll();
|
||||
const started = performance.now();
|
||||
mediaPoll = setInterval(async () => {
|
||||
if (!root.isConnected) return stopMediaPoll();
|
||||
let j;
|
||||
try {
|
||||
j = await api.mediaJob(id);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const secs = ((performance.now() - started) / 1000).toFixed(0);
|
||||
const pct = j.progress != null ? Math.round(j.progress * 100) : null;
|
||||
mount(
|
||||
ui.bar,
|
||||
h("div", { class: "pbar" },
|
||||
j.state === "running"
|
||||
? (pct != null
|
||||
? h("div", { class: "pbar-fill det", style: `width:${Math.max(pct, 1)}%` })
|
||||
: h("div", { class: "pbar-fill run" }))
|
||||
: h("div", { class: "pbar-fill " + (j.state === "done" ? "ok" : "err"), style: "width:100%" })
|
||||
),
|
||||
h("div", { class: "muted small" },
|
||||
(j.phase || j.state) + (pct != null && j.state === "running" ? ` · ${pct}%` : "") + ` · ${secs}s`),
|
||||
(j.log || []).length ? h("pre", { class: "conv-log", style: "max-height:90px" }, j.log.slice(-5).join("\n")) : null
|
||||
);
|
||||
if (j.state !== "running") {
|
||||
stopMediaPoll();
|
||||
ui.goBtn.disabled = false;
|
||||
ui.cancelBtn.style.display = "none";
|
||||
if (j.state === "done") onDone(j);
|
||||
else if (j.state === "error") toast(t("pg.genFailed"), "err");
|
||||
}
|
||||
}, 1000);
|
||||
}
|
||||
function mediaControls(goBtn, onCancelId) {
|
||||
const cancelBtn = h("button", { class: "btn ghost", style: "display:none" }, "✕ " + t("import.cancel"));
|
||||
cancelBtn.addEventListener("click", async () => {
|
||||
try {
|
||||
await api.mediaCancel(onCancelId.id);
|
||||
} catch (e) {
|
||||
toast(e.message, "err");
|
||||
}
|
||||
});
|
||||
return cancelBtn;
|
||||
}
|
||||
function noModelsHint(kindLabel) {
|
||||
return h(
|
||||
"div",
|
||||
{ class: "card" },
|
||||
h("div", { class: "callout" },
|
||||
t("pg.noMediaModels", { kind: kindLabel }),
|
||||
" ",
|
||||
h("a", { href: "#/import" }, t("settings.cmf.importLink") + " ↗"))
|
||||
);
|
||||
}
|
||||
function dice(seedIn) {
|
||||
const b = h("button", { class: "btn ghost sm", title: t("pg.randomSeed") }, "🎲");
|
||||
b.addEventListener("click", () => (seedIn.value = Math.floor(Math.random() * 1e9)));
|
||||
return b;
|
||||
}
|
||||
|
||||
// ════════════════════════════ image panel ════════════════════════════
|
||||
let imagePanel;
|
||||
if (!imageModels.length) {
|
||||
imagePanel = noModelsHint(t("pg.tab.image"));
|
||||
} else {
|
||||
const imModel = h("select", {}, ...imageModels.map((m) => h("option", { value: m.name }, `${m.name} · ${fmtBytes(m.size_bytes)}`)));
|
||||
const imPrompt = h("textarea", { placeholder: t("pg.imagePromptPlaceholder"), rows: 3 });
|
||||
const imW = h("select", {}, ...[256, 384, 512, 640, 768, 1024].map((v) => h("option", { value: v, selected: v === 512 ? true : null }, v)));
|
||||
const imH = h("select", {}, ...[256, 384, 512, 640, 768, 1024].map((v) => h("option", { value: v, selected: v === 512 ? true : null }, v)));
|
||||
const imSteps = h("input", { type: "number", min: 1, max: 100, value: 30 });
|
||||
const imCfg = h("input", { type: "number", step: "0.5", value: 4 });
|
||||
const imSeed = h("input", { type: "number", value: 42 });
|
||||
const imBar = h("div", {});
|
||||
const imOut = h("div", { class: "media-out" }, h("div", { class: "empty" }, t("pg.imageEmpty")));
|
||||
const imMetrics = h("div", { class: "card routecard" }, h("div", { class: "card-head" }, h("h3", {}, t("pg.genMetrics"))), h("div", { class: "empty" }, "—"));
|
||||
const imGo = h("button", { class: "btn primary" }, "🖼 " + t("pg.generate"));
|
||||
const imCancelRef = { id: null };
|
||||
const imCancel = mediaControls(imGo, imCancelRef);
|
||||
|
||||
imGo.addEventListener("click", async () => {
|
||||
const prompt = imPrompt.value.trim();
|
||||
if (!prompt) return;
|
||||
imGo.disabled = true;
|
||||
imCancel.style.display = "";
|
||||
mount(imOut, h("div", { class: "flex" }, h("span", { class: "spinner" }), " ", t("pg.generating")));
|
||||
const params = {
|
||||
kind: "image",
|
||||
model: imModel.value,
|
||||
prompt,
|
||||
width: +imW.value,
|
||||
height: +imH.value,
|
||||
steps: +imSteps.value || 30,
|
||||
cfg: parseFloat(imCfg.value) || 4,
|
||||
seed: +imSeed.value || 42,
|
||||
};
|
||||
try {
|
||||
const r = await api.mediaGenerate(params);
|
||||
imCancelRef.id = r.job;
|
||||
const t0 = performance.now();
|
||||
watchJob(r.job, { bar: imBar, goBtn: imGo, cancelBtn: imCancel }, async (j) => {
|
||||
mount(imBar);
|
||||
const secs = (performance.now() - t0) / 1000;
|
||||
try {
|
||||
const buf = await mediaFileBytes(j.id, "image");
|
||||
const canvas = ppmToCanvas(buf);
|
||||
canvas.className = "media-canvas";
|
||||
const dl = h("button", { class: "btn sm" }, "⬇ PNG");
|
||||
dl.addEventListener("click", () =>
|
||||
canvas.toBlob((b) => {
|
||||
const a = h("a", { href: URL.createObjectURL(b), download: `cortiq-${j.id}.png` });
|
||||
a.click();
|
||||
}, "image/png")
|
||||
);
|
||||
mount(imOut, canvas, h("div", { class: "actions" }, dl));
|
||||
mount(
|
||||
imMetrics,
|
||||
h("div", { class: "card-head" }, h("h3", {}, t("pg.genMetrics"))),
|
||||
kv(t("pg.model"), h("span", { class: "mono" }, j.model)),
|
||||
kv(t("pg.resolution"), `${j.width}×${j.height}`),
|
||||
kv(t("pg.steps"), String(j.steps)),
|
||||
kv("CFG", String(params.cfg)),
|
||||
kv(t("pg.seed"), String(j.seed)),
|
||||
kv(t("pg.genTime"), secs.toFixed(1) + " s"),
|
||||
kv(t("pg.timePerStep"), (secs / j.steps).toFixed(2) + " s"),
|
||||
kv(t("pg.fileSize"), fmtBytes(j.out_bytes))
|
||||
);
|
||||
} catch (e) {
|
||||
mount(imOut, h("div", { class: "callout" }, String(e.message || e)));
|
||||
}
|
||||
});
|
||||
} catch (e) {
|
||||
imGo.disabled = false;
|
||||
imCancel.style.display = "none";
|
||||
mount(imOut, h("div", { class: "callout" }, String(e.message || e)));
|
||||
}
|
||||
});
|
||||
|
||||
imagePanel = h(
|
||||
"div",
|
||||
{ class: "pg" },
|
||||
h(
|
||||
@@ -146,22 +372,175 @@ export async function renderPlayground() {
|
||||
h(
|
||||
"div",
|
||||
{ class: "card" },
|
||||
h("label", { class: "field" }, h("span", {}, t("pg.system")), sysIn),
|
||||
h("label", { class: "field" }, h("span", {}, t("pg.prompt")), promptIn),
|
||||
h("label", { class: "field" }, h("span", {}, t("pg.mode")), modeSeg),
|
||||
modeExtra,
|
||||
h("div", { class: "row" }, h("label", { class: "field" }, h("span", {}, t("pg.temp")), tempIn), h("label", { class: "field" }, h("span", {}, t("pg.maxTokens")), maxTokIn)),
|
||||
h(
|
||||
"div",
|
||||
{ class: "flex", style: "justify-content:space-between" },
|
||||
h("label", { class: "check", style: "margin:0" }, streamCb, t("pg.stream")),
|
||||
sendBtn
|
||||
)
|
||||
field(t("pg.model"), imModel),
|
||||
field(t("pg.prompt"), imPrompt),
|
||||
h("div", { class: "row" }, field(t("pg.width"), imW), field(t("pg.height"), imH)),
|
||||
h("div", { class: "row" },
|
||||
field(t("pg.steps"), imSteps),
|
||||
field("CFG", imCfg, t("pg.cfgHint"))),
|
||||
h("div", { class: "row" },
|
||||
field(t("pg.seed"), h("div", { class: "flex", style: "gap:6px" }, imSeed, dice(imSeed)))),
|
||||
h("div", { class: "actions" }, imCancel, imGo),
|
||||
imBar
|
||||
),
|
||||
h("div", { class: "card" }, h("div", { class: "card-head" }, h("h3", {}, t("pg.answer"))), out)
|
||||
h("div", { class: "card" }, h("div", { class: "card-head" }, h("h3", {}, t("pg.result"))), imOut)
|
||||
),
|
||||
decision
|
||||
)
|
||||
imMetrics
|
||||
);
|
||||
}
|
||||
|
||||
// ════════════════════════════ video panel ════════════════════════════
|
||||
let videoPanel;
|
||||
if (!videoModels.length) {
|
||||
videoPanel = noModelsHint(t("pg.tab.video"));
|
||||
} else {
|
||||
const vModel = h("select", {}, ...videoModels.map((m) => h("option", { value: m.name }, `${m.name} · ${fmtBytes(m.size_bytes)}`)));
|
||||
const vPrompt = h("textarea", { placeholder: t("pg.videoPromptPlaceholder"), rows: 3 });
|
||||
const vW = h("input", { type: "number", step: 32, value: 512 });
|
||||
const vH = h("input", { type: "number", step: 32, value: 288 });
|
||||
const vFrames = h("input", { type: "number", value: 39 });
|
||||
const vSteps = h("input", { type: "number", value: 4 });
|
||||
const vSeed = h("input", { type: "number", value: 42 });
|
||||
const vQuality = h("input", { type: "number", min: 10, max: 100, value: 92 });
|
||||
const vFirst = h("input", { type: "file", accept: "image/*" });
|
||||
const vLast = h("input", { type: "file", accept: "image/*" });
|
||||
const vBar = h("div", {});
|
||||
const vOut = h("div", { class: "media-out" }, h("div", { class: "empty" }, t("pg.videoEmpty")));
|
||||
const vMetrics = h("div", { class: "card routecard" }, h("div", { class: "card-head" }, h("h3", {}, t("pg.genMetrics"))), h("div", { class: "empty" }, "—"));
|
||||
const vGo = h("button", { class: "btn primary" }, "🎬 " + t("pg.generate"));
|
||||
const vCancelRef = { id: null };
|
||||
const vCancel = mediaControls(vGo, vCancelRef);
|
||||
|
||||
vGo.addEventListener("click", async () => {
|
||||
const prompt = vPrompt.value.trim();
|
||||
if (!prompt) return;
|
||||
vGo.disabled = true;
|
||||
vCancel.style.display = "";
|
||||
mount(vOut, h("div", { class: "flex" }, h("span", { class: "spinner" }), " ", t("pg.generating")));
|
||||
const params = {
|
||||
kind: "video",
|
||||
model: vModel.value,
|
||||
prompt,
|
||||
width: +vW.value || 512,
|
||||
height: +vH.value || 288,
|
||||
frames: +vFrames.value || 39,
|
||||
steps: +vSteps.value || 4,
|
||||
seed: +vSeed.value || 42,
|
||||
quality: +vQuality.value || 92,
|
||||
};
|
||||
try {
|
||||
if (vFirst.files[0]) params.first_frame_b64 = await fileToPpmB64(vFirst.files[0], params.width, params.height);
|
||||
if (vLast.files[0]) params.last_frame_b64 = await fileToPpmB64(vLast.files[0], params.width, params.height);
|
||||
const r = await api.mediaGenerate(params);
|
||||
vCancelRef.id = r.job;
|
||||
const t0 = performance.now();
|
||||
watchJob(r.job, { bar: vBar, goBtn: vGo, cancelBtn: vCancel }, async (j) => {
|
||||
mount(vBar);
|
||||
const secs = (performance.now() - t0) / 1000;
|
||||
try {
|
||||
await showVideoResult(j, vOut);
|
||||
const durS = (j.frames || 0) / 24;
|
||||
mount(
|
||||
vMetrics,
|
||||
h("div", { class: "card-head" }, h("h3", {}, t("pg.genMetrics"))),
|
||||
kv(t("pg.model"), h("span", { class: "mono" }, j.model)),
|
||||
kv(t("pg.resolution"), `${j.width}×${j.height}`),
|
||||
kv(t("pg.frames"), `${j.frames} @ 24 fps`),
|
||||
kv(t("pg.duration"), durS.toFixed(1) + " s"),
|
||||
kv(t("pg.steps"), String(j.steps)),
|
||||
kv(t("pg.seed"), String(j.seed)),
|
||||
kv(t("pg.genTime"), secs.toFixed(1) + " s"),
|
||||
kv(t("pg.timePerFrame"), j.frames ? (secs / j.frames).toFixed(2) + " s" : "—"),
|
||||
kv(t("pg.fileSize"), fmtBytes(j.out_bytes))
|
||||
);
|
||||
} catch (e) {
|
||||
mount(vOut, h("div", { class: "callout" }, String(e.message || e)));
|
||||
}
|
||||
});
|
||||
} catch (e) {
|
||||
vGo.disabled = false;
|
||||
vCancel.style.display = "none";
|
||||
mount(vOut, h("div", { class: "callout" }, String(e.message || e)));
|
||||
}
|
||||
});
|
||||
|
||||
async function showVideoResult(j, host) {
|
||||
const downloads = h("div", { class: "actions" });
|
||||
for (const kind of j.outputs || []) {
|
||||
const label = { video: "⬇ AVI", audio: "⬇ WAV", mp4: "⬇ MP4" }[kind];
|
||||
if (!label) continue;
|
||||
const b = h("button", { class: "btn sm" }, label);
|
||||
b.addEventListener("click", async () => {
|
||||
const { url } = await mediaFileUrl(j.id, kind);
|
||||
const ext = kind === "video" ? "avi" : kind === "audio" ? "wav" : "mp4";
|
||||
h("a", { href: url, download: `cortiq-${j.id}.${ext}` }).click();
|
||||
});
|
||||
downloads.appendChild(b);
|
||||
}
|
||||
if ((j.outputs || []).includes("mp4")) {
|
||||
// ffmpeg was present server-side → native playback
|
||||
const { url } = await mediaFileUrl(j.id, "mp4");
|
||||
mount(host, h("video", { src: url, controls: true, class: "media-canvas" }), downloads);
|
||||
return;
|
||||
}
|
||||
// no ffmpeg anywhere: decode the MJPEG AVI ourselves, sync to the wav
|
||||
const buf = await mediaFileBytes(j.id, "video");
|
||||
const frames = aviJpegFrames(buf);
|
||||
let audio = null;
|
||||
if ((j.outputs || []).includes("audio")) {
|
||||
const { url } = await mediaFileUrl(j.id, "audio");
|
||||
audio = h("audio", { src: url });
|
||||
}
|
||||
const player = mjpegPlayer(frames, 24, audio);
|
||||
player.canvas.className = "media-canvas";
|
||||
await player.seek0();
|
||||
const playBtn = h("button", { class: "btn primary sm" }, "▶ " + t("pg.play"));
|
||||
playBtn.addEventListener("click", async () => {
|
||||
if (player.state.playing) {
|
||||
player.pause();
|
||||
playBtn.textContent = "▶ " + t("pg.play");
|
||||
} else {
|
||||
playBtn.textContent = "⏸";
|
||||
await player.play();
|
||||
}
|
||||
});
|
||||
mount(host, player.canvas, h("div", { class: "actions" }, playBtn, ...downloads.childNodes ? [downloads] : []));
|
||||
}
|
||||
|
||||
videoPanel = h(
|
||||
"div",
|
||||
{ class: "pg" },
|
||||
h(
|
||||
"div",
|
||||
{},
|
||||
h(
|
||||
"div",
|
||||
{ class: "card" },
|
||||
field(t("pg.model"), vModel),
|
||||
field(t("pg.prompt"), vPrompt),
|
||||
h("div", { class: "row" }, field(t("pg.width"), vW, t("pg.videoSizeHint")), field(t("pg.height"), vH)),
|
||||
h("div", { class: "row" }, field(t("pg.frames"), vFrames, t("pg.framesHint")), field(t("pg.steps"), vSteps, t("pg.videoStepsHint"))),
|
||||
h("div", { class: "row" },
|
||||
field(t("pg.seed"), h("div", { class: "flex", style: "gap:6px" }, vSeed, dice(vSeed))),
|
||||
field(t("pg.quality"), vQuality)),
|
||||
h("div", { class: "row" }, field(t("pg.firstFrame"), vFirst, t("pg.keyframeHint")), field(t("pg.lastFrame"), vLast)),
|
||||
h("div", { class: "actions" }, vCancel, vGo),
|
||||
vBar
|
||||
),
|
||||
h("div", { class: "card" }, h("div", { class: "card-head" }, h("h3", {}, t("pg.result"))), vOut)
|
||||
),
|
||||
vMetrics
|
||||
);
|
||||
}
|
||||
|
||||
const panels = { chat: chatPanel, image: imagePanel, video: videoPanel };
|
||||
|
||||
mount(
|
||||
root,
|
||||
h("div", { class: "page-head" }, h("h2", {}, t("pg.title")), h("p", {}, t("pg.subtitle"))),
|
||||
tabsSeg,
|
||||
panelHost
|
||||
);
|
||||
mount(panelHost, panels.chat);
|
||||
return root;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user