// Benchmark — two modes over the live pipeline: // 🧭 Smart routing (default): the product's core promise under test — every // probe goes through cortiq-auto and the router's task/tier decision and // chosen model are judged alongside the answer itself. // 🎯 Individual models: the same suite pinned to selected models — compare // quality and speed head-to-head. // A redesign of the old standalone benchmark.html: same scoring ideas // (keyword checks with | synonyms, think-tag stripping, anti-loop detection, // pass/partial/fail), now embedded, localized and CDN-free. import { h, mount, toast, ms, money, moneyEl } from "../ui.js"; import { t } from "../i18n.js"; import { api, testStream } from "../api.js"; // ── built-in suite ────────────────────────────────────────────────────────── // Checks are language-neutral where possible (digits, identifiers, emails); // `check` entries may hold |-separated synonyms — one hit per entry counts. // `quick: true` marks the fast subset. // `label`: router task labels accepted as a correct classification (null = not // judged — the prompt has no natural label in the taxonomy). const SUITE = [ { id: "math-mult", cat: "math", label: ["math"], quick: true, max: 96, prompt: "What is 17 * 23? Answer with the number only.", check: ["391"] }, { id: "math-eq", cat: "math", label: ["math"], max: 128, prompt: "Solve 2x + 6 = 14. Answer with the value of x only.", check: ["4"] }, { id: "logic-syll", cat: "logic", label: ["qa", "math"], max: 96, prompt: "If all bloops are razzies and all razzies are lazzies, are all bloops lazzies? Answer yes or no.", check: ["yes|да|oui|ja|sí|si|evet|是"] }, { id: "count-letters", cat: "logic", label: ["qa", "math"], quick: true, max: 64, prompt: "How many letters are in the word 'cat'? Answer with a digit.", check: ["3"] }, { id: "apples", cat: "logic", label: ["math", "qa"], quick: true, max: 96, prompt: "John has 3 apples, buys 2 more, then eats 1. How many apples does he have? Answer with a digit.", check: ["4"] }, { id: "code-fact", cat: "code", label: ["code"], max: 256, prompt: "Write a Python function factorial(n) that returns n!.", check: ["def", "factorial", "return"] }, { id: "qa-capital", cat: "qa", label: ["qa"], quick: true, max: 48, prompt: "What is the capital of France? One word.", check: ["paris|париж"] }, { id: "primes", cat: "qa", label: ["math", "qa"], max: 96, prompt: "List the first five prime numbers, comma-separated.", check: ["2", "3", "5", "7", "11"] }, { id: "instr-ok", cat: "instruction", label: null, quick: true, max: 32, prompt: "Reply with exactly the word OK and nothing else.", check: ["ok"] }, { id: "extract-email", cat: "extraction", label: ["extraction"], quick: true, max: 64, prompt: "Extract the email address from: 'Contact us at info@example.com today.' Reply with the address only.", check: ["info@example.com"] }, { id: "translate", cat: "translation", label: ["translation"], max: 64, prompt: "Translate to English: «Доброе утро». Reply with the translation only.", check: ["good morning"] }, { id: "format-json", cat: "instruction", label: null, max: 96, prompt: 'Return a JSON object with a single key "n" whose value is 7. JSON only.', check: ['"n"', "7"] }, { id: "summarize", cat: "summarization", label: ["summarization"], max: 96, prompt: "Summarize in one short sentence: 'The cat sat on the warm mat by the window. After a while it curled up and fell asleep in the sun.'", check: ["cat|кот|кошк"] }, { id: "creative", cat: "creative-writing", label: ["creative-writing"], max: 512, prompt: "Write a two-line poem about the sea.", check: ["sea|море|ocean"] }, { id: "chitchat", cat: "chitchat", label: ["chitchat"], quick: true, max: 48, prompt: "Hi! How are you today?", check: [] }, // hard probes: dense enough that the router's complexity model rates them // high tier — this is where tier-based routing actually earns its keep. // Each run picks a random variant: the router treats familiar prompts as // less novel (by design), so a fixed wording would decay to medium/low. { id: "hard-compiler", cat: "hard", label: ["code", "qa"], judge: "route", max: 8, check: [], variants: [ "Design an SSA-based optimizing compiler backend for an exposed-pipeline VLIW media processor: formulate instruction scheduling as an integer linear program with resource and hazard constraints, apply modulo scheduling to software-pipelined inner loops using rotating register files, do spill-aware register allocation via graph coloring with live-range splitting, add a translation-validated peephole pass, and explain how to prove the scheduler can never violate a structural hazard.", "Build the code-generation backend of a JIT for GPU warp-specialized kernels: cost-model-driven instruction selection across divergent branches, software pipelining of memory latency with double-buffered shared memory, register allocation under occupancy pressure with rematerialization, a verified peephole pass over PTX, and a proof sketch that no schedule can deadlock the warp scheduler.", "Construct an optimizing backend for a RISC-V vector coprocessor with configurable lane widths: vector-length-agnostic instruction selection as cost-model search, an integer-linear-program schedule around a chained load-store unit's structural hazards, VL-dependent spill-cost register allocation with live-range splitting, and a translation-validated peephole pass, plus the argument that the scheduler never issues into an occupied pipeline slot.", ] }, { id: "hard-protocol", cat: "hard", label: ["qa", "code"], judge: "route", max: 8, check: [], variants: [ "Design and security-review an end-to-end encrypted coordination protocol for a federated fleet of 100000 IoT devices: lightweight key agreement under 32KB RAM budgets, per-group sender-key trees with post-compromise security, secure firmware-update attestation chained to a transparency log, metadata minimization against a network-level adversary, and revocation of physically captured devices without a round-trip to every peer. Analyze the trade-offs precisely, then specify the exact message flow for rotating a compromised region's keys.", "Design an end-to-end encrypted messaging system for a disaster-relief mesh network that partitions for days: store-and-forward double ratchet with bounded state, group membership changes that commute across partitions, post-compromise security once connectivity returns, spam and sybil resistance without a central server, and forensic-resistant local storage. Weigh the trade-offs, then give the exact key schedule for merging two long-partitioned group histories.", "Architect the security layer of a multi-tenant confidential-computing platform: remote attestation chained through TEE quotes to a transparency log, per-tenant key hierarchies with crypto-agile rotation, sealed storage that survives enclave upgrades, side-channel hardening policy for co-resident tenants, and revocation that cannot be rolled back by a malicious host. Analyze each trade-off, then specify the exact handshake for migrating a sealed workload between hosts.", ] }, ]; const LS_CUSTOM = "allaigate_bench_custom"; const LS_HISTORY = "allaigate_bench_history"; function customTests() { try { return JSON.parse(localStorage.getItem(LS_CUSTOM) || "[]"); } catch { return []; } } function saveCustom(list) { localStorage.setItem(LS_CUSTOM, JSON.stringify(list)); } function history() { try { return JSON.parse(localStorage.getItem(LS_HISTORY) || "[]"); } catch { return []; } } function pushHistory(run) { const l = history(); l.unshift(run); localStorage.setItem(LS_HISTORY, JSON.stringify(l.slice(0, 12))); } function saveHistory(l) { localStorage.setItem(LS_HISTORY, JSON.stringify(l)); } // ── evaluation (ported from the old benchmark, trimmed) ───────────────────── function stripThink(text) { return text.replace(/<(?:\|)?(?:think|thought)(?:\|)?>[\s\S]*?(?:<\/(?:\|)?(?:think|thought)(?:\|)?>|$)/gi, "").trim(); } // tail loop: the same 12+ char chunk repeated 3+ times back to back function looksLooped(text) { const tail = text.slice(-360); for (let len = 12; len <= 60; len += 8) { const probe = tail.slice(-len); if (probe && tail.split(probe).length - 1 >= 3) return true; } return false; } function evaluate(raw, test) { let text = stripThink(raw); if (!text && raw) text = raw; // max_tokens cut inside — judge the raw const resp = text.toLowerCase(); if (!resp.trim()) return { status: "fail", score: 0, note: t("bm.emptyAnswer") }; if (looksLooped(resp)) return { status: "fail", score: 0, note: t("bm.loopDetected") }; if (!test.check.length) return { status: "pass", score: 1, note: "✓" }; let hit = 0; for (const cond of test.check) { const opts = cond.toLowerCase().split("|").map((s) => s.trim()); if (opts.some((o) => o && resp.includes(o))) hit++; } const frac = hit / test.check.length; const status = frac >= 1 ? "pass" : frac >= 0.5 ? "partial" : "fail"; return { status, score: frac, note: `${hit}/${test.check.length}` }; } export async function renderBenchmark() { const modelsData = await api.listModels(); const ids = (modelsData.models || []).map((m) => m.id); const root = h("div"); let running = false; let stopFlag = false; let mode = "routing"; // routing (cortiq-auto, the product's feature) | models const modelCbs = ids.map((id) => ({ id, cb: h("input", { type: "checkbox", checked: ids.length === 1 ? true : null }) })); const quickCb = h("input", { type: "checkbox", checked: true }); const resultsHost = h("div", {}); const progressHost = h("div", {}); const historyHost = h("div", {}); const modeDesc = h("p", { class: "muted", style: "margin:8px 0 0" }, t("bm.mode.routingDesc")); const modelsPick = h("div", { class: "flex wrap", style: "gap:10px; margin-top:12px; display:none" }, ...modelCbs.map(({ id, cb }) => h("label", { class: "check" }, cb, h("span", { class: "mono" }, id)))); const modeSeg = h("div", { class: "seg" }, h("button", { class: "active" }, "🧭 " + t("bm.mode.routing")), h("button", {}, "🎯 " + t("bm.mode.models"))); modeSeg.querySelectorAll("button").forEach((b, i) => b.addEventListener("click", () => { mode = i === 0 ? "routing" : "models"; modeSeg.querySelectorAll("button").forEach((x, j) => x.classList.toggle("active", j === i)); modelsPick.style.display = mode === "models" ? "" : "none"; modeDesc.textContent = t(mode === "routing" ? "bm.mode.routingDesc" : "bm.mode.modelsDesc"); })); function suite() { const base = SUITE.concat(customTests()); return quickCb.checked ? base.filter((x) => x.quick || x.custom) : base; } const runBtn = h("button", { class: "btn primary" }, "▶ " + t("bm.run")); const stopBtn = h("button", { class: "btn ghost", style: "display:none" }, "✕ " + t("import.cancel")); stopBtn.addEventListener("click", () => { stopFlag = true; stopBtn.disabled = true; }); runBtn.addEventListener("click", async () => { const models = mode === "routing" ? ["cortiq-auto"] : modelCbs.filter((m) => m.cb.checked).map((m) => m.id); if (!models.length) { toast(t("bm.pickModel"), "warn"); return; } running = true; stopFlag = false; runBtn.disabled = true; stopBtn.style.display = ""; stopBtn.disabled = false; const tests = suite().filter((x) => mode === "routing" || x.judge !== "route"); const run = { ts: Date.now(), mode, quick: quickCb.checked, models: {}, nTests: tests.length }; for (const model of models) { const rec = { results: [], score: 0, tokS: [], ttft: [], failed: false }; run.models[model] = rec; for (let i = 0; i < tests.length; i++) { if (stopFlag) break; const test = tests[i]; const prompt = test.variants ? test.variants[(Math.random() * test.variants.length) | 0] : test.prompt; mount( progressHost, h("div", { class: "card" }, h("div", { class: "flex" }, h("span", { class: "spinner" }), " ", h("b", { class: "mono" }, mode === "routing" ? t("bm.mode.routing") : model), " · ", t("bm.testOf", { i: i + 1, n: tests.length }), " · ", h("span", { class: "muted" }, test.id)), h("div", { class: "muted small", style: "margin-top:6px; font-style:italic" }, "«" + prompt.slice(0, 110) + (prompt.length > 110 ? "…»" : "»")), h("div", { class: "pbar", style: "margin-top:8px" }, h("div", { class: "pbar-fill det", style: `width:${Math.round((i / tests.length) * 100)}%` }))) ); const started = performance.now(); let text = ""; try { const info = await testStream( // +256: headroom for reasoning models — a tight cap cuts them // mid- and turns a correct model into a false ❌ { model, messages: [{ role: "user", content: prompt }], temperature: 0, max_tokens: (test.max || 128) + 256 }, (d) => (text += d) ); const latency = performance.now() - started; let ev; if (test.judge === "route") { // the test asserts the ROUTER's decision: did it rate this high? ev = info.tier === "high" ? { status: "pass", score: 1, note: "high ✓" } : { status: "fail", score: 0, note: (info.tier || "—") + " ≠ high" }; } else { ev = evaluate(text, test); } const compl = info.usage?.completion_tokens; if (test.judge !== "route" && ev.status === "fail" && !stripThink(text).trim() && compl >= (test.max || 128) + 250) { ev.note = t("bm.cutByCap"); // the whole budget went into } if (compl && latency > 0) { const gen = info.ttft_ms != null ? Math.max(1, latency - info.ttft_ms) : latency; rec.tokS.push(compl / (gen / 1000)); } if (info.ttft_ms != null) rec.ttft.push(info.ttft_ms); const row = { id: test.id, cat: test.cat, ...ev, latency: Math.round(latency), cost: info.cost_usd || 0, answer: stripThink(text).slice(0, 300) }; if (mode === "routing") { // router timed out / fell back — there was no classification to // judge, so it must not count against accuracy row.degraded = !info.task_label || info.task_label === "degraded" || String(info.route_source || "").includes("degraded"); row.task = info.task_label || "—"; row.tier = info.tier || "—"; row.chosen = info.selected_model || "—"; row.source = info.route_source || "—"; row.expected = test.label; // null = classification not judged row.taskOk = test.label && !row.degraded ? test.label.includes(info.task_label) : null; } rec.results.push(row); } catch (e) { rec.results.push({ id: test.id, cat: test.cat, status: "fail", score: 0, note: String(e.message || e).slice(0, 80), latency: 0, answer: "" }); rec.failed = true; } renderResults(run, true); // live: строка появляется сразу после теста } if (stopFlag) break; } mount(progressHost); running = false; runBtn.disabled = false; stopBtn.style.display = "none"; if (!stopFlag) { pushHistory(run); // финальный аккорд: короткий вердикт с процентами и деньгами const first = run.models[Object.keys(run.models)[0]]; if (first) { const st = stats(first); const judged = first.results.filter((r) => r.taskOk != null); const cls = judged.length ? ` · ${judged.filter((r) => r.taskOk).length}/${judged.length}` : ""; const total = Object.values(run.models).reduce((a, rec) => a + stats(rec).cost, 0); toast(t("bm.verdict", { p: Math.round(st.score * 100) + "%" + cls, s: money(total) }), "ok", 6000); } } renderResults(run); renderHistory(); }); function statusIcon(s) { return s === "pass" ? "✅" : s === "partial" ? "◐" : "❌"; } function stats(rec) { const n = rec.results.length || 1; return { score: rec.results.reduce((a, r) => a + r.score, 0) / n, tokS: rec.tokS.length ? rec.tokS.reduce((a, b) => a + b, 0) / rec.tokS.length : null, ttft: rec.ttft.length ? rec.ttft.reduce((a, b) => a + b, 0) / rec.ttft.length : null, cost: rec.results.reduce((a, r) => a + (r.cost || 0), 0), }; } function renderResults(run, live) { const models = Object.keys(run.models); if (!models.length) { mount(resultsHost); return; } const routing = run.mode === "routing"; const detail = h("div", {}); let summary; if (routing) { const rec = run.models[models[0]]; const st = stats(rec); const judged = rec.results.filter((r) => r.taskOk != null); const okCount = judged.filter((r) => r.taskOk).length; // какие модели реально выбирал роутер const dist = {}; for (const r of rec.results) { if (!r.chosen) continue; dist[r.chosen] = dist[r.chosen] || { n: 0, cost: 0 }; dist[r.chosen].n++; dist[r.chosen].cost += r.cost || 0; } const pct = Math.round(st.score * 100); summary = h("div", {}, h("div", { class: "kpis", style: "grid-template-columns: repeat(auto-fit, minmax(150px, 1fr))" }, h("div", { class: "kpi" }, h("div", { class: "big" }, pct + "%"), h("div", { class: "lbl" }, t("bm.answersOk"))), h("div", { class: "kpi" }, h("div", { class: "big" }, judged.length ? `${okCount}/${judged.length}` : "—"), h("div", { class: "lbl" }, t("bm.classAcc"))), h("div", { class: "kpi" }, h("div", { class: "big" }, st.ttft != null ? ms(Math.round(st.ttft)) : "—"), h("div", { class: "lbl" }, t("pg.ttft"))), h("div", { class: "kpi" }, h("div", { class: "big" }, st.tokS != null ? st.tokS.toFixed(1) : "—"), h("div", { class: "lbl" }, "tok/s")), h("div", { class: "kpi" }, h("div", { class: "big" }, moneyEl(st.cost)), h("div", { class: "lbl" }, t("bm.spent")))), h("div", { class: "flex wrap", style: "gap:6px; margin-top:4px" }, h("span", { class: "muted small" }, t("bm.modelDist") + ":"), ...Object.entries(dist).map(([m, d]) => h("span", { class: "badge" }, `${m} × ${d.n}`, d.cost > 0 ? " · " : "", d.cost > 0 ? moneyEl(d.cost) : "")))); } else { const sorted = models.slice().sort((a, b) => stats(run.models[b]).score - stats(run.models[a]).score); summary = h("div", {}, ...sorted.map((m) => { const st = stats(run.models[m]); const pct = Math.round(st.score * 100); return h("div", { class: "bm-sum" }, h("div", { class: "flex" }, h("b", { class: "mono grow" }, m), h("span", { class: "badge " + (pct >= 80 ? "ok" : pct >= 50 ? "" : "bad") }, pct + "%"), st.tokS != null ? h("span", { class: "muted small", style: "margin-left:8px" }, st.tokS.toFixed(1) + " tok/s") : null, st.ttft != null ? h("span", { class: "muted small", style: "margin-left:8px" }, "TTFT " + ms(Math.round(st.ttft))) : null, st.cost > 0 ? h("span", { class: "muted small", style: "margin-left:8px" }, t("bm.spent") + " ", moneyEl(st.cost)) : null), h("div", { class: "sysbar" }, h("i", { class: pct < 50 ? "bad" : "", style: `width:${pct}%` }))); })); } function answerCell(r, m) { const cell = h("td", { class: "bm-cell", title: r.answer || r.note }, statusIcon(r.status) + " ", h("span", { class: "muted small" }, r.note + (r.latency ? ` · ${(r.latency / 1000).toFixed(1)}s` : ""))); cell.addEventListener("click", () => { mount(detail, h("div", { class: "card" }, h("div", { class: "card-title" }, `${r.id} · ${m} · ${statusIcon(r.status)}`), h("pre", { class: "conv-log", style: "max-height:200px" }, r.answer || "—"))); }); return cell; } let header, rows; if (routing) { const rec = run.models[models[0]]; header = h("tr", {}, h("th", {}, t("bm.test")), h("th", {}, t("pg.task")), h("th", {}, t("pg.tier")), h("th", {}, t("pg.model")), h("th", {}, t("bm.answer"))); rows = rec.results.map((r) => h("tr", {}, h("td", {}, h("div", {}, r.id), h("div", { class: "muted small" }, r.cat)), h("td", {}, r.degraded ? h("span", { class: "muted", title: t("bm.routerDegraded") }, "⚠ " + t("bm.degraded")) : r.taskOk == null ? h("span", { class: "muted" }, r.task || "—") : h("span", { class: r.taskOk ? "" : "err" }, (r.taskOk ? "✓ " : "✗ ") + r.task, r.taskOk ? "" : h("div", { class: "muted small" }, t("bm.expectedLbl") + " " + (r.expected || []).join("/")))), h("td", {}, r.tier ? h("span", { class: "badge tier-" + r.tier }, r.tier) : "—"), h("td", {}, h("span", { class: "mono small" }, r.chosen || "—"), r.cost > 0 ? h("div", { class: "muted small" }, moneyEl(r.cost)) : null, r.source && r.source.includes("task-rule") ? h("div", { class: "badge", style: "margin-top:2px" }, "task-rule") : null), answerCell(r, r.chosen || "cortiq-auto"))); } else { header = h("tr", {}, h("th", {}, t("bm.test")), ...models.map((m) => h("th", { class: "mono" }, m))); rows = (run.models[models[0]].results || []).map((_, i) => h("tr", {}, h("td", {}, h("div", {}, run.models[models[0]].results[i].id), h("div", { class: "muted small" }, run.models[models[0]].results[i].cat)), ...models.map((m) => { const r = run.models[m].results[i]; return r ? answerCell(r, m) : h("td", {}, "—"); }))); } mount(resultsHost, h("div", { class: "card" }, h("div", { class: "card-head" }, h("h3", {}, (routing ? "🧭 " : "🎯 ") + t("bm.results")), h("span", { class: "card-head sub" }, (live ? t("bm.liveRunning") + " · " : "") + new Date(run.ts).toLocaleString() + (run.quick ? " · " + t("bm.quick") : ""))), summary, h("div", { class: "table-wrap", style: "margin-top:14px" }, h("table", {}, h("thead", {}, header), h("tbody", {}, ...rows))), h("div", { class: "muted small", style: "margin-top:6px" }, t("bm.cellHint"))), detail); } function renderHistory() { const l = history(); if (!l.length) { mount(historyHost); return; } const clearAll = h("button", { class: "btn ghost sm danger" }, "🗑 " + t("bm.clearHistory")); clearAll.addEventListener("click", () => { saveHistory([]); renderHistory(); }); mount(historyHost, h("div", { class: "card" }, h("div", { class: "card-head" }, h("h3", {}, t("bm.history")), h("div", { class: "right" }, clearAll)), ...l.map((run, idx) => { const models = Object.keys(run.models); const scoreOf = (m) => { const rs = run.models[m].results || []; return rs.length ? rs.reduce((a, r) => a + r.score, 0) / rs.length : 0; }; const best = models.slice().sort((a, b) => scoreOf(b) - scoreOf(a))[0]; const row = h("div", { class: "flex bench-row", style: "cursor:pointer" }, h("span", {}, run.mode === "routing" ? "🧭" : "🎯"), h("span", { class: "muted small" }, new Date(run.ts).toLocaleString()), h("span", { class: "grow mono small" }, run.mode === "routing" ? t("bm.mode.routing") : models.join(", ")), run.quick ? h("span", { class: "badge" }, t("bm.quick")) : null, best ? h("b", {}, Math.round(scoreOf(best) * 100) + "%") : null, h("button", { class: "btn sm ghost danger", title: t("common.delete"), onClick: (e) => { e.stopPropagation(); l.splice(idx, 1); saveHistory(l); renderHistory(); } }, "×")); row.addEventListener("click", () => renderResults(run)); return row; }))); } // ── custom tests (stored in the browser) ── function customCard() { const host = h("div", { class: "card" }); function paint() { const list = customTests(); const promptIn = h("input", { placeholder: t("bm.promptPh") }); const checkIn = h("input", { placeholder: t("bm.checkPh") }); const addBtn = h("button", { class: "btn ghost sm" }, "+ " + t("bm.addTest")); addBtn.addEventListener("click", () => { const prompt = promptIn.value.trim(); const check = checkIn.value.split(",").map((s) => s.trim()).filter(Boolean); if (!prompt || !check.length) return; list.push({ id: "custom-" + (list.length + 1), cat: "custom", custom: true, max: 128, prompt, check }); saveCustom(list); paint(); }); mount(host, h("div", { class: "card-title" }, t("bm.custom")), h("div", { class: "muted small", style: "margin-bottom:8px" }, t("bm.customHint")), ...list.map((tst, i) => h("div", { class: "flex bench-row" }, h("span", { class: "grow small" }, tst.prompt.slice(0, 90)), h("span", { class: "muted small mono" }, tst.check.join(", ").slice(0, 40)), h("button", { class: "btn sm ghost danger", onClick: () => { list.splice(i, 1); saveCustom(list); paint(); } }, "×"))), h("div", { class: "flex", style: "gap:8px; margin-top:8px" }, promptIn, checkIn, addBtn)); } paint(); return host; } mount(root, h("div", { class: "page-head" }, h("h2", {}, t("bm.title")), h("p", {}, t("bm.subtitle"))), ids.length === 0 ? h("div", { class: "callout" }, t("routing.empty")) : null, h("div", { class: "card" }, modeSeg, modeDesc, modelsPick, h("div", { class: "flex", style: "justify-content:space-between; margin-top:14px" }, h("label", { class: "check", style: "margin:0", title: t("bm.quickHint") }, quickCb, "⚡ " + t("bm.quickMode")), h("div", { class: "flex", style: "gap:8px" }, stopBtn, runBtn)), h("div", { class: "muted small", style: "margin-top:6px" }, t("bm.suiteInfo", { q: SUITE.filter((x) => x.quick).length, f: SUITE.length }))), progressHost, resultsHost, customCard(), historyHost); renderHistory(); return root; }