From 9af51b679e9c2496ca1ebc60cfee740e71d40125 Mon Sep 17 00:00:00 2001
From: infosave2007 <urevich55@gmail.com>
Date: Fri, 7 Aug 2026 19:32:47 +0300
Subject: [PATCH] =?UTF-8?q?fix(bench):=20newer=20cortiq=20pretty-prints=20?=
 =?UTF-8?q?--json=20=E2=80=94=20parse=20it,=20and=20report=20honest=20tok/?=
 =?UTF-8?q?s?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The one-click model benchmark showed "error" even though cortiq bench ran
fine: the parser expected the JSON result on a single line, while newer
cortiq releases pretty-print it across many. Extraction now finds the last
balanced JSON object anywhere in the mixed CLI output (both formats work).

Also hardened tok/s selection: steady decode speed (decode_tok_s_steady)
is preferred over incl-prefill, and the loose "contains tok+per" fallback —
which would happily report allocs_per_token=769 as tok/s — now only matches
*tok_s* / *tokens_per* fields.
---
 src/bench.rs | 83 ++++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 71 insertions(+), 12 deletions(-)

diff --git a/src/bench.rs b/src/bench.rs
index e07ff42..e59e1a3 100644
--- a/src/bench.rs
+++ b/src/bench.rs
@@ -32,11 +32,19 @@ fn now() -> u64 {
         .as_secs()
 }
 
-/// tok/s out of the benchmark's JSON: first numeric field whose key mentions
-/// tokens-per-second (the exact name has varied across cortiq releases).
+/// tok/s out of the benchmark's JSON: known field names first (steady decode
+/// speed is the honest number), then any `*tok_s*` field as a fallback. The
+/// old loose "contains tok + per" heuristic could grab `allocs_per_token`.
 fn extract_tok_s(v: &serde_json::Value) -> Option<f64> {
     let obj = v.as_object()?;
-    for want in ["tok_s", "tok_per_s", "tokens_per_sec", "decode_tok_s"] {
+    for want in [
+        "decode_tok_s_steady",
+        "decode_tok_s_incl_prefill",
+        "decode_tok_s",
+        "tok_s",
+        "tok_per_s",
+        "tokens_per_sec",
+    ] {
         if let Some(n) = obj.get(want).and_then(|x| x.as_f64()) {
             return Some(n);
         }
@@ -44,12 +52,44 @@ fn extract_tok_s(v: &serde_json::Value) -> Option<f64> {
     obj.iter()
         .find(|(k, val)| {
             let k = k.to_ascii_lowercase();
-            (k.contains("tok") && (k.contains("/s") || k.contains("_s") || k.contains("per")))
-                && val.as_f64().is_some()
+            (k.contains("tok_s") || k.contains("tokens_per")) && val.as_f64().is_some()
         })
         .and_then(|(_, val)| val.as_f64())
 }
 
+/// Last parseable JSON object in mixed CLI output. Newer cortiq releases
+/// pretty-print the `--json` result over many lines, older ones emit a single
+/// line — this handles both, skipping any log noise around it.
+fn last_json_object(text: &str) -> Option<serde_json::Value> {
+    let mut end = text.len();
+    while let Some(close) = text[..end].rfind('}') {
+        let mut depth = 0i32;
+        let mut start = None;
+        for (i, c) in text[..=close].char_indices().rev() {
+            match c {
+                '}' => depth += 1,
+                '{' => {
+                    depth -= 1;
+                    if depth == 0 {
+                        start = Some(i);
+                        break;
+                    }
+                }
+                _ => {}
+            }
+        }
+        if let Some(st) = start {
+            if let Ok(v) = serde_json::from_str::<serde_json::Value>(&text[st..=close]) {
+                if v.is_object() {
+                    return Some(v);
+                }
+            }
+        }
+        end = close;
+    }
+    None
+}
+
 impl BenchStore {
     pub fn new() -> Arc<Self> {
         Arc::new(Self::default())
@@ -125,13 +165,7 @@ pub fn start(
                 if n > 30 {
                     run.log.drain(0..n - 30);
                 }
-                // the --json contract: one JSON object on its own line
-                let parsed = text
-                    .lines()
-                    .rev()
-                    .map(str::trim)
-                    .find(|l| l.starts_with('{') && l.ends_with('}'))
-                    .and_then(|l| serde_json::from_str::<serde_json::Value>(l).ok());
+                let parsed = last_json_object(&text);
                 match parsed {
                     Some(v) if o.status.success() => {
                         run.tok_s = extract_tok_s(&v);
@@ -163,4 +197,29 @@ mod tests {
         let v = serde_json::json!({"decode_tokens_per_sec": 31.0});
         assert_eq!(extract_tok_s(&v), Some(31.0));
     }
+
+    #[test]
+    fn steady_decode_wins_and_allocs_never_matches() {
+        let v = serde_json::json!({
+            "allocs_per_token": 769.4,
+            "decode_tok_s_incl_prefill": 9.98,
+            "decode_tok_s_steady": 11.86,
+            "prefill_tok_s": 25.5,
+        });
+        assert_eq!(extract_tok_s(&v), Some(11.86));
+        let v = serde_json::json!({"allocs_per_token": 769.4, "seq_len": 140});
+        assert_eq!(extract_tok_s(&v), None);
+    }
+
+    #[test]
+    fn json_found_single_line_and_pretty() {
+        let single = "noise\n{\"tok_s\": 5.0}\nmore noise";
+        assert_eq!(last_json_object(single).unwrap()["tok_s"], 5.0);
+        let pretty = "INFO loading\n{\n  \"decode_tok_s_steady\": 11.86,\n  \"seq_len\": 140\n}\nINFO done {not json";
+        assert_eq!(
+            last_json_object(pretty).unwrap()["decode_tok_s_steady"],
+            11.86
+        );
+        assert!(last_json_object("no braces here").is_none());
+    }
 }