docs: add task-type accuracy benchmark (router vs keyword heuristic)
- bench/tasks.jsonl (37 natural-language prompts, 7 task types) + bench/accuracy.py - measured on the live allaigate router: 100% (37/37) vs keyword heuristic 32% - BENCHMARKS.md accuracy section; README (EN+RU) perf+accuracy tables and test matrix - run.sh runs accuracy when CORTIQ_ROUTER_KEY is set
This commit is contained in:
infosave2007
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Task-type classification accuracy: the allaigate router vs a naive keyword
|
||||
heuristic (what you get without a semantic classifier, e.g. with LiteLLM/Portkey).
|
||||
|
||||
CORTIQ_ROUTER_KEY=cortiq_... python3 bench/accuracy.py
|
||||
|
||||
Env: CORTIQ_ROUTER_KEY (required), ROUTER_URL (default https://138.226.222.209),
|
||||
TAXONOMY (default data-assistant).
|
||||
"""
|
||||
import json, os, ssl, sys, urllib.request
|
||||
|
||||
ROUTER_URL = os.environ.get("ROUTER_URL", "https://138.226.222.209").rstrip("/")
|
||||
TAXONOMY = os.environ.get("TAXONOMY", "data-assistant")
|
||||
KEY = os.environ.get("CORTIQ_ROUTER_KEY", "")
|
||||
CTX = ssl._create_unverified_context()
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
DATA = os.path.join(HERE, "tasks.jsonl")
|
||||
|
||||
|
||||
def router_predict(text):
|
||||
body = json.dumps({"input": {"text": text}, "taxonomy_id": TAXONOMY}).encode()
|
||||
req = urllib.request.Request(
|
||||
ROUTER_URL + "/v1/route",
|
||||
data=body,
|
||||
headers={"content-type": "application/json", "Authorization": "Bearer " + KEY},
|
||||
)
|
||||
with urllib.request.urlopen(req, context=CTX, timeout=30) as r:
|
||||
d = json.load(r)
|
||||
return d.get("decision", {}).get("task_label", "?")
|
||||
|
||||
|
||||
def keyword_predict(text):
|
||||
"""Naive DIY routing without a semantic classifier."""
|
||||
t = text.lower()
|
||||
def has(*words):
|
||||
return any(w in t for w in words)
|
||||
if has("translate", "say ", "in spanish", "in japanese", "in german", "in italian",
|
||||
"to english", "to german", "from french", "render the phrase"):
|
||||
return "translation"
|
||||
if has("summarize", "summary", "tl;dr", "condense", "shorten", "bullet point"):
|
||||
return "summarization"
|
||||
if has("extract", "pull out", "list all", "find every", "find all"):
|
||||
return "extraction"
|
||||
if has("function", "python", "javascript", "rust", "sql", "query", "implement",
|
||||
"bug", "linked list", "code"):
|
||||
return "code"
|
||||
if has("solve", "derivative", "integral", "compute", "sum of", "area of",
|
||||
"limit of", "evaluate", "equation"):
|
||||
return "math"
|
||||
if t.strip().endswith("?") or t.split()[:1] and t.split()[0] in (
|
||||
"who", "what", "when", "where", "which"):
|
||||
return "qa"
|
||||
return "chitchat"
|
||||
|
||||
|
||||
def main():
|
||||
if not KEY:
|
||||
print("set CORTIQ_ROUTER_KEY", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
rows = [json.loads(l) for l in open(DATA) if l.strip()]
|
||||
r_ok = k_ok = 0
|
||||
misses = []
|
||||
for row in rows:
|
||||
gt = row["label"]
|
||||
try:
|
||||
rp = router_predict(row["text"])
|
||||
except Exception as e:
|
||||
rp = "ERR:" + str(e)[:30]
|
||||
kp = keyword_predict(row["text"])
|
||||
r_ok += rp == gt
|
||||
k_ok += kp == gt
|
||||
if rp != gt:
|
||||
misses.append((gt, rp, row["text"][:45]))
|
||||
n = len(rows)
|
||||
print(f"dataset: {n} prompts, {len(set(r['label'] for r in rows))} task types")
|
||||
print(f"allaigate router : {r_ok}/{n} = {100*r_ok/n:.1f}%")
|
||||
print(f"keyword heuristic: {k_ok}/{n} = {100*k_ok/n:.1f}%")
|
||||
if misses:
|
||||
print("\nrouter misses (truth -> predicted | prompt):")
|
||||
for gt, rp, txt in misses:
|
||||
print(f" {gt:>13} -> {rp:<13} | {txt}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+9
-2
@@ -91,6 +91,13 @@ else
|
||||
echo "portkey | (not installed — npm i -g @portkey-ai/gateway)"
|
||||
fi
|
||||
|
||||
# --- task-type routing accuracy (optional; needs a live router key) ----------
|
||||
if [ -n "${CORTIQ_ROUTER_KEY:-}" ]; then
|
||||
echo
|
||||
echo "== task-type routing accuracy (allaigate router vs keyword heuristic) =="
|
||||
python3 "$HERE/accuracy.py"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Note: numbers are gateway overhead over a zero-latency mock on one machine;"
|
||||
echo "subtract mock-direct to isolate proxy cost. Re-run on your own hardware."
|
||||
echo "Note: latency numbers are gateway overhead over a zero-latency mock on one"
|
||||
echo "machine; subtract mock-direct to isolate proxy cost. Re-run on your hardware."
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
{"text": "Write a Python function to reverse a string", "label": "code"}
|
||||
{"text": "Reverse a linked list in place", "label": "code"}
|
||||
{"text": "Given an array of integers, return the indices of the two that add up to a target", "label": "code"}
|
||||
{"text": "My loop prints one extra line at the end — what's wrong with it?", "label": "code"}
|
||||
{"text": "Merge two sorted arrays into a single sorted array", "label": "code"}
|
||||
{"text": "Make this faster: it compares every element with every other element", "label": "code"}
|
||||
{"text": "Solve x^2 - 5x + 6 = 0", "label": "math"}
|
||||
{"text": "If a train covers 60 miles in 1.5 hours, how fast is it going?", "label": "math"}
|
||||
{"text": "A rectangle is twice as long as it is wide and its area is 50 — what are its sides?", "label": "math"}
|
||||
{"text": "What's 15 percent of 240?", "label": "math"}
|
||||
{"text": "Two dice are rolled — what are the odds the total is seven?", "label": "math"}
|
||||
{"text": "The sequence 2, 6, 18, 54 — what comes next?", "label": "math"}
|
||||
{"text": "Translate 'good morning' into Spanish", "label": "translation"}
|
||||
{"text": "'Guten Tag' — what does that mean in English?", "label": "translation"}
|
||||
{"text": "What's the Spanish word for butterfly?", "label": "translation"}
|
||||
{"text": "Put 'I would like a coffee, please' into French", "label": "translation"}
|
||||
{"text": "How do Italians say 'see you tomorrow'?", "label": "translation"}
|
||||
{"text": "Give me the gist of the text below in one line", "label": "summarization"}
|
||||
{"text": "What are the main takeaways from this report?", "label": "summarization"}
|
||||
{"text": "Boil this down to a few key points", "label": "summarization"}
|
||||
{"text": "In a sentence, what is this article about?", "label": "summarization"}
|
||||
{"text": "Cut this down to just the essentials", "label": "summarization"}
|
||||
{"text": "Ugh, Mondays. How was your weekend?", "label": "chitchat"}
|
||||
{"text": "I'm a little bored, tell me something interesting", "label": "chitchat"}
|
||||
{"text": "Do you think pineapple belongs on pizza?", "label": "chitchat"}
|
||||
{"text": "Feeling great today, the sun is finally out!", "label": "chitchat"}
|
||||
{"text": "Any fun plans for the holidays?", "label": "chitchat"}
|
||||
{"text": "Which one is the tallest mountain on Earth?", "label": "qa"}
|
||||
{"text": "At what temperature does water freeze in Fahrenheit?", "label": "qa"}
|
||||
{"text": "Who is the author of Pride and Prejudice?", "label": "qa"}
|
||||
{"text": "How many continents are there?", "label": "qa"}
|
||||
{"text": "What is the currency used in Japan called?", "label": "qa"}
|
||||
{"text": "From the paragraph below, grab all the phone numbers", "label": "extraction"}
|
||||
{"text": "Which companies are named in this report?", "label": "extraction"}
|
||||
{"text": "Give me just the dates and amounts from this receipt", "label": "extraction"}
|
||||
{"text": "Pick out the email addresses in this message", "label": "extraction"}
|
||||
{"text": "What are the product SKUs mentioned here?", "label": "extraction"}
|
||||
Reference in New Issue
Block a user