github/ai-model-router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cli): add --json output flag to doctor, models, route, run, and analytics (#13)

Browse Source 
- Adds optional `--json` / `-j` flag to CLI commands for scriptable output
- Redirects non-JSON logging messages to stderr under JSON mode
- Adds test suite for CLI JSON serialization in backend/tests/test_cli.py
- Updates documentation in README.md
This commit is contained in:
Sanaullah Turab
2026-09-11 19:15:22 +05:30
committed by GitHub
parent e6d9640112
commit 077b3e2af4
3 changed files with 206 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -1
View File
@@ -179,7 +179,8 @@ npm run dev
## CLI Usage
The built-in Typer CLI provides terminal commands for inspection, diagnostics, and testing (use `modelrouter` if installed via pip, or `python backend/app/cli/main.py` from source):
The built-in Typer CLI provides terminal commands for inspection, diagnostics, and testing (use `modelrouter` if installed via pip, or `python backend/app/cli/main.py` from source).
You can append `--json` or `-j` to most commands to output raw, pipeable JSON instead of Rich formatted tables:
```bash
# Launch the AI Traffic Control Room Web UI and API Gateway
backend/app/cli/main.py
+101 -32
View File
@@ -3,6 +3,8 @@ import os
import asyncio
import httpx
import typer
import json
from typing import Any, Callable
from rich.console import Console
from rich.table import Table
from rich.panel import Panel
@@ -29,62 +31,90 @@ from sqlalchemy import select
app = typer.Typer(help="Model Router: Intelligent LLM Request Routing Platform")
console = Console(safe_box=True)
err_console = Console(stderr=True, safe_box=True)
settings = get_settings()
def _render_output(json_output: bool, data: Any, render_fn: Callable[[], None]) -> None:
if json_output:
if hasattr(data, "model_dump_json"):
print(data.model_dump_json(indent=2))
elif isinstance(data, list) and all(hasattr(x, "model_dump") for x in data):
print(json.dumps([m.model_dump(mode="json") for m in data], indent=2))
else:
print(json.dumps(data, indent=2))
else:
render_fn()
@app.command()
def doctor():
def doctor(
json_output: bool = typer.Option(False, "--json", "-j", help="Output raw JSON instead of formatted tables"),
):
"""Run environment, connectivity, and dependency health checks."""
console.print(Panel.fit("[bold cyan]Model Router Diagnostics (modelrouter doctor)[/bold cyan]"))
data = {
"python": {"version": f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}", "ok": True},
"database": {"ok": False, "message": ""},
"ollama": {"ok": False, "models": [], "message": ""},
"mock_engine": {"ok": True},
"providers": {
"openai": {"configured": bool(settings.OPENAI_API_KEY)},
"anthropic": {"configured": bool(settings.ANTHROPIC_API_KEY)},
"gemini": {"configured": bool(settings.GEMINI_API_KEY)},
}
}
# 1. Python Check
py_ver = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
console.print(f" [green][OK][/green] Python: [bold]{py_ver}[/bold]")
# 2. Database Check
try:
asyncio.run(init_db())
console.print(" [green][OK][/green] SQLite Database: [bold]Initialized & Ready[/bold]")
data["database"]["ok"] = True
data["database"]["message"] = "Initialized & Ready"
except Exception as exc:
console.print(f" [red][FAIL][/red] Database Error: {exc}")
data["database"]["message"] = str(exc)
# 3. Ollama Connectivity Check
ollama_ok = False
try:
r = httpx.get(f"{settings.OLLAMA_BASE_URL}/api/tags", timeout=1.5)
if r.status_code == 200:
models = [m.get("name") for m in r.json().get("models", [])]
console.print(f" [green][OK][/green] Ollama (Local): [bold]Connected[/bold] ({len(models)} local models found)")
ollama_ok = True
data["ollama"]["ok"] = True
data["ollama"]["models"] = models
except Exception:
pass
if not ollama_ok:
if not data["ollama"]["ok"]:
data["ollama"]["message"] = f"Not running on {settings.OLLAMA_BASE_URL}"
def _render() -> None:
console.print(Panel.fit("[bold cyan]Model Router Diagnostics (modelrouter doctor)[/bold cyan]"))
console.print(f" [green][OK][/green] Python: [bold]{data['python']['version']}[/bold]")
if data["database"]["ok"]:
console.print(" [green][OK][/green] SQLite Database: [bold]Initialized & Ready[/bold]")
else:
console.print(f" [red][FAIL][/red] Database Error: {data['database']['message']}")
if data["ollama"]["ok"]:
console.print(f" [green][OK][/green] Ollama (Local): [bold]Connected[/bold] ({len(data['ollama']['models'])} local models found)")
else:
console.print(f" [yellow][!][/yellow] Ollama (Local): [dim]Not running on {settings.OLLAMA_BASE_URL} (Optional, local fallback available)[/dim]")
# 4. Mock Engine
console.print(" [green][OK][/green] Mock Engine: [bold]Ready (Zero API keys required)[/bold]")
# 5. External Providers
if settings.OPENAI_API_KEY:
console.print(" [green][OK][/green] OpenAI Provider: [bold]API Key Configured in .env[/bold]")
for prov, name in [("openai", "OpenAI"), ("anthropic", "Anthropic"), ("gemini", "Gemini")]:
if data["providers"][prov]["configured"]:
console.print(f" [green][OK][/green] {name} Provider: [bold]API Key Configured in .env[/bold]")
else:
console.print(" [yellow][!][/yellow] OpenAI Provider: [dim]Not configured (Optional)[/dim]")
if settings.ANTHROPIC_API_KEY:
console.print(" [green][OK][/green] Anthropic Provider: [bold]API Key Configured in .env[/bold]")
else:
console.print(" [yellow][!][/yellow] Anthropic Provider: [dim]Not configured (Optional)[/dim]")
if settings.GEMINI_API_KEY:
console.print(" [green][OK][/green] Gemini Provider: [bold]API Key Configured in .env[/bold]")
else:
console.print(" [yellow][!][/yellow] Gemini Provider: [dim]Not configured (Optional)[/dim]")
console.print(f" [yellow][!][/yellow] {name} Provider: [dim]Not configured (Optional)[/dim]")
console.print("\n[bold green]Ready for local-first intelligent routing![/bold green]\n")
_render_output(json_output, data, _render)
@app.command()
def models():
def models(
json_output: bool = typer.Option(False, "--json", "-j", help="Output raw JSON instead of formatted tables"),
):
"""List all registered candidate models and their capability scores."""
async def _list():
await init_db()
@@ -92,6 +122,7 @@ def models():
res = await db.execute(select(ModelRecord).order_by(ModelRecord.tier, ModelRecord.name))
records = res.scalars().all()
def _render() -> None:
table = Table(title="Model Router Registry", border_style="bright_blue")
table.add_column("Model ID", style="cyan", no_wrap=True)
table.add_column("Tier", style="magenta")
@@ -114,6 +145,9 @@ def models():
)
console.print(table)
data = [db_model_to_meta(m) for m in records]
_render_output(json_output, data, _render)
asyncio.run(_list())
@@ -121,6 +155,7 @@ def models():
def route(
prompt: str = typer.Argument(..., help="Prompt text to route"),
policy: str = typer.Option("balanced", "--policy", "-p", help="Routing policy to use"),
json_output: bool = typer.Option(False, "--json", "-j", help="Output raw JSON instead of formatted tables"),
):
"""Analyze prompt and explain why the optimal model was selected (Dry Run)."""
async def _route():
@@ -148,6 +183,7 @@ def route(
policy_name=policy,
)
def _render() -> None:
console.print(Panel.fit(
f"[bold white]Prompt:[/bold white] \"{prompt}\"\n"
f"[bold cyan]Task Type:[/bold cyan] {analysis.task_type.value} | [bold cyan]Complexity:[/bold cyan] {analysis.complexity_label.value} ({analysis.complexity:.2f})\n"
@@ -167,6 +203,8 @@ def route(
for cid, reason in decision.rejected_candidates.items():
console.print(f" [dim]- {cid}: {reason}[/dim]")
_render_output(json_output, decision, _render)
asyncio.run(_route())
@@ -174,6 +212,7 @@ def route(
def run(
prompt: str = typer.Argument(..., help="Prompt to route and execute"),
policy: str = typer.Option("balanced", "--policy", "-p", help="Routing policy to use"),
json_output: bool = typer.Option(False, "--json", "-j", help="Output raw JSON instead of formatted tables"),
):
"""Route request and execute inference through the chosen provider."""
async def _run():
@@ -191,7 +230,11 @@ def run(
policy_name=policy,
)
console.print(f"[bold cyan]Routing to:[/bold cyan] {decision.selected_model} via provider: {decision.provider}...")
routing_msg = f"[bold cyan]Routing to:[/bold cyan] {decision.selected_model} via provider: {decision.provider}..."
if json_output:
err_console.print(routing_msg)
else:
console.print(routing_msg)
resp, fallback_used, orig_m, fb_reason = await execute_with_fallback(
prompt=prompt,
@@ -200,6 +243,7 @@ def run(
all_models=models,
)
def _render() -> None:
console.print(Panel(
resp.content,
title=f"RESPONSE from {resp.model} ({'FALLBACK: ' + orig_m if fallback_used else 'PRIMARY'})",
@@ -207,6 +251,25 @@ def run(
border_style="green" if not fallback_used else "yellow",
))
data = {
"decision": json.loads(decision.model_dump_json()) if hasattr(decision, "model_dump_json") else (decision.model_dump() if hasattr(decision, "model_dump") else decision),
"response": json.loads(resp.model_dump_json()) if hasattr(resp, "model_dump_json") else {
"content": getattr(resp, "content", ""),
"model": getattr(resp, "model", ""),
"provider_latency_ms": getattr(resp, "provider_latency_ms", 0),
"total_tokens": getattr(resp, "total_tokens", 0),
},
"fallback": {
"used": fallback_used,
"original_model": orig_m,
"reason": fb_reason,
}
}
if hasattr(resp, "model_dump") and not hasattr(resp, "model_dump_json"):
data["response"] = resp.model_dump()
_render_output(json_output, data, _render)
asyncio.run(_run())
@@ -245,13 +308,17 @@ def traffic(limit: int = 10):
@app.command()
def analytics():
def analytics(
json_output: bool = typer.Option(False, "--json", "-j", help="Output raw JSON instead of formatted tables"),
):
"""Display system-wide routing performance and cost savings analytics."""
async def _analytics():
await init_db()
from app.analytics.service import get_system_analytics
async with AsyncSessionLocal() as db:
data = await get_system_analytics(db)
def _render() -> None:
console.print(Panel.fit(
f"[bold cyan]Total Routed Requests:[/bold cyan] {data['total_requests']}\n"
f"[bold green]Average Latency:[/bold green] {data['avg_latency_ms']}ms (Routing overhead: {data['avg_routing_latency_ms']}ms)\n"
@@ -262,6 +329,8 @@ def analytics():
border_style="bright_blue",
))
_render_output(json_output, data, _render)
asyncio.run(_analytics())
backend/tests/test_cli.py
+46
View File
@@ -0,0 +1,46 @@
import json
import pytest
from typer.testing import CliRunner
from app.cli.main import app
runner = CliRunner()
def test_cli_doctor_json():
result = runner.invoke(app, ["doctor", "--json"])
assert result.exit_code == 0
data = json.loads(result.stdout)
assert "python" in data
assert "database" in data
assert "mock_engine" in data
def test_cli_models_json():
result = runner.invoke(app, ["models", "--json"])
assert result.exit_code == 0
data = json.loads(result.stdout)
assert isinstance(data, list)
if len(data) > 0:
assert "id" in data[0]
assert "provider" in data[0]
def test_cli_route_json():
result = runner.invoke(app, ["route", "test prompt", "--json"])
assert result.exit_code == 0
data = json.loads(result.stdout)
assert "selected_model" in data
assert "estimated_latency_ms" in data
def test_cli_run_json():
result = runner.invoke(app, ["run", "test prompt", "--json"])
assert result.exit_code == 0
data = json.loads(result.stdout)
assert "decision" in data
assert "response" in data
assert "fallback" in data
def test_cli_analytics_json():
result = runner.invoke(app, ["analytics", "--json"])
assert result.exit_code == 0
data = json.loads(result.stdout)
assert "total_requests" in data
assert "avg_latency_ms" in data
assert "savings" in data