feat: complete production-quality Model Router AI traffic control platform
This commit is contained in:
Vardhman Gupta
@@ -0,0 +1,150 @@
|
||||
import uuid
|
||||
import datetime
|
||||
from typing import List, Dict, Any, Optional
|
||||
from app.models.schemas import (
|
||||
ModelMetadata,
|
||||
RequestAnalysis,
|
||||
RoutingDecision,
|
||||
CandidateScore,
|
||||
PriorityLevel,
|
||||
)
|
||||
from app.router.scoring import filter_candidate_models, compute_candidate_score
|
||||
from app.router.rules_engine import evaluate_routing_rules
|
||||
from app.storage.models import RoutingRuleRecord
|
||||
|
||||
|
||||
def generate_routing_explanation(
|
||||
selected_model: ModelMetadata,
|
||||
analysis: RequestAnalysis,
|
||||
policy_name: str,
|
||||
rule_name: Optional[str] = None,
|
||||
) -> List[str]:
|
||||
reasons = []
|
||||
|
||||
if rule_name:
|
||||
reasons.append(f"Matching rule applied: \"{rule_name}\"")
|
||||
|
||||
# Task explanation
|
||||
reasons.append(f"Task classified as {analysis.task_type.value} with {analysis.complexity_label.value} complexity ({analysis.complexity:.2f})")
|
||||
|
||||
# Reasoning / Code requirements
|
||||
if analysis.reasoning_required:
|
||||
reasons.append("Multi-step reasoning & deep inference required for problem domain")
|
||||
if analysis.coding_required:
|
||||
reasons.append("Syntactical & algorithmic coding capabilities required")
|
||||
|
||||
# Model specific match
|
||||
if selected_model.tier.value == "POWER":
|
||||
reasons.append(f"High reasoning score ({selected_model.quality_score * 100:.0f}%) and deep context ({selected_model.context_window:,} tokens)")
|
||||
elif selected_model.tier.value == "FAST":
|
||||
reasons.append(f"Low latency priority and high speed score ({selected_model.speed_score * 100:.0f}%)")
|
||||
else:
|
||||
reasons.append("Balanced quality and cost profile selected for optimal efficiency")
|
||||
|
||||
# Cost / local factor
|
||||
if selected_model.cost_per_input_token == 0.0:
|
||||
reasons.append("Local zero-cost execution prioritized")
|
||||
elif analysis.cost_sensitivity == PriorityLevel.HIGH:
|
||||
reasons.append("Cost sensitivity is HIGH — model offers lowest token pricing")
|
||||
|
||||
# Policy
|
||||
reasons.append(f"Evaluated under '{policy_name}' policy weights")
|
||||
|
||||
return reasons
|
||||
|
||||
|
||||
def route_request(
|
||||
analysis: RequestAnalysis,
|
||||
available_models: List[ModelMetadata],
|
||||
policy_weights: Dict[str, float],
|
||||
policy_name: str = "balanced",
|
||||
rules: Optional[List[RoutingRuleRecord]] = None,
|
||||
budget_percent: float = 0.0,
|
||||
request_id: Optional[str] = None,
|
||||
) -> RoutingDecision:
|
||||
req_id = request_id or str(uuid.uuid4())
|
||||
decision_id = f"dec_{uuid.uuid4().hex[:12]}"
|
||||
now_iso = datetime.datetime.utcnow().isoformat()
|
||||
|
||||
# 1. Rule evaluation
|
||||
rule_action, rule_target, rule_name = None, None, None
|
||||
if rules:
|
||||
rule_action, rule_target, rule_name = evaluate_routing_rules(rules, analysis, budget_percent)
|
||||
|
||||
# 2. Hard filter candidates
|
||||
eligible_models, rejected_dict = filter_candidate_models(available_models, analysis)
|
||||
|
||||
if not eligible_models:
|
||||
# Fallback to any active model if everything was filtered
|
||||
eligible_models = [m for m in available_models if m.is_active] or available_models
|
||||
|
||||
# Handle Rule Action: ROUTE_TO specific model
|
||||
if rule_action == "ROUTE_TO" and rule_target:
|
||||
exact_match = next((m for m in eligible_models if m.id == rule_target), None)
|
||||
if exact_match:
|
||||
scores = [compute_candidate_score(m, analysis, policy_weights) for m in eligible_models]
|
||||
score_map = {s.model_id: s for s in scores}
|
||||
selected_score = score_map.get(exact_match.id) or compute_candidate_score(exact_match, analysis, policy_weights)
|
||||
|
||||
reasons = generate_routing_explanation(exact_match, analysis, policy_name, rule_name)
|
||||
return RoutingDecision(
|
||||
decision_id=decision_id,
|
||||
request_id=req_id,
|
||||
selected_model=exact_match.id,
|
||||
selected_model_name=exact_match.name,
|
||||
provider=exact_match.provider,
|
||||
tier=exact_match.tier.value if hasattr(exact_match.tier, "value") else str(exact_match.tier),
|
||||
confidence=0.98,
|
||||
policy_used=policy_name,
|
||||
reasons=reasons,
|
||||
candidate_scores=scores,
|
||||
rejected_candidates=rejected_dict,
|
||||
estimated_cost_usd=selected_score.estimated_cost_usd,
|
||||
estimated_latency_ms=selected_score.estimated_latency_ms,
|
||||
rule_applied=rule_name,
|
||||
timestamp=now_iso,
|
||||
)
|
||||
|
||||
# Handle Rule Action: FORCE_TIER
|
||||
if rule_action == "FORCE_TIER" and rule_target:
|
||||
tier_filtered = [m for m in eligible_models if m.tier.value.upper() == rule_target.upper()]
|
||||
if tier_filtered:
|
||||
eligible_models = tier_filtered
|
||||
|
||||
# 3. Score all eligible models
|
||||
candidate_scores: List[CandidateScore] = []
|
||||
for model in eligible_models:
|
||||
sc = compute_candidate_score(model, analysis, policy_weights)
|
||||
candidate_scores.append(sc)
|
||||
|
||||
# Sort descending by overall_score
|
||||
candidate_scores.sort(key=lambda s: s.overall_score, reverse=True)
|
||||
best_candidate = candidate_scores[0]
|
||||
selected_model = next(m for m in eligible_models if m.id == best_candidate.model_id)
|
||||
|
||||
# Confidence calculation: score difference from second candidate
|
||||
if len(candidate_scores) > 1:
|
||||
gap = best_candidate.overall_score - candidate_scores[1].overall_score
|
||||
confidence = min(0.99, max(0.70, round(0.80 + (gap / 100.0), 2)))
|
||||
else:
|
||||
confidence = 0.95
|
||||
|
||||
reasons = generate_routing_explanation(selected_model, analysis, policy_name, rule_name)
|
||||
|
||||
return RoutingDecision(
|
||||
decision_id=decision_id,
|
||||
request_id=req_id,
|
||||
selected_model=selected_model.id,
|
||||
selected_model_name=selected_model.name,
|
||||
provider=selected_model.provider,
|
||||
tier=selected_model.tier.value if hasattr(selected_model.tier, "value") else str(selected_model.tier),
|
||||
confidence=confidence,
|
||||
policy_used=policy_name,
|
||||
reasons=reasons,
|
||||
candidate_scores=candidate_scores,
|
||||
rejected_candidates=rejected_dict,
|
||||
estimated_cost_usd=best_candidate.estimated_cost_usd,
|
||||
estimated_latency_ms=best_candidate.estimated_latency_ms,
|
||||
rule_applied=rule_name,
|
||||
timestamp=now_iso,
|
||||
)
|
||||
@@ -0,0 +1,73 @@
|
||||
from typing import List, Optional, Tuple
|
||||
from app.storage.models import RoutingRuleRecord
|
||||
from app.models.schemas import RequestAnalysis
|
||||
|
||||
|
||||
def evaluate_routing_rules(
|
||||
rules: List[RoutingRuleRecord],
|
||||
analysis: RequestAnalysis,
|
||||
current_budget_percent: float = 0.0,
|
||||
) -> Tuple[Optional[str], Optional[str], Optional[str]]:
|
||||
"""
|
||||
Evaluates enabled routing rules in order of priority (highest priority number first).
|
||||
Returns: (action_type, action_target, matched_rule_name) or (None, None, None)
|
||||
"""
|
||||
sorted_rules = sorted([r for r in rules if r.is_enabled], key=lambda x: x.priority, reverse=True)
|
||||
|
||||
for rule in sorted_rules:
|
||||
field = rule.condition_field.lower()
|
||||
op = rule.condition_operator
|
||||
target_val = rule.condition_value
|
||||
|
||||
matched = False
|
||||
|
||||
if field == "task_type":
|
||||
val = analysis.task_type.value
|
||||
if op == "==":
|
||||
matched = (val == target_val)
|
||||
elif op == "!=":
|
||||
matched = (val != target_val)
|
||||
elif op == "contains":
|
||||
matched = (target_val in val)
|
||||
|
||||
elif field == "complexity":
|
||||
val = analysis.complexity
|
||||
try:
|
||||
target_num = float(target_val)
|
||||
if op == ">=":
|
||||
matched = (val >= target_num)
|
||||
elif op == ">":
|
||||
matched = (val > target_num)
|
||||
elif op == "<=":
|
||||
matched = (val <= target_num)
|
||||
elif op == "<":
|
||||
matched = (val < target_num)
|
||||
elif op == "==":
|
||||
matched = (abs(val - target_num) < 0.01)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
elif field in ("budget_percent", "monthly_budget_percent"):
|
||||
try:
|
||||
target_num = float(target_val)
|
||||
if op == ">=":
|
||||
matched = (current_budget_percent >= target_num)
|
||||
elif op == ">":
|
||||
matched = (current_budget_percent > target_num)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
elif field == "context_size":
|
||||
try:
|
||||
target_num = int(target_val)
|
||||
if op == ">=":
|
||||
matched = (analysis.context_size >= target_num)
|
||||
elif op == ">":
|
||||
matched = (analysis.context_size > target_num)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if matched:
|
||||
return rule.action_type, rule.action_target, rule.name
|
||||
|
||||
return None, None, None
|
||||
@@ -0,0 +1,154 @@
|
||||
from typing import List, Dict, Tuple, Optional
|
||||
from app.models.schemas import ModelMetadata, RequestAnalysis, CandidateScore, PriorityLevel
|
||||
|
||||
|
||||
def filter_candidate_models(
|
||||
models: List[ModelMetadata],
|
||||
analysis: RequestAnalysis,
|
||||
) -> Tuple[List[ModelMetadata], Dict[str, str]]:
|
||||
"""
|
||||
Hard-filtering step: eliminates models that cannot satisfy non-negotiable requirements.
|
||||
Returns: (eligible_models, rejected_candidates_with_reasons)
|
||||
"""
|
||||
eligible: List[ModelMetadata] = []
|
||||
rejected: Dict[str, str] = {}
|
||||
|
||||
for m in models:
|
||||
if not m.is_active:
|
||||
rejected[m.id] = "Model is currently disabled/inactive."
|
||||
continue
|
||||
|
||||
# 1. Context Window check
|
||||
if analysis.context_size > m.context_window:
|
||||
rejected[m.id] = (
|
||||
f"Context window ({m.context_window} tokens) is smaller than required input ({analysis.context_size} tokens)."
|
||||
)
|
||||
continue
|
||||
|
||||
# 2. Vision requirement check
|
||||
if analysis.vision_required and not m.supports_vision:
|
||||
rejected[m.id] = "Task requires vision / multimodal capabilities which this model does not support."
|
||||
continue
|
||||
|
||||
# 3. Coding capability check (for high complexity code tasks)
|
||||
if analysis.coding_required and analysis.complexity >= 0.70 and not m.supports_coding:
|
||||
rejected[m.id] = "Task requires specialized coding capabilities for complex logic."
|
||||
continue
|
||||
|
||||
# 4. Deep reasoning capability check
|
||||
if analysis.reasoning_required and analysis.complexity >= 0.85 and not m.supports_reasoning:
|
||||
rejected[m.id] = "Task requires advanced reasoning capabilities for multi-step problem solving."
|
||||
continue
|
||||
|
||||
eligible.append(m)
|
||||
|
||||
# If all filtered out (edge case), return active models to avoid total failure
|
||||
if not eligible and models:
|
||||
for m in models:
|
||||
if m.is_active:
|
||||
eligible.append(m)
|
||||
rejected.clear()
|
||||
|
||||
return eligible, rejected
|
||||
|
||||
|
||||
def compute_candidate_score(
|
||||
model: ModelMetadata,
|
||||
analysis: RequestAnalysis,
|
||||
weights: Dict[str, float],
|
||||
) -> CandidateScore:
|
||||
"""
|
||||
Multi-criteria configurable scoring formula.
|
||||
Quality, Speed, CostEfficiency, CapabilityMatch, Reliability.
|
||||
Normalized score: 0 to 100.
|
||||
"""
|
||||
# 1. Quality Component (0.0 to 1.0)
|
||||
quality_comp = model.quality_score
|
||||
if analysis.quality_requirement == PriorityLevel.HIGH:
|
||||
# Boost premium models for high quality requirements
|
||||
if model.tier.value == "POWER":
|
||||
quality_comp = min(1.0, quality_comp * 1.15)
|
||||
elif analysis.quality_requirement == PriorityLevel.LOW:
|
||||
quality_comp = model.quality_score * 0.9
|
||||
|
||||
# 2. Speed Component (0.0 to 1.0)
|
||||
speed_comp = model.speed_score
|
||||
if analysis.latency_priority == PriorityLevel.HIGH:
|
||||
if model.tier.value == "FAST":
|
||||
speed_comp = min(1.0, speed_comp * 1.20)
|
||||
elif analysis.latency_priority == PriorityLevel.LOW:
|
||||
speed_comp = model.speed_score * 0.85
|
||||
|
||||
# 3. Cost Efficiency Component (0.0 to 1.0)
|
||||
# Zero-cost local models get 1.0, cheaper cloud models get high score
|
||||
combined_token_cost = (model.cost_per_input_token * 1000) + (model.cost_per_output_token * 1000)
|
||||
if combined_token_cost == 0.0:
|
||||
cost_efficiency = 1.0
|
||||
else:
|
||||
# Scale: $0.0001 -> 0.95, $0.01 -> 0.50, $0.05 -> 0.10
|
||||
cost_efficiency = max(0.05, min(0.99, 1.0 / (1.0 + (combined_token_cost * 100.0))))
|
||||
|
||||
if analysis.cost_sensitivity == PriorityLevel.HIGH:
|
||||
cost_efficiency = min(1.0, cost_efficiency * 1.25)
|
||||
|
||||
# 4. Capability Match Component (0.0 to 1.0)
|
||||
cap_points = 0.0
|
||||
total_caps = 0.0
|
||||
|
||||
if analysis.coding_required:
|
||||
total_caps += 1.0
|
||||
if model.supports_coding:
|
||||
cap_points += 1.0
|
||||
|
||||
if analysis.reasoning_required:
|
||||
total_caps += 1.0
|
||||
if model.supports_reasoning:
|
||||
cap_points += 1.0
|
||||
|
||||
if analysis.vision_required:
|
||||
total_caps += 1.0
|
||||
if model.supports_vision:
|
||||
cap_points += 1.0
|
||||
|
||||
capability_comp = (cap_points / total_caps) if total_caps > 0 else 0.90
|
||||
|
||||
# 5. Reliability Component (0.0 to 1.0)
|
||||
reliability_comp = model.reliability_score
|
||||
|
||||
# Weighted Sum
|
||||
w_q = weights.get("quality_weight", 0.35)
|
||||
w_c = weights.get("cost_weight", 0.25)
|
||||
w_s = weights.get("speed_weight", 0.20)
|
||||
w_cap = weights.get("capability_weight", 0.15)
|
||||
w_r = weights.get("reliability_weight", 0.05)
|
||||
total_w = w_q + w_c + w_s + w_cap + w_r
|
||||
|
||||
raw_score = (
|
||||
(quality_comp * w_q)
|
||||
+ (cost_efficiency * w_c)
|
||||
+ (speed_comp * w_s)
|
||||
+ (capability_comp * w_cap)
|
||||
+ (reliability_comp * w_r)
|
||||
) / (total_w if total_w > 0 else 1.0)
|
||||
|
||||
overall_score = round(raw_score * 100.0, 2)
|
||||
|
||||
# Estimated metrics
|
||||
est_latency = 120.0 if model.tier.value == "FAST" else (350.0 if model.tier.value == "BALANCED" else 750.0)
|
||||
est_cost = (analysis.context_size * model.cost_per_input_token) + (250 * model.cost_per_output_token)
|
||||
|
||||
return CandidateScore(
|
||||
model_id=model.id,
|
||||
model_name=model.name,
|
||||
provider=model.provider,
|
||||
tier=model.tier.value if hasattr(model.tier, "value") else str(model.tier),
|
||||
overall_score=overall_score,
|
||||
quality_component=round(quality_comp * 100.0, 1),
|
||||
cost_component=round(cost_efficiency * 100.0, 1),
|
||||
speed_component=round(speed_comp * 100.0, 1),
|
||||
capability_component=round(capability_comp * 100.0, 1),
|
||||
reliability_component=round(reliability_comp * 100.0, 1),
|
||||
estimated_latency_ms=est_latency,
|
||||
estimated_cost_usd=round(est_cost, 6),
|
||||
eligible=True,
|
||||
)
|
||||
Reference in New Issue
Block a user