diff --git a/CHANGELOG.md b/CHANGELOG.md index 73f5509..cf28b57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/). semantically near a previous one (cosine threshold, TTL, ring buffer), skipping the model call. Partitioned by routing signature; hit-rate and cost saved are tracked and shown on the dashboard; configurable from Settings (`[cache]`). +- **MCP server** (`POST /mcp`, JSON-RPC 2.0): exposes routing to MCP-native orchestrators + via two tools — `cortiq_route` (classify → decision) and `cortiq_chat` (route → answer). ## [0.1.0] - 2026-06-30 diff --git a/README.md b/README.md index 8e88412..0616228 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,10 @@ One OpenAI-compatible endpoint → automatic model selection from your pool ## ✨ Highlights -- **Drop-in OpenAI API.** Point any OpenAI client at the gateway and send `model: "cortiq-auto"`. +- **Drop-in OpenAI API + streaming.** Point any OpenAI client at the gateway and send `model: "cortiq-auto"`; `stream: true` is fully supported (SSE). +- **Multi-protocol.** OpenAI Chat & Embeddings, **Anthropic Messages** (in/out, streaming), and an **MCP** server (`POST /mcp`) exposing routing as tools for agent orchestrators. - **Intelligent routing.** `complexity.tier` → ordered model pool (`low → local`, `high → cloud`), with fallback and graceful degradation when the router is down. +- **Semantic cache.** Optional embedding-based cache returns a stored answer for prompts semantically near a previous one — skipping the model call (hit-rate & savings on the dashboard). - **Built-in multilingual admin console.** Manage models, routing, protocols, keys and secrets from a web UI — **no hand-editing TOML, no restart** (changes are hot-reloaded). Available in **7 languages** (en, ru, de, fr, es, zh, tr) with light/dark themes. - **Live analytics.** Per-request stats (tokens, cost, latency, success rate, failovers), time-series charts, breakdowns by model/tier/task, and a Prometheus `GET /metrics` endpoint. - **Playground.** Send a prompt through the live pipeline and inspect the routing decision. @@ -147,12 +149,12 @@ Each adapter is toggled in the config — take only what you need. | Protocol | Endpoint | Status | |---|---|---| -| OpenAI Chat Completions | `POST /v1/chat/completions` | ✅ implemented | +| OpenAI Chat Completions | `POST /v1/chat/completions` | ✅ implemented (+ streaming) | +| OpenAI Embeddings | `POST /v1/embeddings` | ✅ implemented | +| Anthropic Messages | `POST /v1/messages` | ✅ implemented (+ streaming) | +| MCP (Model Context Protocol) | `POST /mcp` (JSON-RPC) | ✅ implemented | | OpenAI Completions (legacy) | `POST /v1/completions` | planned | -| OpenAI Embeddings | `POST /v1/embeddings` | planned | | OpenAI Models | `GET /v1/models` | planned | -| Anthropic Messages | `POST /v1/messages` | planned | -| MCP (Model Context Protocol) | JSON-RPC / SSE | planned | | Native passthrough | `POST /route` | planned | ## Supported providers (outbound) @@ -160,7 +162,7 @@ Each adapter is toggled in the config — take only what you need. | Provider | Covers | |---|---| | `openai` (OpenAI-compatible) | OpenAI, OpenRouter, Together, Groq, **vLLM, llama.cpp server, LM Studio, Ollama (`/v1`)** | -| `anthropic` | Claude Messages API (planned) | +| `anthropic` | Claude Messages API (+ streaming) | | `ollama` | native Ollama API | | `http` | arbitrary HTTP endpoint (custom adapter) | @@ -228,10 +230,12 @@ cargo test # config round-trip, routing validation ## Status -🚧 Implemented so far: OpenAI Chat Completions, routing with fallback & graceful -degradation, cost/token accounting, the **embedded multilingual admin console with -hot config reload**, statistics and Prometheus `GET /metrics`. Planned: streaming -(SSE), Anthropic provider/inbound, embeddings, MCP. See the roadmap in +Implemented: OpenAI Chat Completions **with SSE streaming**, **Anthropic** provider + +inbound `/v1/messages` (streaming), **embeddings**, a **semantic cache**, **MCP** server, +routing with fallback & graceful degradation, cost/token accounting, the **embedded +multilingual admin console with hot config reload**, statistics and Prometheus +`GET /metrics`. Planned: OpenAI Completions/Models, native passthrough, per-account +routing, feedback loop. See the roadmap in [docs/ARCHITECTURE.md](https://github.com/infosave2007/cortiq-gateway/blob/master/docs/ARCHITECTURE.md). Contributions welcome — see [CONTRIBUTING.md](https://github.com/infosave2007/cortiq-gateway/blob/master/CONTRIBUTING.md). diff --git a/README.ru.md b/README.ru.md index 082ef86..b9a71a1 100644 --- a/README.ru.md +++ b/README.ru.md @@ -141,12 +141,12 @@ print(resp.choices[0].message.content) | Протокол | Endpoint | Статус | |---|---|---| -| OpenAI Chat Completions | `POST /v1/chat/completions` | ✅ реализовано | +| OpenAI Chat Completions | `POST /v1/chat/completions` | ✅ реализовано (+ стриминг) | +| OpenAI Embeddings | `POST /v1/embeddings` | ✅ реализовано | +| Anthropic Messages | `POST /v1/messages` | ✅ реализовано (+ стриминг) | +| MCP (Model Context Protocol) | `POST /mcp` (JSON-RPC) | ✅ реализовано | | OpenAI Completions (legacy) | `POST /v1/completions` | план | -| OpenAI Embeddings | `POST /v1/embeddings` | план | | OpenAI Models | `GET /v1/models` | план | -| Anthropic Messages | `POST /v1/messages` | план | -| MCP (Model Context Protocol) | JSON-RPC / SSE | план | | Native passthrough | `POST /route` | план | ## Поддерживаемые провайдеры (исходящие) @@ -154,7 +154,7 @@ print(resp.choices[0].message.content) | Провайдер | Покрывает | |---|---| | `openai` (OpenAI-совместимый) | OpenAI, OpenRouter, Together, Groq, **vLLM, llama.cpp server, LM Studio, Ollama (`/v1`)** | -| `anthropic` | Claude Messages API (план) | +| `anthropic` | Claude Messages API (+ стриминг) | | `ollama` | нативный Ollama API | | `http` | произвольный HTTP-эндпоинт (свой адаптер) | @@ -179,10 +179,12 @@ cargo test # round-trip конфига, валидация ро ## Статус -🚧 Уже работает: OpenAI Chat Completions, роутинг с fallback и мягкой деградацией, -учёт стоимости/токенов, **встроенная многоязычная панель управления с горячей -перезагрузкой конфига**, статистика и Prometheus `GET /metrics`. В планах: стриминг -(SSE), провайдер/вход Anthropic, embeddings, MCP. Дорожная карта — +Реализовано: OpenAI Chat Completions **со стримингом (SSE)**, провайдер и вход +**Anthropic** `/v1/messages` (стриминг), **embeddings**, **семантический кэш**, +**MCP**-сервер, роутинг с fallback и мягкой деградацией, учёт стоимости/токенов, +**встроенная многоязычная панель управления с горячей перезагрузкой конфига**, +статистика и Prometheus `GET /metrics`. В планах: OpenAI Completions/Models, native +passthrough, per-account роутинг, feedback-loop. Дорожная карта — [docs/ARCHITECTURE.ru.md](docs/ARCHITECTURE.ru.md). Контрибьюции приветствуются — см. [CONTRIBUTING.ru.md](CONTRIBUTING.ru.md). diff --git a/src/admin/mod.rs b/src/admin/mod.rs index f1ca96c..5c79fc1 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -281,7 +281,7 @@ async fn meta() -> ApiResult { "caps": ["tools", "vision"], "protocols_impl": { "openai_chat": true, "openai_completions": false, "openai_embeddings": true, - "openai_models": false, "anthropic_messages": true, "mcp": false, "native_passthrough": false + "openai_models": false, "anthropic_messages": true, "mcp": true, "native_passthrough": false }, "languages": ["en", "ru", "de", "fr", "es", "zh", "tr"], })) diff --git a/src/protocols/mcp.rs b/src/protocols/mcp.rs new file mode 100644 index 0000000..e2e1284 --- /dev/null +++ b/src/protocols/mcp.rs @@ -0,0 +1,162 @@ +//! MCP (Model Context Protocol) server: `POST /mcp` (JSON-RPC 2.0). +//! +//! Exposes the gateway's routing to MCP-native orchestrators as two tools: +//! `cortiq_route` (classify a prompt → routing decision) and `cortiq_chat` +//! (run a prompt through intelligent routing → answer). Single JSON endpoint; +//! notifications are accepted with `202`. Gated by `protocols.mcp`. + +use crate::model::{ChatRequest, GenParams, Message, RequestMeta, RoutingDirective}; +use crate::state::SharedState; +use axum::response::{IntoResponse, Response}; +use axum::{extract::State, http::StatusCode, routing::post, Json, Router}; +use serde_json::{json, Value}; + +pub fn routes() -> Router { + Router::new().route("/mcp", post(handler)) +} + +fn parse_routing(model: &str) -> RoutingDirective { + if let Some(rest) = model.strip_prefix("cortiq-auto") { + RoutingDirective::Auto { + profile: rest.strip_prefix(':').map(|p| p.to_string()), + } + } else { + RoutingDirective::Pinned { + model_id: model.to_string(), + } + } +} + +fn tools_list() -> Value { + json!([ + { + "name": "cortiq_route", + "description": "Classify a prompt and return the routing decision (task type, complexity tier, score, and the model the gateway would select).", + "inputSchema": { + "type": "object", + "properties": { "text": { "type": "string", "description": "The prompt to classify." } }, + "required": ["text"] + } + }, + { + "name": "cortiq_chat", + "description": "Run a prompt through the gateway's intelligent routing and return the selected model's answer.", + "inputSchema": { + "type": "object", + "properties": { + "prompt": { "type": "string", "description": "The user prompt." }, + "model": { "type": "string", "description": "Optional: cortiq-auto[:profile] or a model id (default cortiq-auto)." } + }, + "required": ["prompt"] + } + } + ]) +} + +fn tool_text(text: String, is_error: bool) -> Value { + json!({ "content": [{ "type": "text", "text": text }], "isError": is_error }) +} + +async fn handle_tool_call( + state: &SharedState, + params: &Value, +) -> std::result::Result { + let name = params["name"].as_str().unwrap_or(""); + let args = ¶ms["arguments"]; + let live = state.live(); + match name { + "cortiq_route" => { + let text = args["text"].as_str().unwrap_or(""); + let prof = live.cfg.route.profile.clone(); + match live.router.route(text, &prof).await { + Ok(Some(d)) => Ok(tool_text( + format!( + "task_label={}\ncomplexity_tier={}\ncomplexity_score={:.3}\ncandidates={:?}", + d.task_label, + d.complexity_tier, + d.complexity_score, + live.routing.candidates(&d.complexity_tier) + ), + false, + )), + _ => Ok(tool_text( + "router unavailable — would use the default model".to_string(), + false, + )), + } + } + "cortiq_chat" => { + let prompt = args["prompt"].as_str().unwrap_or(""); + let model = args["model"].as_str().unwrap_or("cortiq-auto"); + let req = ChatRequest { + routing: parse_routing(model), + messages: vec![Message { + role: "user".into(), + content: prompt.to_string(), + tool_calls: vec![], + }], + tools: vec![], + params: GenParams::default(), + stream: false, + meta: RequestMeta { + account: "mcp".into(), + protocol: "mcp".into(), + ..Default::default() + }, + }; + match state.pipeline.run(req, state).await { + Ok(resp) => { + let answer = resp + .choices + .first() + .map(|c| c.message.content.clone()) + .unwrap_or_default(); + Ok(tool_text(answer, false)) + } + Err(e) => Ok(tool_text(e.to_string(), true)), + } + } + _ => Err((-32602, format!("unknown tool '{name}'"))), + } +} + +async fn handler(State(state): State, Json(req): Json) -> Response { + if !state.live().cfg.protocols.mcp { + return (StatusCode::NOT_FOUND, "mcp protocol is disabled").into_response(); + } + + let method = req["method"].as_str().unwrap_or("").to_string(); + let id = req.get("id").cloned(); + + // JSON-RPC notifications carry no id and expect no response + if id.is_none() || method.starts_with("notifications/") { + return StatusCode::ACCEPTED.into_response(); + } + let id = id.unwrap_or(Value::Null); + + let result: std::result::Result = match method.as_str() { + "initialize" => { + let pv = req["params"]["protocolVersion"] + .as_str() + .unwrap_or("2024-11-05") + .to_string(); + Ok(json!({ + "protocolVersion": pv, + "capabilities": { "tools": {} }, + "serverInfo": { "name": "cortiq-gateway", "version": env!("CARGO_PKG_VERSION") } + })) + } + "ping" => Ok(json!({})), + "tools/list" => Ok(json!({ "tools": tools_list() })), + "tools/call" => handle_tool_call(&state, &req["params"]).await, + other => Err((-32601, format!("method not found: {other}"))), + }; + + let body = match result { + Ok(r) => json!({ "jsonrpc": "2.0", "id": id, "result": r }), + Err((code, message)) => { + json!({ "jsonrpc": "2.0", "id": id, "error": { "code": code, "message": message } }) + } + }; + Json(body).into_response() +} diff --git a/src/protocols/mod.rs b/src/protocols/mod.rs index d95b26c..8794ae6 100644 --- a/src/protocols/mod.rs +++ b/src/protocols/mod.rs @@ -2,6 +2,7 @@ //! its protocol to/from the canonical model. Enabled via flags in `[protocols]`. pub mod anthropic_messages; +pub mod mcp; pub mod openai_chat; pub mod openai_embeddings; @@ -16,5 +17,6 @@ pub fn build_router() -> Router { .merge(openai_chat::routes()) .merge(openai_embeddings::routes()) .merge(anthropic_messages::routes()) - // TODO: openai_completions, openai_models, mcp, native_passthrough. + .merge(mcp::routes()) + // TODO: openai_completions, openai_models, native_passthrough. }