From 1715ae7c42a80ff6c6a786279b0490740ea4f700 Mon Sep 17 00:00:00 2001 From: Jose Daniel Date: Fri, 16 Oct 2026 02:20:43 -0500 Subject: [PATCH 1/2] feat(mcp): observer-mcp, a read-only MCP server for investigating traces Eleven read-only tools over the HTTP API (overview, traces, failed traces with reasons, one trace with spans/tools/provider chain/scores, alerts, datasets, runs, comparison, metrics query). Trace and span input/output are never returned unless OBSERVER_MCP_CONTENT=1. A test fails if the reader can use anything but GET or the metrics query. Optional extra 'mcp'. Roadmap step 6 done. --- backend/app/ops/mcp_server.py | 322 +++++++++++++++++++++++++++++++ backend/pyproject.toml | 5 + backend/tests/test_mcp_server.py | 221 +++++++++++++++++++++ docs/mcp.md | 47 +++++ 4 files changed, 595 insertions(+) create mode 100644 backend/app/ops/mcp_server.py create mode 100644 backend/tests/test_mcp_server.py create mode 100644 docs/mcp.md diff --git a/backend/app/ops/mcp_server.py b/backend/app/ops/mcp_server.py new file mode 100644 index 0000000..cf615e1 --- /dev/null +++ b/backend/app/ops/mcp_server.py @@ -0,0 +1,322 @@ +"""Read-only MCP server: let an assistant (Claude, Codex) investigate Observer without screenshots. + +``observer-mcp`` speaks MCP over stdio and talks to a running Observer through its HTTP API. It can +only read: every call is a ``GET`` or the metrics query (a ``POST`` that computes, never writes), +and the paths are fixed here. Trace and span **input/output are never returned** unless +``OBSERVER_MCP_CONTENT=1`` is set; what an assistant gets is names, types, status, durations, +attributes, scores and the provider chain: enough to diagnose a failure. + +Configure with ``OBSERVER_URL`` (default http://127.0.0.1:8100) and ``OBSERVATORY_API_KEY``. +""" + +from __future__ import annotations + +import argparse +import os +from collections.abc import Mapping +from datetime import datetime +from typing import Any + +import httpx +from mcp.server.fastmcp import FastMCP +from mcp.types import ToolAnnotations + +DEFAULT_URL = "http://127.0.0.1:8100" +MAX_SPANS = 200 +MAX_ROWS = 100 +_READ_ONLY = ToolAnnotations(readOnlyHint=True, destructiveHint=False, openWorldHint=False) + + +class ObserverReadError(RuntimeError): + """Observer answered with an error or could not be reached (the message is safe to show).""" + + +def content_enabled() -> bool: + return os.getenv("OBSERVER_MCP_CONTENT", "").strip().lower() in {"1", "true", "yes", "on"} + + +class ObserverReader: + """HTTP access limited to reading. There is no method that writes.""" + + def __init__( + self, + url: str = DEFAULT_URL, + api_key: str = "", + *, + transport: httpx.AsyncBaseTransport | None = None, + timeout: float = 30.0, + ) -> None: + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + self._client = httpx.AsyncClient( + base_url=url.rstrip("/"), + headers=headers, + timeout=timeout, + trust_env=False, + transport=transport, + ) + + async def get(self, path: str, **params: Any) -> Any: + clean = {k: v for k, v in params.items() if v is not None} + return await self._send("GET", path, params=clean or None) + + async def query(self, body: Mapping[str, Any]) -> Any: + """``POST /v1/analytics/query``: computes aggregates, changes nothing.""" + return await self._send("POST", "/v1/analytics/query", json=dict(body)) + + async def _send(self, method: str, path: str, **kwargs: Any) -> Any: + try: + response = await self._client.request(method, path, **kwargs) + except httpx.HTTPError as exc: + raise ObserverReadError(f"Observer unreachable ({exc.__class__.__name__})") from exc + if response.status_code >= 400: + raise ObserverReadError(f"{method} {path} answered {response.status_code}") + return response.json() + + async def aclose(self) -> None: + await self._client.aclose() + + +# ── shaping: compact, content-free views ────────────────────────────────────── + + +def _ms(start: str | None, end: str | None) -> float | None: + if not start or not end: + return None + try: + delta = datetime.fromisoformat(end) - datetime.fromisoformat(start) + except ValueError: + return None + return round(delta.total_seconds() * 1000.0, 1) + + +def trace_row(trace: Mapping[str, Any]) -> dict[str, Any]: + return { + "id": trace.get("id"), + "name": trace.get("name"), + "status": trace.get("status"), + "project_id": trace.get("project_id"), + "environment": trace.get("environment"), + "created_at": trace.get("created_at"), + "duration_ms": _ms(trace.get("start_time"), trace.get("end_time")), + } + + +def span_view(span: Mapping[str, Any], *, content: bool) -> dict[str, Any]: + out = { + "id": span.get("id"), + "parent_id": span.get("parent_id"), + "name": span.get("name"), + "type": span.get("span_type"), + "status": span.get("status"), + "duration_ms": _ms(span.get("start_time"), span.get("end_time")), + "attributes": span.get("attributes") or {}, + "tokens_input": span.get("tokens_input"), + "tokens_output": span.get("tokens_output"), + "cost_usd": span.get("cost_usd"), + } + if content: + out["input"], out["output"] = span.get("input"), span.get("output") + return out + + +def evaluation_view(ev: Mapping[str, Any]) -> dict[str, Any]: + result = ev.get("result") if isinstance(ev.get("result"), dict) else {} + return { + "type": ev.get("evaluator_type"), + "score": ev.get("score"), + "grade": result.get("grade"), + "passed": result.get("passed"), + "health": result.get("health"), + "correctness": result.get("correctness"), + "criteria": [ + {k: c.get(k) for k in ("name", "score", "detail")} + for c in result.get("criteria", []) + if isinstance(c, dict) + ], + } + + +def trace_detail( + trace: Mapping[str, Any], + spans: list[Mapping[str, Any]], + evaluations: list[Mapping[str, Any]], + *, + content: bool, +) -> dict[str, Any]: + views = [span_view(s, content=content) for s in spans[:MAX_SPANS]] + chain = next((s for s in views if s["name"] == "llm.chain"), None) + tools = [ + {"name": s["name"], "status": s["status"], "duration_ms": s["duration_ms"]} + for s in views + if s["type"] == "tool" + ] + detail = trace_row(trace) + detail.update( + { + "session_id": trace.get("session_id"), + "metadata": trace.get("metadata") or {}, + "provider_chain": (chain or {}).get("attributes", {}).get("attempts_text"), + "answered_by": (chain or {}).get("attributes", {}).get("answered_by"), + "tools": tools, + "evaluations": [evaluation_view(e) for e in evaluations], + "spans": views, + "spans_truncated": len(spans) > MAX_SPANS, + "content_included": content, + } + ) + return detail + + +# ── the server ──────────────────────────────────────────────────────────────── + + +def build_server(reader: ObserverReader) -> FastMCP: + mcp = FastMCP( + "observer", + instructions=( + "Read-only access to an Observer instance: traces of an AI agent, health and quality " + "scores, provider fallbacks, alerts, test-case datasets and version comparisons. Start " + "with observer_overview, then list_failed_traces, then get_trace for one failure." + ), + ) + + def tool(fn): + return mcp.tool(annotations=_READ_ONLY)(fn) + + @tool + async def observer_overview(hours: int = 24, project_id: str = "manitos") -> dict[str, Any]: + """Health at a glance: turn quality over the last `hours` and the open alerts.""" + quality = await reader.get( + "/v1/analytics/manitos-quality", hours=hours, project_id=project_id + ) + alerts = await reader.get("/v1/alerts/events", state="open", limit=20) + return {"quality": quality, "open_alerts": alerts} + + @tool + async def list_traces( + status: str | None = None, + environment: str | None = None, + project_id: str | None = None, + search: str | None = None, + limit: int = 20, + ) -> dict[str, Any]: + """Recent traces, newest first (status: ok | error). Metadata only.""" + data = await reader.get( + "/v1/traces/", + status=status, + environment=environment, + project_id=project_id, + search=search, + limit=max(1, min(limit, MAX_ROWS)), + ) + return {"traces": [trace_row(t) for t in data.get("traces", [])]} + + @tool + async def list_failed_traces( + limit: int = 20, health_below: float = 0.6, project_id: str = "manitos" + ) -> dict[str, Any]: + """Traces that failed: status=error, plus turns whose operational-health score fell below + `health_below` (rule-based evaluation). Each row says why.""" + limit = max(1, min(limit, MAX_ROWS)) + errors = await reader.get("/v1/traces/", status="error", project_id=project_id, limit=limit) + low = await reader.get( + "/v1/evaluations/", + evaluator_type="rule_based", + max_score=health_below, + limit=limit, + ) + rows: dict[str, dict[str, Any]] = {} + for trace in errors.get("traces", []): + rows[trace["id"]] = {**trace_row(trace), "reasons": ["trace status is error"]} + for ev in low.get("evaluations", []): + row = rows.setdefault(ev["trace_id"], {"id": ev["trace_id"], "reasons": []}) + view = evaluation_view(ev) + row["reasons"].append(f"health score {view['score']} (grade {view['grade']})") + row["health_score"] = view["score"] + return {"count": len(rows), "traces": list(rows.values())[:limit]} + + @tool + async def get_trace(trace_id: str) -> dict[str, Any]: + """One trace in full detail without its text: spans (name, type, status, duration, + attributes), tools called, the provider chain, and its scores. Input/output are omitted + unless the server runs with OBSERVER_MCP_CONTENT=1.""" + trace = await reader.get(f"/v1/traces/{trace_id}") + spans = await reader.get(f"/v1/traces/{trace_id}/spans") + evaluations = await reader.get(f"/v1/traces/{trace_id}/evaluations") + return trace_detail(trace, spans, evaluations, content=content_enabled()) + + @tool + async def list_alerts(state: str | None = None) -> dict[str, Any]: + """Alert events (state: open | resolved) and how many are open.""" + return await reader.get("/v1/alerts/events", state=state, limit=50) + + @tool + async def list_datasets() -> dict[str, Any]: + """Test-case datasets with their case counts.""" + return {"datasets": await reader.get("/v1/datasets")} + + @tool + async def get_dataset(dataset_id: str) -> dict[str, Any]: + """A dataset with its cases: what was observed and what each case expects.""" + return await reader.get(f"/v1/datasets/{dataset_id}") + + @tool + async def list_runs(dataset_id: str) -> dict[str, Any]: + """Runs of a dataset (one per agent version), newest first, with pass/fail/unknown.""" + return {"runs": await reader.get(f"/v1/datasets/{dataset_id}/runs")} + + @tool + async def compare_runs( + dataset_id: str, base_run_id: str, candidate_run_id: str + ) -> dict[str, Any]: + """Case-by-case difference between two runs: regressed, improved, same, unknown, missing, + with health and duration deltas.""" + return await reader.get( + f"/v1/datasets/{dataset_id}/compare", base=base_run_id, candidate=candidate_run_id + ) + + @tool + async def metrics_schema() -> dict[str, Any]: + """Views, measures, dimensions and aggregations that query_metrics accepts.""" + return await reader.get("/v1/analytics/query/schema") + + @tool + async def query_metrics( + view: str, + metrics: list[dict[str, str]], + dimensions: list[str] | None = None, + filters: list[dict[str, Any]] | None = None, + hours: int = 24, + granularity: str | None = None, + row_limit: int = 50, + ) -> dict[str, Any]: + """Aggregate metrics (see metrics_schema), e.g. view="turns", metrics=[{"measure": + "duration_ms", "aggregation": "p95"}], dimensions=["model"]. Read-only.""" + body: dict[str, Any] = { + "view": view, + "metrics": metrics, + "dimensions": dimensions or [], + "filters": filters or [], + "hours": hours, + "row_limit": max(1, min(row_limit, MAX_ROWS)), + } + if granularity: + body["granularity"] = granularity + return await reader.query(body) + + return mcp + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="observer-mcp", description="Read-only MCP server for an Observer instance (stdio)." + ) + parser.add_argument("--url", default=os.getenv("OBSERVER_URL", DEFAULT_URL)) + parser.add_argument("--api-key", default=os.getenv("OBSERVATORY_API_KEY", "")) + args = parser.parse_args(argv) + build_server(ObserverReader(args.url, args.api_key)).run(transport="stdio") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index f8c8c86..cafde26 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -32,17 +32,22 @@ Issues = "https://github.com/Magic-Lab-Studio/observer/issues" observer-integration-passive-gate = "app.ops.manitos_passive_gate:main" observer-manitos-passive-gate = "app.ops.manitos_passive_gate:main" observer-regression = "app.ops.regression_gate:main" +observer-mcp = "app.ops.mcp_server:main" [project.optional-dependencies] sqlite = [ "aiosqlite>=0.20.0", ] +mcp = [ + "mcp>=1.2,<2", +] dev = [ "pytest>=8.0", "pytest-asyncio>=0.23.0", "ruff>=0.5", "mypy>=1.10", "aiosqlite>=0.20.0", + "mcp>=1.2,<2", ] [tool.hatch.build.targets.wheel] diff --git a/backend/tests/test_mcp_server.py b/backend/tests/test_mcp_server.py new file mode 100644 index 0000000..ea95231 --- /dev/null +++ b/backend/tests/test_mcp_server.py @@ -0,0 +1,221 @@ +import json +import re +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest +from httpx import ASGITransport +from mcp.shared.memory import create_connected_server_and_client_session + +from app.main import app +from app.models import Evaluation, Span, Trace +from app.ops import mcp_server +from app.ops.mcp_server import ObserverReader, ObserverReadError, build_server, trace_detail + + +async def _trace(db, *, status="ok", degraded=False, ms=1500): + created = datetime.now(timezone.utc) - timedelta(minutes=5) + trace = Trace( + name="manitos.turn", + start_time=created, + end_time=created + timedelta(milliseconds=ms), + status=status, + project_id="manitos", + environment="test", + created_at=created, + ) + db.add(trace) + await db.flush() + db.add( + Span( + trace_id=trace.id, + name="manitos.turn.lifecycle", + span_type="turn", + start_time=created, + end_time=created + timedelta(milliseconds=ms), + status=status, + inp={"text": "PRIVATE QUESTION"}, + out={"text": "PRIVATE ANSWER"}, + attributes={"duration_ms": ms, "llm_degraded": degraded}, + ) + ) + db.add( + Span( + trace_id=trace.id, + name="llm.chain", + span_type="event", + start_time=created, + status="ok", + attributes={"answered_by": "groq · qwen", "attempts_text": "groq:q=429>gemini:f=ok"}, + ) + ) + db.add( + Span( + trace_id=trace.id, name="web_search", span_type="tool", start_time=created, status="ok" + ) + ) + await db.commit() + return trace + + +async def _session(client_ready): + reader = ObserverReader("http://test", transport=ASGITransport(app=app)) + return build_server(reader), reader + + +async def _call(session, name, **args): + result = await session.call_tool(name, args) + assert not result.isError, result.content + (part,) = [json.loads(c.text) for c in result.content if hasattr(c, "text")] + return part + + +@pytest.mark.anyio +async def test_every_tool_is_declared_read_only(client): + server, _ = await _session(client) + async with create_connected_server_and_client_session(server._mcp_server) as session: + tools = (await session.list_tools()).tools + names = {t.name for t in tools} + assert { + "observer_overview", + "list_failed_traces", + "get_trace", + "compare_runs", + "query_metrics", + } <= names + assert all(t.annotations and t.annotations.readOnlyHint for t in tools) + assert not any(re.search(r"delete|create|update|write|save|ack|remove", n) for n in names) + + +@pytest.mark.anyio +async def test_get_trace_omits_content_and_shows_chain_tools_and_scores(client, db_session): + trace = await _trace(db_session) + db_session.add( + Evaluation( + trace_id=trace.id, + evaluator_type="rule_based", + score=0.89, + criteria={}, + result={ + "grade": "B", + "passed": True, + "health": {"score": 1.0, "grade": "A"}, + "correctness": {"status": "unknown", "score": None}, + "criteria": [{"name": "latency", "score": 1.0, "detail": "1500 ms"}], + }, + ) + ) + await db_session.commit() + server, _ = await _session(client) + async with create_connected_server_and_client_session(server._mcp_server) as session: + detail = await _call(session, "get_trace", trace_id=trace.id) + assert "PRIVATE" not in json.dumps(detail) and detail["content_included"] is False + assert detail["provider_chain"] == "groq:q=429>gemini:f=ok" + assert detail["answered_by"] == "groq · qwen" + assert [t["name"] for t in detail["tools"]] == ["web_search"] + ev = detail["evaluations"][0] + assert ev["grade"] == "B" and ev["correctness"]["status"] == "unknown" + assert ev["criteria"][0]["name"] == "latency" + + +def test_content_only_with_the_explicit_switch(monkeypatch): + trace = {"id": "t", "name": "n", "status": "ok"} + spans = [{"id": "s", "name": "x", "span_type": "turn", "input": {"a": "SECRET"}, "output": {}}] + assert "SECRET" not in json.dumps(trace_detail(trace, spans, [], content=False)) + assert "SECRET" in json.dumps(trace_detail(trace, spans, [], content=True)) + monkeypatch.setenv("OBSERVER_MCP_CONTENT", "1") + assert mcp_server.content_enabled() + monkeypatch.delenv("OBSERVER_MCP_CONTENT") + assert not mcp_server.content_enabled() + + +@pytest.mark.anyio +async def test_failed_traces_combine_errors_and_low_health(client, db_session): + bad = await _trace(db_session, status="error") + slow = await _trace(db_session) + await _trace(db_session) # healthy: must not appear + db_session.add( + Evaluation( + trace_id=slow.id, + evaluator_type="rule_based", + score=0.3, + criteria={}, + result={"grade": "D", "passed": False}, + ) + ) + await db_session.commit() + server, _ = await _session(client) + async with create_connected_server_and_client_session(server._mcp_server) as session: + out = await _call(session, "list_failed_traces", limit=10) + ids = {t["id"]: t for t in out["traces"]} + assert set(ids) == {bad.id, slow.id} and out["count"] == 2 + assert ids[bad.id]["reasons"] == ["trace status is error"] + assert "health score 0.3" in ids[slow.id]["reasons"][0] + + +@pytest.mark.anyio +async def test_datasets_runs_comparison_and_metrics_are_readable(client, db_session): + source = await _trace(db_session) + ds = (await client.post("/v1/datasets", json={"name": "d"})).json()["id"] + item = ( + await client.post( + f"/v1/datasets/{ds}/items", + json={"trace_id": source.id, "expect": {"health": {"no_degradation": True}}}, + ) + ).json()["id"] + good, bad = await _trace(db_session), await _trace(db_session, degraded=True) + runs = [] + for trace in (good, bad): + runs.append( + ( + await client.post( + f"/v1/datasets/{ds}/runs", + json={"results": [{"item_id": item, "trace_id": trace.id}]}, + ) + ).json()["id"] + ) + server, _ = await _session(client) + async with create_connected_server_and_client_session(server._mcp_server) as session: + assert (await _call(session, "list_datasets"))["datasets"][0]["items"] == 1 + assert len((await _call(session, "get_dataset", dataset_id=ds))["items"]) == 1 + assert len((await _call(session, "list_runs", dataset_id=ds))["runs"]) == 2 + diff = await _call( + session, "compare_runs", dataset_id=ds, base_run_id=runs[0], candidate_run_id=runs[1] + ) + assert diff["summary"]["regressed"] == 1 + schema = await _call(session, "metrics_schema") + assert "turns" in schema["views"] + rows = await _call( + session, + "query_metrics", + view="turns", + metrics=[{"measure": "count", "aggregation": "count"}], + hours=6, + ) + assert rows["samples"] >= 1 + overview = await _call(session, "observer_overview", hours=6) + assert "quality" in overview and "open_alerts" in overview + listed = await _call(session, "list_traces", status="error", limit=5) + assert listed == {"traces": []} + + +@pytest.mark.anyio +async def test_errors_are_reported_cleanly(client): + server, _ = await _session(client) + async with create_connected_server_and_client_session(server._mcp_server) as session: + result = await session.call_tool("get_trace", {"trace_id": "not-a-uuid"}) + assert result.isError and "400" in result.content[0].text + down = ObserverReader("http://127.0.0.1:1") + with pytest.raises(ObserverReadError, match="unreachable"): + await down.get("/v1/datasets") + await down.aclose() + + +def test_the_reader_can_only_read(): + source = Path(mcp_server.__file__).read_text() + assert not re.search(r"\.(put|patch|delete)\(", source) + methods = set(re.findall(r'_send\(\s*"(\w+)"', source)) + assert methods == {"GET", "POST"} + # the single POST is the metrics query, which computes and writes nothing + assert source.count('"/v1/analytics/query"') == 1 + assert not [m for m in dir(ObserverReader) if re.match(r"(delete|create|update|post|put)", m)] diff --git a/docs/mcp.md b/docs/mcp.md new file mode 100644 index 0000000..1467d74 --- /dev/null +++ b/docs/mcp.md @@ -0,0 +1,47 @@ +# Read-only MCP server + +`observer-mcp` lets an assistant (Claude Code, Codex, any MCP client) investigate an Observer +instance without screenshots or copy-paste. It speaks MCP over stdio and reads from a running +Observer through its HTTP API. + +```bash +pip install -e "./backend[mcp]" # the `mcp` package is an optional extra +claude mcp add observer \ + -e OBSERVER_URL=http://127.0.0.1:8100 -e OBSERVATORY_API_KEY= \ + -- /path/to/backend/.venv/bin/observer-mcp +``` + +Any other client: run `observer-mcp` (or `python -m app.ops.mcp_server`) as a stdio server with +`OBSERVER_URL` (default `http://127.0.0.1:8100`) and, when the backend requires one, +`OBSERVATORY_API_KEY`. + +## What it can do + +Every tool is read-only (declared with `readOnlyHint`), and there is no tool that writes. + +| Tool | Answers | +|---|---| +| `observer_overview` | Turn quality over the last N hours and the open alerts | +| `list_traces` | Recent traces (filter by status, environment, project, search) | +| `list_failed_traces` | Traces with status `error` plus turns whose health score fell below a threshold, each with the reason | +| `get_trace` | One trace: spans (name, type, status, duration, attributes), tools called, the provider chain, answering model, scores | +| `list_alerts` | Alert events, open or resolved | +| `list_datasets`, `get_dataset` | Test-case datasets and their cases | +| `list_runs`, `compare_runs` | Runs per agent version and the case-by-case difference between two | +| `metrics_schema`, `query_metrics` | Flexible aggregates (view, measures, dimensions, filters) | + +A good session: `observer_overview` → `list_failed_traces` → `get_trace` on one → `compare_runs` +to see whether a fix helped. + +## Privacy and safety + +* **No content by default.** Trace and span `input`/`output` are never returned. An assistant gets + names, types, status, durations, attributes, scores and the provider chain, which is enough to + diagnose most failures. Set `OBSERVER_MCP_CONTENT=1` on the server only if you accept that the + assistant sees stored prompts and replies. +* **Read-only by construction.** The server can issue `GET` requests and the metrics query + (`POST /v1/analytics/query`, which computes and writes nothing); the paths are fixed in code and a + test fails if any other method or a write path appears. +* **Local transport.** stdio only: nothing listens on a port. It still talks to Observer over HTTP, + so the usual API key applies. +* Lists are capped (100 rows, 200 spans per trace). From ce578a6446f9836f53f45c58596fd8953cdc61fe Mon Sep 17 00:00:00 2001 From: Jose Daniel Date: Fri, 16 Oct 2026 02:21:07 -0500 Subject: [PATCH 2/2] docs: README and roadmap for observer-mcp (step 6) --- README.md | 7 +++++++ docs/roadmap.md | 10 +++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9186949..ba1c9db 100644 --- a/README.md +++ b/README.md @@ -280,6 +280,13 @@ Scores of the previous rubric are replaced when the turn is re-scored. A backgro backfills history. Evaluations are idempotent (the id derives from the trace and the rubric). The Evaluations page shows health by model, both axes, the grade and the per-criterion breakdown. +### Read-only MCP server + +`observer-mcp` lets Claude Code, Codex or any MCP client investigate Observer without screenshots: +failed traces with reasons, one trace's spans, tools and provider chain, alerts, datasets, version +comparisons and metric queries. Read-only, and trace input/output are never returned unless +`OBSERVER_MCP_CONTENT=1`. Setup and tools: [docs/mcp.md](docs/mcp.md). + ### Test cases (datasets) On a trace page, **Save as case** stores a failure as a reusable test case in a dataset. A case is diff --git a/docs/roadmap.md b/docs/roadmap.md index 885bf2b..83f482c 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -27,7 +27,7 @@ and write down what was learned or changed. Keep entries short. | 3 | Compare runs by version | done | this PR | | 4 | Regression command for CI | done | this PR | | 5 | Standard OTLP ingestion | planned | | -| 6 | Read-only MCP server | planned | | +| 6 | Read-only MCP server | done | this PR | | 7 | Configurable capture privacy | later | | | 8 | Ingestion reliability (persistent queue, backoff, metrics) | later | | @@ -102,6 +102,11 @@ Today `OTLPExporter` posts Observer's own JSON to `/v1/traces/batch`; there is n Tools to list failed traces, compare runs and fetch evidence, so Claude or Codex can investigate without screenshots. Read-only first. +- **Built:** `observer-mcp` (`app/ops/mcp_server.py`, stdio, optional extra `mcp`): 11 read-only + tools over the HTTP API. Content (`input`/`output`) is never returned unless + `OBSERVER_MCP_CONTENT=1`. Tools, setup and privacy: `docs/mcp.md`. A test fails if the reader can + use any method other than `GET` or the metrics query. + ## Later - **Capture privacy modes per project** (metadata only, redacted, full), filtered before export. @@ -138,3 +143,6 @@ investigate without screenshots. Read-only first. export tool spans and reply text before the gate can verify them on real traces. - 2026-10-16: Manitos PR #36 merged: tool spans exported. Tool expectations are now verifiable on new traces; text phrases remain `unknown`. Observer services migrated and restarted. +- 2026-10-16: step 6 done. `observer-mcp`: 11 read-only tools, no content by default. Verified over + real stdio with an MCP client against a synthetic Observer (an OTLP-ingested trace with its tool + span was listed and read). Steps 1-6 complete; what remains is under "Later".