[ { "schema": "fitness-report/1", "server": { "url": "https://docs.mcp.cloudflare.com/mcp", "slug": "docs-mcp-cloudflare-com", "era": "modern", "negotiatedVersion": "2026-07-28", "serverInfo": { "name": "docs-ai-search", "version": "0.4.13" }, "instructions": null, "capabilities": { "prompts": { "listChanged": true }, "tools": { "listChanged": true } }, "transportShape": "json", "sessionful": false, "credentialContext": "anonymous", "discover": { "_meta": { "io.modelcontextprotocol/serverInfo": { "name": "docs-ai-search", "version": "0.4.13" } }, "ttlMs": 0, "cacheScope": "private", "supportedVersions": [ "2026-07-28" ], "capabilities": { "prompts": { "listChanged": true }, "tools": { "listChanged": true } }, "resultType": "complete" } }, "run": { "id": "docs-mcp-cloudflare-com-2026-08-19T20-56-16-949Z", "startedAt": "2026-08-19T20:56:16.949Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "cc8bb834df89e21d9158be1d172d66ea4ccb1969561fa58ec61a888cd76aa5f8", "taskBudget": 20000 }, "probes": { "specCurrency": "2026-07-28", "findings": [ { "id": "spec-currency", "pass": true, "detail": "negotiated 2026-07-28 (modern era, >= 2026-07-28)", "evidence": { "negotiatedVersion": "2026-07-28", "era": "modern" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-11-25 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "result": { "protocolVersion": "2025-11-25", "capabilities": { "tools": { "listChanged": true }, "prompts": { "listChanged": true } }, "serverInfo": { "name": "docs-ai-search", "version": "0.4.13" } }, "jsonrpc": "2.0", "id": "fitness-probe-bogus-version" }, "answeredVersion": "2025-11-25" } }, { "id": "header-mismatch-accepted", "pass": true, "detail": "rejected the mismatched Mcp-Name with HTTP 400 / JSON-RPC -32020, as SEP-2243 requires", "evidence": { "status": 400, "jsonRpcErrorCode": -32020, "headerName": "__fitness_report_mismatched_name__", "bodyName": "__fitness_report_probe_tool__", "envelope": { "jsonrpc": "2.0", "error": { "code": -32020, "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\"", "data": { "mismatch": { "header": "__fitness_report_mismatched_name__", "body": "the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\"" } } }, "id": "fitness-probe-header-mismatch" } } }, { "id": "server-discover-present", "pass": true, "detail": "server/discover answered with supportedVersions [2026-07-28]", "evidence": { "supportedVersions": [ "2026-07-28" ], "capabilities": { "prompts": { "listChanged": true }, "tools": { "listChanged": true } }, "hasInstructions": false } }, { "id": "cache-hints", "pass": true, "detail": "tools/list cache hints: ttlMs=0, cacheScope=private", "evidence": { "ttlMs": 0, "cacheScope": "private", "credentialContext": "anonymous" } }, { "id": "deprecation-surface", "pass": true, "detail": "no deleted or deprecated surface advertised", "evidence": { "deletedVocabulary": [], "notes": [], "toolCount": 2, "era": "modern" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power", "construct", "null_baseline" ], "records": [ { "gate": "protocol_hygiene", "ok": true, "costTier": "free", "reason": "ok", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [] } }, { "gate": "structural", "ok": false, "costTier": "free", "reason": "too_few_generated", "detail": { "nRequested": 3, "nGenerated": 2, "nHolding": 2, "admissionRate": 0.6666666666666666, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": false, "reason": "too_few_generated", "explain": "REJECT (too_few_generated): 2 admitted cases is below the minimum 8. Every case held, over a sample too small to mean anything." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": false, "costTier": "free", "reason": "below_minimum_suite_size", "detail": { "nTasks": 2, "minTasks": 8, "toolCount": 2, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 2, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } }, { "gate": "construct", "ok": true, "costTier": "paid", "verdict": { "outcome": "PASS", "k": 2, "n": 2, "threshold": 0.9, "alpha": 0.05, "pValue": 1 }, "reason": "ok", "detail": { "n": 2, "nIntended": 2, "rate": 1, "errors": 0, "errorRate": 0, "maxErrorRate": 0.05, "compromised": false, "reps": 1, "published": { "outcome": "EXTEND", "k": 2, "n": 2, "threshold": 0.9, "alpha": 0.05, "pValue": 1, "rawOutcome": "PASS", "downgraded": true, "wilsonLow": 0.34237195288961925, "wilsonHigh": 1, "requiredN": 78, "achievedPower": 0.03999999999999998, "detectableRate": 0.8, "reason": "downgraded_underpowered" }, "constructOracle": "A reference pass counts only when it both satisfied the check and landed a successful call on a tool the task expects. The reference agent is given the answer key, so a text check alone would pass against a server that returned nothing.", "note": "One rep per task in v0, for cost. evalgate runs three; the divergence is recorded here rather than hidden." } }, { "gate": "null_baseline", "ok": false, "costTier": "cheap", "reason": "noise_exceeds_signal", "detail": { "tNull": 1, "tAblate": 1, "killThreshold": 0.5, "outcome": "KILL", "halts": true, "ok": false, "reason": "noise_exceeds_signal", "rates": [ { "label": "no-tools", "k": 2, "n": 2, "rate": 1 }, { "label": "stubbed-empty", "k": 1, "n": 2, "rate": 0.5 }, { "label": "random-valid-args", "k": 1, "n": 2, "rate": 0.5 } ], "percentile": 95, "ratio": 0.5, "signalSource": "first-try success on the scored drive", "measuredBeforePaidTier": true, "decidedAfter": "drive", "explain": "KILL (noise_exceeds_signal): a null model passed 1.000 of the suite against the real agent's 1.000, at or above the 0.5 kill threshold of 0.500 (worst offender: no-tools at 1.000). The suite is measuring the model, not the server." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "structural" }, "outcome": "INSUFFICIENT_SURFACE", "traceLinks": { "mcp": "https://fitness-report.pages.dev/traces/docs-mcp-cloudflare-com-2026-08-19T20-56-16-949Z/mcp.jsonl", "agent": "https://fitness-report.pages.dev/traces/docs-mcp-cloudflare-com-2026-08-19T20-56-16-949Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitness-report.pages.dev%2Ftraces%2Fdocs-mcp-cloudflare-com-2026-08-19T20-56-16-949Z%2Fmcp.jsonl;https%3A%2F%2Ffitness-report.pages.dev%2Ftraces%2Fdocs-mcp-cloudflare-com-2026-08-19T20-56-16-949Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "docs-mcp-cloudflare-com", "kind": "mcp+llm", "startedAt": "2026-08-19T20:56:16.949Z", "endedAt": "2026-08-19T20:57:48.448Z", "durationMs": 91499, "command": [ "fitness-report", "https://docs.mcp.cloudflare.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 86, "meta": 2, "message": 28, "event": 21, "turn": 33, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 16, "inputTokens": 44518, "outputTokens": 1707, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.106106, "priced": true } ], "summary": { "models": 1, "assistantTurns": 16, "turns": 33, "echoedTurns": 0, "inputTokens": 44518, "outputTokens": 1707, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.106106, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "search_cloudflare_documentation", "calls": 5, "errors": 0, "p50Ms": 3897, "p95Ms": 4683, "origin": "mcp", "pending": 0 }, { "name": "migrate_pages_to_workers_guide", "calls": 3, "errors": 0, "p50Ms": 104, "p95Ms": 154, "origin": "mcp", "pending": 0 }, { "name": "__fitness_report_probe_tool__", "calls": 1, "errors": 1, "p50Ms": 177, "p95Ms": 177, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T20:56:17.258Z", "source": "tool", "name": "__fitness_report_probe_tool__", "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\" (code -32020)" } ], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "docs-mcp-cloudflare-com", "kind": "mcp", "startedAt": "2026-08-19T20:56:16.949Z", "endedAt": "2026-08-19T20:57:48.432Z", "durationMs": 91483, "command": [ "fitness-report", "https://docs.mcp.cloudflare.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 51, "meta": 1, "message": 28, "event": 21, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [ { "name": "search_cloudflare_documentation", "calls": 5, "errors": 0, "p50Ms": 3897, "p95Ms": 4683, "origin": "mcp", "pending": 0 }, { "name": "migrate_pages_to_workers_guide", "calls": 3, "errors": 0, "p50Ms": 104, "p95Ms": 154, "origin": "mcp", "pending": 0 }, { "name": "__fitness_report_probe_tool__", "calls": 1, "errors": 1, "p50Ms": 177, "p95Ms": 177, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T20:56:17.258Z", "source": "tool", "name": "__fitness_report_probe_tool__", "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\" (code -32020)" } ] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "docs-mcp-cloudflare-com", "kind": "llm", "startedAt": "2026-08-19T20:56:16.949Z", "endedAt": "2026-08-19T20:57:48.448Z", "durationMs": 91499, "command": [ "fitness-report", "https://docs.mcp.cloudflare.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 35, "meta": 1, "message": 0, "event": 0, "turn": 33, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 16, "inputTokens": 44518, "outputTokens": 1707, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.106106, "priced": true } ], "summary": { "models": 1, "assistantTurns": 16, "turns": 33, "echoedTurns": 0, "inputTokens": 44518, "outputTokens": 1707, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.106106, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "search_cloudflare_documentation", "calls": 6, "errors": 0, "p50Ms": 5260, "p95Ms": 7944, "origin": "turn", "pending": 0 }, { "name": "migrate_pages_to_workers_guide", "calls": 3, "errors": 0, "p50Ms": 3534, "p95Ms": 8205, "origin": "turn", "pending": 0 } ], "errors": [] } }, "toolsPlane": "mcp" }, "scoreNotes": [ "The drive ran for evidence only. A gate had already refused, so no score is published from it." ], "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs.", "Operator ran with --evidence-drive: the refusal stands and no score is published, but the drive ran anyway so the recording exists." ] }, { "schema": "fitness-report/1", "server": { "url": "https://mcp.docs.astro.build/mcp", "slug": "mcp-docs-astro-build", "era": "legacy", "negotiatedVersion": "2025-06-18", "serverInfo": { "name": "Astro Docs server", "version": "1.0.0" }, "instructions": null, "capabilities": { "logging": {}, "tools": { "listChanged": true } }, "transportShape": "sse", "sessionful": false, "credentialContext": "anonymous" }, "run": { "id": "mcp-docs-astro-build-2026-08-19T22-07-34-673Z", "startedAt": "2026-08-19T22:07:34.673Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "dc5bede2ad9a9752fa135defee2d5c3425ea32dacd9fe96502c7c3293b78f03f", "taskBudget": 20000 }, "probes": { "specCurrency": "2025-06-18", "findings": [ { "id": "spec-currency", "pass": false, "detail": "negotiated 2025-06-18 (legacy era; the current revision is 2026-07-28)", "evidence": { "negotiatedVersion": "2025-06-18", "era": "legacy" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-06-18 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "result": { "protocolVersion": "2025-06-18", "capabilities": { "logging": {}, "tools": { "listChanged": true } }, "serverInfo": { "name": "Astro Docs server", "version": "1.0.0" } }, "jsonrpc": "2.0", "id": "fitness-probe-bogus-version" }, "answeredVersion": "2025-06-18" } }, { "id": "header-mismatch-accepted", "pass": null, "detail": "could not check: SEP-2243 header conformance is defined on the modern era only; this connection negotiated 2025-06-18 (legacy)" }, { "id": "server-discover-present", "pass": null, "detail": "could not check: server/discover is a 2026-07-28 MUST; this connection negotiated 2025-06-18 (legacy)" }, { "id": "cache-hints", "pass": null, "detail": "could not check: ttlMs/cacheScope are 2026-07-28 fields; this connection negotiated 2025-06-18 (legacy)" }, { "id": "deprecation-surface", "pass": null, "detail": "could not check: deleted-vocabulary checks are defined against 2026-07-28; this connection negotiated 2025-06-18", "evidence": { "deletedVocabulary": [], "notes": [], "toolCount": 1, "era": "legacy" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power", "construct", "null_baseline" ], "records": [ { "gate": "protocol_hygiene", "ok": false, "costTier": "free", "reason": "hygiene_findings_present", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [ "spec-currency" ] } }, { "gate": "structural", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nRequested": 12, "nGenerated": 12, "nHolding": 12, "admissionRate": 1, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": true, "reason": "ok", "explain": "OK: 12/12 admitted cases hold, admission rate 100.0%. Necessary, not sufficient: run the construct gate." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nTasks": 12, "minTasks": 8, "toolCount": 1, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 12, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } }, { "gate": "construct", "ok": true, "costTier": "paid", "verdict": { "outcome": "PASS", "k": 12, "n": 12, "threshold": 0.9, "alpha": 0.05, "pValue": 1 }, "reason": "ok", "detail": { "n": 12, "nIntended": 12, "rate": 1, "errors": 0, "errorRate": 0, "maxErrorRate": 0.05, "compromised": false, "reps": 1, "published": { "outcome": "EXTEND", "k": 12, "n": 12, "threshold": 0.9, "alpha": 0.05, "pValue": 1, "rawOutcome": "PASS", "downgraded": true, "wilsonLow": 0.7574992425007574, "wilsonHigh": 1, "requiredN": 78, "achievedPower": 0.20543105023999975, "detectableRate": 0.8, "reason": "downgraded_underpowered" }, "constructOracle": "A reference pass counts only when it both satisfied the check and landed a successful call on a tool the task expects. The reference agent is given the answer key, so a text check alone would pass against a server that returned nothing.", "note": "One rep per task in v0, for cost. evalgate runs three; the divergence is recorded here rather than hidden." } }, { "gate": "null_baseline", "ok": false, "costTier": "cheap", "reason": "noise_exceeds_signal", "detail": { "tNull": 1, "tAblate": 1, "killThreshold": 0.5, "outcome": "KILL", "halts": true, "ok": false, "reason": "noise_exceeds_signal", "rates": [ { "label": "no-tools", "k": 12, "n": 12, "rate": 1 }, { "label": "stubbed-empty", "k": 3, "n": 12, "rate": 0.25 }, { "label": "random-valid-args", "k": 0, "n": 12, "rate": 0 } ], "percentile": 95, "ratio": 0.5, "signalSource": "first-try success on the scored drive", "measuredBeforePaidTier": true, "decidedAfter": "drive", "explain": "KILL (noise_exceeds_signal): a null model passed 1.000 of the suite against the real agent's 1.000, at or above the 0.5 kill threshold of 0.500 (worst offender: no-tools at 1.000). The suite is measuring the model, not the server." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "null_baseline" }, "outcome": "DEGENERATE", "traceLinks": { "mcp": "https://fitnessreport.dev/traces/mcp-docs-astro-build-2026-08-19T22-07-34-673Z/mcp.jsonl", "agent": "https://fitnessreport.dev/traces/mcp-docs-astro-build-2026-08-19T22-07-34-673Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-docs-astro-build-2026-08-19T22-07-34-673Z%2Fmcp.jsonl;https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-docs-astro-build-2026-08-19T22-07-34-673Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-docs-astro-build", "kind": "mcp+llm", "startedAt": "2026-08-19T22:07:34.673Z", "endedAt": "2026-08-19T22:12:20.166Z", "durationMs": 285493, "command": [ "fitness-report", "https://mcp.docs.astro.build/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 409, "meta": 2, "message": 88, "event": 138, "turn": 179, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 85, "inputTokens": 293876, "outputTokens": 9407, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.681822, "priced": true } ], "summary": { "models": 1, "assistantTurns": 85, "turns": 179, "echoedTurns": 0, "inputTokens": 293876, "outputTokens": 9407, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.681822, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "search_astro_docs", "calls": 39, "errors": 0, "p50Ms": 365, "p95Ms": 504, "origin": "mcp", "pending": 0 } ], "errors": [], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-docs-astro-build", "kind": "mcp", "startedAt": "2026-08-19T22:07:34.673Z", "endedAt": "2026-08-19T22:12:20.148Z", "durationMs": 285475, "command": [ "fitness-report", "https://mcp.docs.astro.build/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 228, "meta": 1, "message": 88, "event": 138, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [ { "name": "search_astro_docs", "calls": 39, "errors": 0, "p50Ms": 365, "p95Ms": 504, "origin": "mcp", "pending": 0 } ], "errors": [] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-docs-astro-build", "kind": "llm", "startedAt": "2026-08-19T22:07:34.673Z", "endedAt": "2026-08-19T22:12:20.166Z", "durationMs": 285493, "command": [ "fitness-report", "https://mcp.docs.astro.build/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 181, "meta": 1, "message": 0, "event": 0, "turn": 179, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 85, "inputTokens": 293876, "outputTokens": 9407, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.681822, "priced": true } ], "summary": { "models": 1, "assistantTurns": 85, "turns": 179, "echoedTurns": 0, "inputTokens": 293876, "outputTokens": 9407, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.681822, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "search_astro_docs", "calls": 54, "errors": 0, "p50Ms": 2557, "p95Ms": 4219, "origin": "turn", "pending": 0 } ], "errors": [] } }, "toolsPlane": "mcp" }, "scoreNotes": [ "The drive ran for evidence only. A gate had already refused, so no score is published from it." ], "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs." ] }, { "schema": "fitness-report/1", "server": { "url": "https://knowledge-mcp.global.api.aws/mcp", "slug": "knowledge-mcp-global-api-aws", "era": "legacy", "negotiatedVersion": "2025-03-26", "serverInfo": { "name": "AWSKnowledgeMCP", "version": "1.0.0" }, "instructions": null, "capabilities": { "tools": { "listChanged": false } }, "transportShape": "json", "sessionful": true, "credentialContext": "anonymous" }, "run": { "id": "knowledge-mcp-global-api-aws-2026-08-19T21-18-31-987Z", "startedAt": "2026-08-19T21:18:31.987Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "20b7b4df299562a7ec5c5270c1958681580f7b7685fbbe312b5316452eda6225", "taskBudget": 20000 }, "probes": { "specCurrency": "2025-03-26", "findings": [ { "id": "spec-currency", "pass": false, "detail": "negotiated 2025-03-26 (legacy era; the current revision is 2026-07-28)", "evidence": { "negotiatedVersion": "2025-03-26", "era": "legacy" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-03-26 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "jsonrpc": "2.0", "id": "fitness-probe-bogus-version", "result": { "protocolVersion": "2025-03-26", "capabilities": { "tools": { "listChanged": false } }, "serverInfo": { "name": "AWSKnowledgeMCP", "version": "1.0.0" } } }, "answeredVersion": "2025-03-26" } }, { "id": "header-mismatch-accepted", "pass": null, "detail": "could not check: SEP-2243 header conformance is defined on the modern era only; this connection negotiated 2025-03-26 (legacy)" }, { "id": "server-discover-present", "pass": null, "detail": "could not check: server/discover is a 2026-07-28 MUST; this connection negotiated 2025-03-26 (legacy)" }, { "id": "cache-hints", "pass": null, "detail": "could not check: ttlMs/cacheScope are 2026-07-28 fields; this connection negotiated 2025-03-26 (legacy)" }, { "id": "deprecation-surface", "pass": null, "detail": "could not check: deleted-vocabulary checks are defined against 2026-07-28; this connection negotiated 2025-03-26", "evidence": { "deletedVocabulary": [], "notes": [], "toolCount": 5, "era": "legacy" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power", "construct", "null_baseline" ], "records": [ { "gate": "protocol_hygiene", "ok": false, "costTier": "free", "reason": "hygiene_findings_present", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [ "spec-currency" ] } }, { "gate": "structural", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nRequested": 13, "nGenerated": 8, "nHolding": 8, "admissionRate": 0.6153846153846154, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": true, "reason": "ok", "explain": "OK: 8/8 admitted cases hold, admission rate 61.5%. Necessary, not sufficient: run the construct gate." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nTasks": 8, "minTasks": 8, "toolCount": 5, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 8, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } }, { "gate": "construct", "ok": false, "costTier": "paid", "verdict": { "outcome": "FAIL", "k": 0, "n": 8, "threshold": 0.9, "alpha": 0.05, "pValue": 9.999999999999979e-09 }, "reason": "below_min_rate", "detail": { "n": 8, "nIntended": 0, "rate": 0, "errors": 0, "errorRate": 0, "maxErrorRate": 0.05, "compromised": false, "reps": 1, "published": { "outcome": "FAIL", "k": 0, "n": 8, "threshold": 0.9, "alpha": 0.05, "pValue": 9.999999999999979e-09, "rawOutcome": "FAIL", "downgraded": false, "wilsonLow": 0, "wilsonHigh": 0.3244156195108769, "requiredN": 78, "achievedPower": 0.20308223999999972, "detectableRate": 0.8, "reason": "failed" }, "constructOracle": "A reference pass counts only when it both satisfied the check and landed a successful call on a tool the task expects. The reference agent is given the answer key, so a text check alone would pass against a server that returned nothing.", "note": "One rep per task in v0, for cost. evalgate runs three; the divergence is recorded here rather than hidden." } }, { "gate": "null_baseline", "ok": false, "costTier": "cheap", "reason": "degenerate_no_signal", "detail": { "tNull": 0.75, "tAblate": 0, "killThreshold": 0, "outcome": "INDETERMINATE", "halts": true, "ok": false, "reason": "degenerate_no_signal", "rates": [ { "label": "no-tools", "k": 6, "n": 8, "rate": 0.75 }, { "label": "stubbed-empty", "k": 1, "n": 8, "rate": 0.125 }, { "label": "random-valid-args", "k": 0, "n": 8, "rate": 0 } ], "percentile": 95, "ratio": 0.5, "signalSource": "construct reference rate (the drive did not run)", "measuredBeforePaidTier": true, "decidedAfter": "construct", "explain": "INDETERMINATE (degenerate_no_signal): the real agent passed nothing, so there is no signal for the noise floor to be compared against. This halts exactly like KILL. It is a different finding from \"noise exceeds signal\" and is reported as one." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "construct" }, "outcome": "GATE_FAILED", "traceLinks": { "mcp": "https://fitness-report.pages.dev/traces/knowledge-mcp-global-api-aws-2026-08-19T21-18-31-987Z/mcp.jsonl", "agent": "https://fitness-report.pages.dev/traces/knowledge-mcp-global-api-aws-2026-08-19T21-18-31-987Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitness-report.pages.dev%2Ftraces%2Fknowledge-mcp-global-api-aws-2026-08-19T21-18-31-987Z%2Fmcp.jsonl;https%3A%2F%2Ffitness-report.pages.dev%2Ftraces%2Fknowledge-mcp-global-api-aws-2026-08-19T21-18-31-987Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "knowledge-mcp-global-api-aws", "kind": "mcp+llm", "startedAt": "2026-08-19T21:18:31.987Z", "endedAt": "2026-08-19T21:23:27.232Z", "durationMs": 295245, "command": [ "fitness-report", "https://knowledge-mcp.global.api.aws/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 264, "meta": 2, "message": 47, "event": 87, "turn": 126, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 60, "inputTokens": 206450, "outputTokens": 7463, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.48753, "priced": true } ], "summary": { "models": 1, "assistantTurns": 60, "turns": 126, "echoedTurns": 0, "inputTokens": 206450, "outputTokens": 7463, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.48753, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "aws___search_documentation", "calls": 23, "errors": 0, "p50Ms": null, "p95Ms": null, "origin": "mcp", "pending": 23 }, { "name": "aws___list_regions", "calls": 7, "errors": 0, "p50Ms": null, "p95Ms": null, "origin": "mcp", "pending": 7 }, { "name": "aws___read_documentation", "calls": 6, "errors": 0, "p50Ms": null, "p95Ms": null, "origin": "mcp", "pending": 6 }, { "name": "aws___retrieve_skill", "calls": 1, "errors": 0, "p50Ms": null, "p95Ms": null, "origin": "mcp", "pending": 1 } ], "errors": [], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "knowledge-mcp-global-api-aws", "kind": "mcp", "startedAt": "2026-08-19T21:18:31.987Z", "endedAt": "2026-08-19T21:23:27.213Z", "durationMs": 295226, "command": [ "fitness-report", "https://knowledge-mcp.global.api.aws/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 136, "meta": 1, "message": 47, "event": 87, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [ { "name": "aws___search_documentation", "calls": 23, "errors": 0, "p50Ms": null, "p95Ms": null, "origin": "mcp", "pending": 23 }, { "name": "aws___list_regions", "calls": 7, "errors": 0, "p50Ms": null, "p95Ms": null, "origin": "mcp", "pending": 7 }, { "name": "aws___read_documentation", "calls": 6, "errors": 0, "p50Ms": null, "p95Ms": null, "origin": "mcp", "pending": 6 }, { "name": "aws___retrieve_skill", "calls": 1, "errors": 0, "p50Ms": null, "p95Ms": null, "origin": "mcp", "pending": 1 } ], "errors": [] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "knowledge-mcp-global-api-aws", "kind": "llm", "startedAt": "2026-08-19T21:18:31.987Z", "endedAt": "2026-08-19T21:23:27.232Z", "durationMs": 295245, "command": [ "fitness-report", "https://knowledge-mcp.global.api.aws/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 128, "meta": 1, "message": 0, "event": 0, "turn": 126, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 60, "inputTokens": 206450, "outputTokens": 7463, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.48753, "priced": true } ], "summary": { "models": 1, "assistantTurns": 60, "turns": 126, "echoedTurns": 0, "inputTokens": 206450, "outputTokens": 7463, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.48753, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "aws___search_documentation", "calls": 28, "errors": 18, "p50Ms": 3218, "p95Ms": 5229, "origin": "turn", "pending": 0 }, { "name": "aws___list_regions", "calls": 7, "errors": 5, "p50Ms": 3916, "p95Ms": 5072, "origin": "turn", "pending": 0 }, { "name": "aws___read_documentation", "calls": 7, "errors": 5, "p50Ms": 3508, "p95Ms": 3744, "origin": "turn", "pending": 0 } ], "errors": [ { "t": "2026-08-19T21:21:24.969Z", "source": "tool", "name": "aws___list_regions", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:21:29.066Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:21:35.269Z", "source": "tool", "name": "aws___list_regions", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:21:37.919Z", "source": "tool", "name": "aws___list_regions", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:21:44.977Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:21:48.386Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:21:51.741Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:21:54.897Z", "source": "tool", "name": "aws___read_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:21:58.507Z", "source": "tool", "name": "aws___list_regions", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:22:06.275Z", "source": "tool", "name": "aws___read_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:22:10.019Z", "source": "tool", "name": "aws___read_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:22:16.421Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:22:19.044Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:22:21.615Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:22:24.848Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:22:32.042Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:22:35.518Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:22:38.484Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:22:41.869Z", "source": "tool", "name": "aws___read_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:22:49.056Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:22:51.985Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:22:55.273Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:22:58.491Z", "source": "tool", "name": "aws___read_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:23:02.042Z", "source": "tool", "name": "aws___list_regions", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:23:05.997Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:23:15.145Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:23:18.012Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" }, { "t": "2026-08-19T21:23:21.321Z", "source": "tool", "name": "aws___search_documentation", "message": "Error: Error POSTing to endpoint: {\"success\":false,\"error\":\"Http operation is not supported for gateway protocol type MCP\"}" } ] } }, "toolsPlane": "mcp" }, "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs." ] }, { "schema": "fitness-report/1", "server": { "url": "https://mcp.better-auth.com/mcp", "slug": "mcp-better-auth-com", "era": "legacy", "negotiatedVersion": "2025-11-25", "serverInfo": { "name": "better-auth", "version": "0.0.1" }, "instructions": "Use this server to search and retrieve Better Auth documentation. Call search_docs first to find relevant pages, then get_doc to read full content.", "capabilities": { "tools": { "listChanged": true } }, "transportShape": "json", "sessionful": false, "credentialContext": "anonymous" }, "run": { "id": "mcp-better-auth-com-2026-08-19T22-01-51-765Z", "startedAt": "2026-08-19T22:01:51.765Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "84fcf99d7cdc4fa449be1e355e4acbf0bd8b40ae43ed3fc0d33b23e3a392a4b7", "taskBudget": 20000 }, "probes": { "specCurrency": "2025-11-25", "findings": [ { "id": "spec-currency", "pass": false, "detail": "negotiated 2025-11-25 (legacy era; the current revision is 2026-07-28)", "evidence": { "negotiatedVersion": "2025-11-25", "era": "legacy" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-11-25 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "result": { "protocolVersion": "2025-11-25", "capabilities": { "tools": { "listChanged": true } }, "serverInfo": { "name": "better-auth", "version": "0.0.1" }, "instructions": "Use this server to search and retrieve Better Auth documentation. Call search_docs first to find relevant pages, then get_doc to read full content." }, "jsonrpc": "2.0", "id": "fitness-probe-bogus-version" }, "answeredVersion": "2025-11-25" } }, { "id": "header-mismatch-accepted", "pass": null, "detail": "could not check: SEP-2243 header conformance is defined on the modern era only; this connection negotiated 2025-11-25 (legacy)" }, { "id": "server-discover-present", "pass": null, "detail": "could not check: server/discover is a 2026-07-28 MUST; this connection negotiated 2025-11-25 (legacy)" }, { "id": "cache-hints", "pass": null, "detail": "could not check: ttlMs/cacheScope are 2026-07-28 fields; this connection negotiated 2025-11-25 (legacy)" }, { "id": "deprecation-surface", "pass": null, "detail": "could not check: deleted-vocabulary checks are defined against 2026-07-28; this connection negotiated 2025-11-25", "evidence": { "deletedVocabulary": [ { "where": "tools/search_docs.execution", "what": "the execution field was deleted in 2026-07-28" }, { "where": "tools/get_doc.execution", "what": "the execution field was deleted in 2026-07-28" } ], "notes": [], "toolCount": 2, "era": "legacy" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power", "construct", "null_baseline" ], "records": [ { "gate": "protocol_hygiene", "ok": false, "costTier": "free", "reason": "hygiene_findings_present", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [ "spec-currency" ] } }, { "gate": "structural", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nRequested": 12, "nGenerated": 9, "nHolding": 9, "admissionRate": 0.75, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": true, "reason": "ok", "explain": "OK: 9/9 admitted cases hold, admission rate 75.0%. Necessary, not sufficient: run the construct gate." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nTasks": 9, "minTasks": 8, "toolCount": 2, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 9, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } }, { "gate": "construct", "ok": true, "costTier": "paid", "verdict": { "outcome": "PASS", "k": 9, "n": 9, "threshold": 0.9, "alpha": 0.05, "pValue": 1 }, "reason": "ok", "detail": { "n": 9, "nIntended": 9, "rate": 1, "errors": 0, "errorRate": 0, "maxErrorRate": 0.05, "compromised": false, "reps": 1, "published": { "outcome": "EXTEND", "k": 9, "n": 9, "threshold": 0.9, "alpha": 0.05, "pValue": 1, "rawOutcome": "PASS", "downgraded": true, "wilsonLow": 0.7008472464490407, "wilsonHigh": 1, "requiredN": 78, "achievedPower": 0.08564172799999986, "detectableRate": 0.8, "reason": "downgraded_underpowered" }, "constructOracle": "A reference pass counts only when it both satisfied the check and landed a successful call on a tool the task expects. The reference agent is given the answer key, so a text check alone would pass against a server that returned nothing.", "note": "One rep per task in v0, for cost. evalgate runs three; the divergence is recorded here rather than hidden." } }, { "gate": "null_baseline", "ok": false, "costTier": "cheap", "reason": "noise_exceeds_signal", "detail": { "tNull": 0.8888888888888888, "tAblate": 1, "killThreshold": 0.5, "outcome": "KILL", "halts": true, "ok": false, "reason": "noise_exceeds_signal", "rates": [ { "label": "no-tools", "k": 8, "n": 9, "rate": 0.8888888888888888 }, { "label": "stubbed-empty", "k": 0, "n": 9, "rate": 0 }, { "label": "random-valid-args", "k": 0, "n": 9, "rate": 0 } ], "percentile": 95, "ratio": 0.5, "signalSource": "first-try success on the scored drive", "measuredBeforePaidTier": true, "decidedAfter": "drive", "explain": "KILL (noise_exceeds_signal): a null model passed 0.889 of the suite against the real agent's 1.000, at or above the 0.5 kill threshold of 0.500 (worst offender: no-tools at 0.889). The suite is measuring the model, not the server." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "null_baseline" }, "outcome": "DEGENERATE", "traceLinks": { "mcp": "https://fitnessreport.dev/traces/mcp-better-auth-com-2026-08-19T22-01-51-765Z/mcp.jsonl", "agent": "https://fitnessreport.dev/traces/mcp-better-auth-com-2026-08-19T22-01-51-765Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-better-auth-com-2026-08-19T22-01-51-765Z%2Fmcp.jsonl;https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-better-auth-com-2026-08-19T22-01-51-765Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-better-auth-com", "kind": "mcp+llm", "startedAt": "2026-08-19T22:01:51.765Z", "endedAt": "2026-08-19T22:06:26.446Z", "durationMs": 274681, "command": [ "fitness-report", "https://mcp.better-auth.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 357, "meta": 2, "message": 106, "event": 84, "turn": 163, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 77, "inputTokens": 295430, "outputTokens": 9556, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.68642, "priced": true } ], "summary": { "models": 1, "assistantTurns": 77, "turns": 163, "echoedTurns": 0, "inputTokens": 295430, "outputTokens": 9556, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.68642, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "get_doc", "calls": 25, "errors": 0, "p50Ms": 155, "p95Ms": 355, "origin": "mcp", "pending": 0 }, { "name": "search_docs", "calls": 23, "errors": 0, "p50Ms": 394, "p95Ms": 552, "origin": "mcp", "pending": 0 } ], "errors": [], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-better-auth-com", "kind": "mcp", "startedAt": "2026-08-19T22:01:51.765Z", "endedAt": "2026-08-19T22:06:26.428Z", "durationMs": 274663, "command": [ "fitness-report", "https://mcp.better-auth.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 192, "meta": 1, "message": 106, "event": 84, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [ { "name": "get_doc", "calls": 25, "errors": 0, "p50Ms": 155, "p95Ms": 355, "origin": "mcp", "pending": 0 }, { "name": "search_docs", "calls": 23, "errors": 0, "p50Ms": 394, "p95Ms": 552, "origin": "mcp", "pending": 0 } ], "errors": [] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-better-auth-com", "kind": "llm", "startedAt": "2026-08-19T22:01:51.765Z", "endedAt": "2026-08-19T22:06:26.446Z", "durationMs": 274681, "command": [ "fitness-report", "https://mcp.better-auth.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 165, "meta": 1, "message": 0, "event": 0, "turn": 163, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 77, "inputTokens": 295430, "outputTokens": 9556, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.68642, "priced": true } ], "summary": { "models": 1, "assistantTurns": 77, "turns": 163, "echoedTurns": 0, "inputTokens": 295430, "outputTokens": 9556, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.68642, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "get_doc", "calls": 27, "errors": 0, "p50Ms": 2470, "p95Ms": 7369, "origin": "turn", "pending": 0 }, { "name": "search_docs", "calls": 25, "errors": 0, "p50Ms": 1606, "p95Ms": 6744, "origin": "turn", "pending": 0 } ], "errors": [] } }, "toolsPlane": "mcp" }, "scoreNotes": [ "The drive ran for evidence only. A gate had already refused, so no score is published from it." ], "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs." ] }, { "schema": "fitness-report/1", "server": { "url": "https://docs.mcp.cloudflare.com/mcp", "slug": "docs-mcp-cloudflare-com", "era": "modern", "negotiatedVersion": "2026-07-28", "serverInfo": { "name": "docs-ai-search", "version": "0.4.13" }, "instructions": null, "capabilities": { "prompts": { "listChanged": true }, "tools": { "listChanged": true } }, "transportShape": "json", "sessionful": false, "credentialContext": "anonymous", "discover": { "_meta": { "io.modelcontextprotocol/serverInfo": { "name": "docs-ai-search", "version": "0.4.13" } }, "ttlMs": 0, "cacheScope": "private", "supportedVersions": [ "2026-07-28" ], "capabilities": { "prompts": { "listChanged": true }, "tools": { "listChanged": true } }, "resultType": "complete" } }, "run": { "id": "docs-mcp-cloudflare-com-2026-08-19T22-06-43-422Z", "startedAt": "2026-08-19T22:06:43.422Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "f6ee3b2f0175afaecd11083ddb550bff854521642e6de6f156992430e9768896", "taskBudget": 20000 }, "probes": { "specCurrency": "2026-07-28", "findings": [ { "id": "spec-currency", "pass": true, "detail": "negotiated 2026-07-28 (modern era, >= 2026-07-28)", "evidence": { "negotiatedVersion": "2026-07-28", "era": "modern" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-11-25 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "result": { "protocolVersion": "2025-11-25", "capabilities": { "tools": { "listChanged": true }, "prompts": { "listChanged": true } }, "serverInfo": { "name": "docs-ai-search", "version": "0.4.13" } }, "jsonrpc": "2.0", "id": "fitness-probe-bogus-version" }, "answeredVersion": "2025-11-25" } }, { "id": "header-mismatch-accepted", "pass": true, "detail": "rejected the mismatched Mcp-Name with HTTP 400 / JSON-RPC -32020, as SEP-2243 requires", "evidence": { "status": 400, "jsonRpcErrorCode": -32020, "headerName": "__fitness_report_mismatched_name__", "bodyName": "__fitness_report_probe_tool__", "envelope": { "jsonrpc": "2.0", "error": { "code": -32020, "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\"", "data": { "mismatch": { "header": "__fitness_report_mismatched_name__", "body": "the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\"" } } }, "id": "fitness-probe-header-mismatch" } } }, { "id": "server-discover-present", "pass": true, "detail": "server/discover answered with supportedVersions [2026-07-28]", "evidence": { "supportedVersions": [ "2026-07-28" ], "capabilities": { "prompts": { "listChanged": true }, "tools": { "listChanged": true } }, "hasInstructions": false } }, { "id": "cache-hints", "pass": true, "detail": "tools/list cache hints: ttlMs=0, cacheScope=private", "evidence": { "ttlMs": 0, "cacheScope": "private", "credentialContext": "anonymous" } }, { "id": "deprecation-surface", "pass": true, "detail": "no deleted or deprecated surface advertised", "evidence": { "deletedVocabulary": [], "notes": [], "toolCount": 2, "era": "modern" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power" ], "records": [ { "gate": "protocol_hygiene", "ok": true, "costTier": "free", "reason": "ok", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [] } }, { "gate": "structural", "ok": false, "costTier": "free", "reason": "too_few_generated", "detail": { "nRequested": 12, "nGenerated": 5, "nHolding": 5, "admissionRate": 0.4166666666666667, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": false, "reason": "too_few_generated", "explain": "REJECT (too_few_generated): 5 admitted cases is below the minimum 8. Every case held, over a sample too small to mean anything." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": false, "costTier": "free", "reason": "below_minimum_suite_size", "detail": { "nTasks": 5, "minTasks": 8, "toolCount": 2, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 5, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "structural" }, "outcome": "INSUFFICIENT_SURFACE", "traceLinks": { "mcp": "https://fitnessreport.dev/traces/docs-mcp-cloudflare-com-2026-08-19T22-06-43-422Z/mcp.jsonl", "agent": "https://fitnessreport.dev/traces/docs-mcp-cloudflare-com-2026-08-19T22-06-43-422Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fdocs-mcp-cloudflare-com-2026-08-19T22-06-43-422Z%2Fmcp.jsonl;https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fdocs-mcp-cloudflare-com-2026-08-19T22-06-43-422Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "docs-mcp-cloudflare-com", "kind": "mcp+llm", "startedAt": "2026-08-19T22:06:43.422Z", "endedAt": "2026-08-19T22:07:13.156Z", "durationMs": 29734, "command": [ "fitness-report", "https://docs.mcp.cloudflare.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 17, "meta": 2, "message": 12, "event": 1, "turn": 0, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [ { "name": "__fitness_report_probe_tool__", "calls": 1, "errors": 1, "p50Ms": 203, "p95Ms": 203, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T22:06:43.770Z", "source": "tool", "name": "__fitness_report_probe_tool__", "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\" (code -32020)" } ], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "docs-mcp-cloudflare-com", "kind": "mcp", "startedAt": "2026-08-19T22:06:43.422Z", "endedAt": "2026-08-19T22:07:13.141Z", "durationMs": 29719, "command": [ "fitness-report", "https://docs.mcp.cloudflare.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 15, "meta": 1, "message": 12, "event": 1, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [ { "name": "__fitness_report_probe_tool__", "calls": 1, "errors": 1, "p50Ms": 203, "p95Ms": 203, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T22:06:43.770Z", "source": "tool", "name": "__fitness_report_probe_tool__", "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\" (code -32020)" } ] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "docs-mcp-cloudflare-com", "kind": "llm", "startedAt": "2026-08-19T22:06:43.422Z", "endedAt": "2026-08-19T22:07:13.156Z", "durationMs": 29734, "command": [ "fitness-report", "https://docs.mcp.cloudflare.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 2, "meta": 1, "message": 0, "event": 0, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [], "errors": [] } }, "toolsPlane": "mcp" }, "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs." ] }, { "schema": "fitness-report/1", "server": { "url": "https://mcp.api.coingecko.com/mcp", "slug": "mcp-api-coingecko-com", "era": "legacy", "negotiatedVersion": "2025-11-25", "serverInfo": { "name": "coingecko_coingecko_typescript_api", "version": "7.0.0" }, "instructions": "\n This is the coingecko MCP server.\n\n Available tools:\n - search_docs: Search SDK documentation to find the right methods and parameters.\n - execute: Run TypeScript code against a pre-authenticated SDK client. Define an async run(client) function.\n\n Workflow:\n - If unsure about the API, call search_docs first.\n - Write complete solutions in a single execute call when possible. For large datasets, use API filters to narrow results or paginate within a single execute block.\n - If execute returns an error, read the error and fix your code rather than retrying the same approach.\n - Variables do not persist between execute calls. Return or log all data you need.\n - Individual HTTP requests to the API have a 30-second timeout. If a request times out, try a smaller query or add filters.\n - Code execution has a total timeout of approximately 5 minutes. If your code times out, simplify it or break it into smaller steps.\n ", "capabilities": { "logging": {}, "tools": {} }, "transportShape": "sse", "sessionful": true, "credentialContext": "anonymous" }, "run": { "id": "mcp-api-coingecko-com-2026-08-19T21-52-31-372Z", "startedAt": "2026-08-19T21:52:31.372Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "e8a87607cfe744bd6da032a15425dd430c75a91bc280e217ec6e56be57960fbb", "taskBudget": 20000 }, "probes": { "specCurrency": "2025-11-25", "findings": [ { "id": "spec-currency", "pass": false, "detail": "negotiated 2025-11-25 (legacy era; the current revision is 2026-07-28)", "evidence": { "negotiatedVersion": "2025-11-25", "era": "legacy" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-11-25 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "result": { "protocolVersion": "2025-11-25", "capabilities": { "tools": {}, "logging": {} }, "serverInfo": { "name": "coingecko_coingecko_typescript_api", "version": "7.0.0" }, "instructions": "\n This is the coingecko MCP server.\n\n Available tools:\n - search_docs: Search SDK documentation to find the right methods and parameters.\n - execute: Run TypeScript code against a pre-authenticated SDK client. Define an async run(client) function.\n\n Workflow:\n - If unsure about the API, call search_docs first.\n - Write complete solutions in a single execute call when possible. For large datasets, use API filters to narrow results or paginate within a single execute block.\n - If execute returns an error, read the error and fix your code rather than retrying the same approach.\n - Variables do not persist between execute calls. Return or log all data you need.\n - Individual HTTP requests to the API have a 30-second timeout. If a request times out, try a smaller query or add filters.\n - Code execution has a total timeout of approximately 5 minutes. If your code times out, simplify it or break it into smaller steps.\n " }, "jsonrpc": "2.0", "id": "fitness-probe-bogus-version" }, "answeredVersion": "2025-11-25" } }, { "id": "header-mismatch-accepted", "pass": null, "detail": "could not check: SEP-2243 header conformance is defined on the modern era only; this connection negotiated 2025-11-25 (legacy)" }, { "id": "server-discover-present", "pass": null, "detail": "could not check: server/discover is a 2026-07-28 MUST; this connection negotiated 2025-11-25 (legacy)" }, { "id": "cache-hints", "pass": null, "detail": "could not check: ttlMs/cacheScope are 2026-07-28 fields; this connection negotiated 2025-11-25 (legacy)" }, { "id": "deprecation-surface", "pass": null, "detail": "could not check: deleted-vocabulary checks are defined against 2026-07-28; this connection negotiated 2025-11-25", "evidence": { "deletedVocabulary": [], "notes": [], "toolCount": 2, "era": "legacy" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power", "construct", "null_baseline" ], "records": [ { "gate": "protocol_hygiene", "ok": false, "costTier": "free", "reason": "hygiene_findings_present", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [ "spec-currency" ] } }, { "gate": "structural", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nRequested": 12, "nGenerated": 12, "nHolding": 12, "admissionRate": 1, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": true, "reason": "ok", "explain": "OK: 12/12 admitted cases hold, admission rate 100.0%. Necessary, not sufficient: run the construct gate." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nTasks": 12, "minTasks": 8, "toolCount": 2, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 12, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } }, { "gate": "construct", "ok": true, "costTier": "paid", "verdict": { "outcome": "PASS", "k": 12, "n": 12, "threshold": 0.9, "alpha": 0.05, "pValue": 1 }, "reason": "ok", "detail": { "n": 12, "nIntended": 12, "rate": 1, "errors": 0, "errorRate": 0, "maxErrorRate": 0.05, "compromised": false, "reps": 1, "published": { "outcome": "EXTEND", "k": 12, "n": 12, "threshold": 0.9, "alpha": 0.05, "pValue": 1, "rawOutcome": "PASS", "downgraded": true, "wilsonLow": 0.7574992425007574, "wilsonHigh": 1, "requiredN": 78, "achievedPower": 0.20543105023999975, "detectableRate": 0.8, "reason": "downgraded_underpowered" }, "constructOracle": "A reference pass counts only when it both satisfied the check and landed a successful call on a tool the task expects. The reference agent is given the answer key, so a text check alone would pass against a server that returned nothing.", "note": "One rep per task in v0, for cost. evalgate runs three; the divergence is recorded here rather than hidden." } }, { "gate": "null_baseline", "ok": false, "costTier": "cheap", "reason": "noise_exceeds_signal", "detail": { "tNull": 0.9166666666666666, "tAblate": 0.5833333333333334, "killThreshold": 0.2916666666666667, "outcome": "KILL", "halts": true, "ok": false, "reason": "noise_exceeds_signal", "rates": [ { "label": "no-tools", "k": 11, "n": 12, "rate": 0.9166666666666666 }, { "label": "stubbed-empty", "k": 0, "n": 12, "rate": 0 }, { "label": "random-valid-args", "k": 0, "n": 12, "rate": 0 } ], "percentile": 95, "ratio": 0.5, "signalSource": "first-try success on the scored drive", "measuredBeforePaidTier": true, "decidedAfter": "drive", "explain": "KILL (noise_exceeds_signal): a null model passed 0.917 of the suite against the real agent's 0.583, at or above the 0.5 kill threshold of 0.292 (worst offender: no-tools at 0.917). The suite is measuring the model, not the server." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "null_baseline" }, "outcome": "DEGENERATE", "traceLinks": { "mcp": "https://fitnessreport.dev/traces/mcp-api-coingecko-com-2026-08-19T21-52-31-372Z/mcp.jsonl", "agent": "https://fitnessreport.dev/traces/mcp-api-coingecko-com-2026-08-19T21-52-31-372Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-api-coingecko-com-2026-08-19T21-52-31-372Z%2Fmcp.jsonl;https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-api-coingecko-com-2026-08-19T21-52-31-372Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-api-coingecko-com", "kind": "mcp+llm", "startedAt": "2026-08-19T21:52:31.372Z", "endedAt": "2026-08-19T22:00:50.653Z", "durationMs": 499281, "command": [ "fitness-report", "https://mcp.api.coingecko.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 503, "meta": 2, "message": 146, "event": 111, "turn": 242, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 115, "inputTokens": 287146, "outputTokens": 17667, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.750962, "priced": true } ], "summary": { "models": 1, "assistantTurns": 115, "turns": 242, "echoedTurns": 0, "inputTokens": 287146, "outputTokens": 17667, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.750962, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "execute", "calls": 56, "errors": 33, "p50Ms": 734, "p95Ms": 1183, "origin": "mcp", "pending": 0 }, { "name": "search_docs", "calls": 12, "errors": 0, "p50Ms": 107, "p95Ms": 258, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T21:56:12.876Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:14.051Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:14.470Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:14.912Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:15.266Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:15.701Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:16.037Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:16.376Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:16.737Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:17.341Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:17.782Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:25.231Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'bitcoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:56:28.351Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; localization: boolean; tickers: boolean; market_data: boolean; community_data: boolean; developer_data: boolean; sparkline: boolean; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'bitcoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:56:37.008Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'litecoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:56:41.016Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; localization: boolean; tickers: boolean; market_data: boolean; community_data: boolean; developer_data: boolean; sparkline: boolean; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'litecoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:56:58.231Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'ethereum' });" }, { "t": "2026-08-19T21:57:00.430Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'ethereum' });" }, { "t": "2026-08-19T21:57:08.410Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Contract'. Did you mean 'get'?\n const res = await client.coins.contract.getId({" }, { "t": "2026-08-19T21:57:16.976Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Expected 2-3 arguments, but got 1.\n const res = await client.coins.contract.get({" }, { "t": "2026-08-19T21:57:43.037Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Exchanges'. Did you mean 'getID'?\n const data = await client.exchanges.getId({ id: 'binance' });" }, { "t": "2026-08-19T21:57:48.454Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; }' is not assignable to parameter of type 'string'.\n const data = await client.exchanges.getID({ id: 'binance' });" }, { "t": "2026-08-19T21:58:03.933Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Exchanges'. Did you mean 'getID'?\n const binance = await client.exchanges.getId({ id: 'binance' });\n- Property 'getId' does not exist on type 'Exchanges'. Did you mean 'getID'?\n const kraken = await client.exchanges.getId({ id: 'kraken' });" }, { "t": "2026-08-19T21:58:06.296Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; }' is not assignable to parameter of type 'string'.\n const binance = await client.exchanges.getID({ id: 'binance' });\n- Argument of type '{ id: string; }' is not assignable to parameter of type 'string'.\n const kraken = await client.exchanges.getID({ id: 'kraken' });" }, { "t": "2026-08-19T21:58:18.750Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'asset_platforms' does not exist on type 'Coingecko'. Did you mean 'assetPlatforms'?\n const platforms = await client.asset_platforms.get({});" }, { "t": "2026-08-19T21:58:50.802Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'bitcoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:58:55.283Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; localization: boolean; tickers: boolean; market_data: boolean; community_data: boolean; developer_data: boolean; sparkline: boolean; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'bitcoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:59:06.743Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'litecoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false });" }, { "t": "2026-08-19T21:59:11.215Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; localization: boolean; tickers: boolean; market_data: boolean; community_data: boolean; developer_data: boolean; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'litecoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false });" }, { "t": "2026-08-19T21:59:24.144Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'ethereum', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:59:26.598Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; localization: boolean; tickers: boolean; market_data: boolean; community_data: boolean; developer_data: boolean; sparkline: boolean; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'ethereum', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:59:40.210Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Expected 2-3 arguments, but got 1.\n const res = await client.coins.contract.get({" }, { "t": "2026-08-19T21:59:47.914Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type 'string' is not assignable to parameter of type 'ContractGetParams'.\n const res = await client.coins.contract.get('ethereum', '0x2260fac5e5542a773aa44fbcfedf7c193bc2c599');" }, { "t": "2026-08-19T22:00:12.606Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'id' does not exist on type 'Exchanges'.\n const data = await client.exchanges.id.get({ id: 'binance' });" } ], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-api-coingecko-com", "kind": "mcp", "startedAt": "2026-08-19T21:52:31.372Z", "endedAt": "2026-08-19T22:00:50.629Z", "durationMs": 499257, "command": [ "fitness-report", "https://mcp.api.coingecko.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 259, "meta": 1, "message": 146, "event": 111, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [ { "name": "execute", "calls": 56, "errors": 33, "p50Ms": 734, "p95Ms": 1183, "origin": "mcp", "pending": 0 }, { "name": "search_docs", "calls": 12, "errors": 0, "p50Ms": 107, "p95Ms": 258, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T21:56:12.876Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:14.051Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:14.470Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:14.912Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:15.266Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:15.701Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:16.037Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:16.376Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:16.737Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:17.341Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:17.782Z", "source": "tool", "name": "execute", "message": "The code is missing a top-level `run` function. Write code within this template:\n\n```\nasync function run(client) {\n // Fill this out\n}\n```" }, { "t": "2026-08-19T21:56:25.231Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'bitcoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:56:28.351Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; localization: boolean; tickers: boolean; market_data: boolean; community_data: boolean; developer_data: boolean; sparkline: boolean; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'bitcoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:56:37.008Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'litecoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:56:41.016Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; localization: boolean; tickers: boolean; market_data: boolean; community_data: boolean; developer_data: boolean; sparkline: boolean; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'litecoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:56:58.231Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'ethereum' });" }, { "t": "2026-08-19T21:57:00.430Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'ethereum' });" }, { "t": "2026-08-19T21:57:08.410Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Contract'. Did you mean 'get'?\n const res = await client.coins.contract.getId({" }, { "t": "2026-08-19T21:57:16.976Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Expected 2-3 arguments, but got 1.\n const res = await client.coins.contract.get({" }, { "t": "2026-08-19T21:57:43.037Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Exchanges'. Did you mean 'getID'?\n const data = await client.exchanges.getId({ id: 'binance' });" }, { "t": "2026-08-19T21:57:48.454Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; }' is not assignable to parameter of type 'string'.\n const data = await client.exchanges.getID({ id: 'binance' });" }, { "t": "2026-08-19T21:58:03.933Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Exchanges'. Did you mean 'getID'?\n const binance = await client.exchanges.getId({ id: 'binance' });\n- Property 'getId' does not exist on type 'Exchanges'. Did you mean 'getID'?\n const kraken = await client.exchanges.getId({ id: 'kraken' });" }, { "t": "2026-08-19T21:58:06.296Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; }' is not assignable to parameter of type 'string'.\n const binance = await client.exchanges.getID({ id: 'binance' });\n- Argument of type '{ id: string; }' is not assignable to parameter of type 'string'.\n const kraken = await client.exchanges.getID({ id: 'kraken' });" }, { "t": "2026-08-19T21:58:18.750Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'asset_platforms' does not exist on type 'Coingecko'. Did you mean 'assetPlatforms'?\n const platforms = await client.asset_platforms.get({});" }, { "t": "2026-08-19T21:58:50.802Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'bitcoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:58:55.283Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; localization: boolean; tickers: boolean; market_data: boolean; community_data: boolean; developer_data: boolean; sparkline: boolean; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'bitcoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:59:06.743Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'litecoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false });" }, { "t": "2026-08-19T21:59:11.215Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; localization: boolean; tickers: boolean; market_data: boolean; community_data: boolean; developer_data: boolean; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'litecoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false });" }, { "t": "2026-08-19T21:59:24.144Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'ethereum', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:59:26.598Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; localization: boolean; tickers: boolean; market_data: boolean; community_data: boolean; developer_data: boolean; sparkline: boolean; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'ethereum', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:59:40.210Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Expected 2-3 arguments, but got 1.\n const res = await client.coins.contract.get({" }, { "t": "2026-08-19T21:59:47.914Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type 'string' is not assignable to parameter of type 'ContractGetParams'.\n const res = await client.coins.contract.get('ethereum', '0x2260fac5e5542a773aa44fbcfedf7c193bc2c599');" }, { "t": "2026-08-19T22:00:12.606Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'id' does not exist on type 'Exchanges'.\n const data = await client.exchanges.id.get({ id: 'binance' });" } ] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-api-coingecko-com", "kind": "llm", "startedAt": "2026-08-19T21:52:31.372Z", "endedAt": "2026-08-19T22:00:50.653Z", "durationMs": 499281, "command": [ "fitness-report", "https://mcp.api.coingecko.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 244, "meta": 1, "message": 0, "event": 0, "turn": 242, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 115, "inputTokens": 287146, "outputTokens": 17667, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.750962, "priced": true } ], "summary": { "models": 1, "assistantTurns": 115, "turns": 242, "echoedTurns": 0, "inputTokens": 287146, "outputTokens": 17667, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.750962, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "execute", "calls": 65, "errors": 22, "p50Ms": 2384, "p95Ms": 6597, "origin": "turn", "pending": 0 }, { "name": "search_docs", "calls": 16, "errors": 0, "p50Ms": 2992, "p95Ms": 5796, "origin": "turn", "pending": 0 } ], "errors": [ { "t": "2026-08-19T21:56:25.230Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'bitcoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:56:28.350Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; localization: boolean; tickers: boolean; market_data: boolean; community_data: boolean; developer_data: boolean; sparkline: boolean; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'bitcoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:56:37.007Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'litecoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:56:41.016Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; localization: boolean; tickers: boolean; market_data: boolean; community_data: boolean; developer_data: boolean; sparkline: boolean; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'litecoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:56:58.230Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'ethereum' });" }, { "t": "2026-08-19T21:57:00.429Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'ethereum' });" }, { "t": "2026-08-19T21:57:08.409Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Contract'. Did you mean 'get'?\n const res = await client.coins.contract.getId({" }, { "t": "2026-08-19T21:57:16.975Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Expected 2-3 arguments, but got 1.\n const res = await client.coins.contract.get({" }, { "t": "2026-08-19T21:57:43.037Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Exchanges'. Did you mean 'getID'?\n const data = await client.exchanges.getId({ id: 'binance' });" }, { "t": "2026-08-19T21:57:48.454Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; }' is not assignable to parameter of type 'string'.\n const data = await client.exchanges.getID({ id: 'binance' });" }, { "t": "2026-08-19T21:58:03.933Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Exchanges'. Did you mean 'getID'?\n const binance = await client.exchanges.getId({ id: 'binance' });\n- Property 'getId' does not exist on type 'Exchanges'. Did you mean 'getID'?\n const kraken = await client.exchanges.getId({ id: 'kraken' });" }, { "t": "2026-08-19T21:58:06.296Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; }' is not assignable to parameter of type 'string'.\n const binance = await client.exchanges.getID({ id: 'binance' });\n- Argument of type '{ id: string; }' is not assignable to parameter of type 'string'.\n const kraken = await client.exchanges.getID({ id: 'kraken' });" }, { "t": "2026-08-19T21:58:18.749Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'asset_platforms' does not exist on type 'Coingecko'. Did you mean 'assetPlatforms'?\n const platforms = await client.asset_platforms.get({});" }, { "t": "2026-08-19T21:58:50.801Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'bitcoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:58:55.282Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; localization: boolean; tickers: boolean; market_data: boolean; community_data: boolean; developer_data: boolean; sparkline: boolean; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'bitcoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:59:06.743Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'litecoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false });" }, { "t": "2026-08-19T21:59:11.214Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; localization: boolean; tickers: boolean; market_data: boolean; community_data: boolean; developer_data: boolean; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'litecoin', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false });" }, { "t": "2026-08-19T21:59:24.144Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'getId' does not exist on type 'Coins'. Did you mean 'getID'?\n const data = await client.coins.getId({ id: 'ethereum', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:59:26.597Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type '{ id: string; localization: boolean; tickers: boolean; market_data: boolean; community_data: boolean; developer_data: boolean; sparkline: boolean; }' is not assignable to parameter of type 'string'.\n const data = await client.coins.getID({ id: 'ethereum', localization: false, tickers: false, market_data: false, community_data: false, developer_data: false, sparkline: false });" }, { "t": "2026-08-19T21:59:40.210Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Expected 2-3 arguments, but got 1.\n const res = await client.coins.contract.get({" }, { "t": "2026-08-19T21:59:47.914Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Argument of type 'string' is not assignable to parameter of type 'ContractGetParams'.\n const res = await client.coins.contract.get('ethereum', '0x2260fac5e5542a773aa44fbcfedf7c193bc2c599');" }, { "t": "2026-08-19T22:00:12.606Z", "source": "tool", "name": "execute", "message": "The code contains TypeScript diagnostics:\n- Property 'id' does not exist on type 'Exchanges'.\n const data = await client.exchanges.id.get({ id: 'binance' });" } ] } }, "toolsPlane": "mcp" }, "scoreNotes": [ "The drive ran for evidence only. A gate had already refused, so no score is published from it." ], "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs." ] }, { "schema": "fitness-report/1", "server": { "url": "https://mcp.context7.com/mcp", "slug": "mcp-context7-com", "era": "modern", "negotiatedVersion": "2026-07-28", "serverInfo": { "name": "Context7", "icons": [ { "src": "https://context7.com/context7-icon-green.png", "mimeType": "image/png" } ], "version": "4.0.1", "websiteUrl": "https://context7.com", "description": "Context7 provides up-to-date documentation and code examples for libraries and frameworks." }, "instructions": "Use this server to fetch current documentation whenever the user asks about a library, framework, SDK, API, CLI tool, or cloud service \u2014 even well-known ones like React, Next.js, Prisma, Express, Tailwind, Django, or Spring Boot. This includes API syntax, configuration, version migration, library-specific debugging, setup instructions, and CLI tool usage. Use even when you think you know the answer \u2014 your training data may not reflect recent changes. Prefer this over web search for library docs.\n\nDo not use for: refactoring, writing scripts from scratch, debugging business logic, code review, or general programming concepts.", "capabilities": { "prompts": { "listChanged": true }, "resources": { "listChanged": true }, "tools": { "listChanged": true } }, "transportShape": "json", "sessionful": false, "credentialContext": "anonymous", "discover": { "_meta": { "io.modelcontextprotocol/serverInfo": { "name": "Context7", "icons": [ { "src": "https://context7.com/context7-icon-green.png", "mimeType": "image/png" } ], "version": "4.0.1", "websiteUrl": "https://context7.com", "description": "Context7 provides up-to-date documentation and code examples for libraries and frameworks." } }, "ttlMs": 0, "cacheScope": "private", "supportedVersions": [ "2026-07-28" ], "capabilities": { "prompts": { "listChanged": true }, "resources": { "listChanged": true }, "tools": { "listChanged": true } }, "instructions": "Use this server to fetch current documentation whenever the user asks about a library, framework, SDK, API, CLI tool, or cloud service \u2014 even well-known ones like React, Next.js, Prisma, Express, Tailwind, Django, or Spring Boot. This includes API syntax, configuration, version migration, library-specific debugging, setup instructions, and CLI tool usage. Use even when you think you know the answer \u2014 your training data may not reflect recent changes. Prefer this over web search for library docs.\n\nDo not use for: refactoring, writing scripts from scratch, debugging business logic, code review, or general programming concepts.", "resultType": "complete" } }, "run": { "id": "mcp-context7-com-2026-08-19T21-45-24-040Z", "startedAt": "2026-08-19T21:45:24.040Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "ba7e1ef6bb5e02f935bcf6fc20699972eb3e961bec90f96ae3116d374dc0704e", "taskBudget": 20000 }, "probes": { "specCurrency": "2026-07-28", "findings": [ { "id": "spec-currency", "pass": true, "detail": "negotiated 2026-07-28 (modern era, >= 2026-07-28)", "evidence": { "negotiatedVersion": "2026-07-28", "era": "modern" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-11-25 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "result": { "protocolVersion": "2025-11-25", "capabilities": { "prompts": { "listChanged": true }, "resources": { "listChanged": true }, "tools": { "listChanged": true } }, "serverInfo": { "name": "Context7", "version": "4.0.1", "websiteUrl": "https://context7.com", "description": "Context7 provides up-to-date documentation and code examples for libraries and frameworks.", "icons": [ { "src": "https://context7.com/context7-icon-green.png", "mimeType": "image/png" } ] }, "instructions": "Use this server to fetch current documentation whenever the user asks about a library, framework, SDK, API, CLI tool, or cloud service \u2014 even well-known ones like React, Next.js, Prisma, Express, Tailwind, Django, or Spring Boot. This includes API syntax, configuration, version migration, library-specific debugging, setup instructions, and CLI tool usage. Use even when you think you know the answer \u2014 your training data may not reflect recent changes. Prefer this over web search for library docs.\n\nDo not use for: refactoring, writing scripts from scratch, debugging business logic, code review, or general programming concepts." }, "jsonrpc": "2.0", "id": "fitness-probe-bogus-version" }, "answeredVersion": "2025-11-25" } }, { "id": "header-mismatch-accepted", "pass": true, "detail": "rejected the mismatched Mcp-Name with HTTP 400 / JSON-RPC -32020, as SEP-2243 requires", "evidence": { "status": 400, "jsonRpcErrorCode": -32020, "headerName": "__fitness_report_mismatched_name__", "bodyName": "__fitness_report_probe_tool__", "envelope": { "jsonrpc": "2.0", "error": { "code": -32020, "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\"", "data": { "mismatch": { "header": "__fitness_report_mismatched_name__", "body": "the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\"" } } }, "id": "fitness-probe-header-mismatch" } } }, { "id": "server-discover-present", "pass": true, "detail": "server/discover answered with supportedVersions [2026-07-28]", "evidence": { "supportedVersions": [ "2026-07-28" ], "capabilities": { "prompts": { "listChanged": true }, "resources": { "listChanged": true }, "tools": { "listChanged": true } }, "hasInstructions": true } }, { "id": "cache-hints", "pass": true, "detail": "tools/list cache hints: ttlMs=0, cacheScope=private", "evidence": { "ttlMs": 0, "cacheScope": "private", "credentialContext": "anonymous" } }, { "id": "deprecation-surface", "pass": true, "detail": "no deleted or deprecated surface advertised", "evidence": { "deletedVocabulary": [], "notes": [], "toolCount": 2, "era": "modern" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power", "construct", "null_baseline" ], "records": [ { "gate": "protocol_hygiene", "ok": true, "costTier": "free", "reason": "ok", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [] } }, { "gate": "structural", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nRequested": 12, "nGenerated": 10, "nHolding": 10, "admissionRate": 0.8333333333333334, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": true, "reason": "ok", "explain": "OK: 10/10 admitted cases hold, admission rate 83.3%. Necessary, not sufficient: run the construct gate." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nTasks": 10, "minTasks": 8, "toolCount": 2, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 10, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } }, { "gate": "construct", "ok": true, "costTier": "paid", "verdict": { "outcome": "PASS", "k": 10, "n": 10, "threshold": 0.9, "alpha": 0.05, "pValue": 1 }, "reason": "ok", "detail": { "n": 10, "nIntended": 10, "rate": 1, "errors": 0, "errorRate": 0, "maxErrorRate": 0.05, "compromised": false, "reps": 1, "published": { "outcome": "EXTEND", "k": 10, "n": 10, "threshold": 0.9, "alpha": 0.05, "pValue": 1, "rawOutcome": "PASS", "downgraded": true, "wilsonLow": 0.7224598312333834, "wilsonHigh": 1, "requiredN": 78, "achievedPower": 0.12087388159999982, "detectableRate": 0.8, "reason": "downgraded_underpowered" }, "constructOracle": "A reference pass counts only when it both satisfied the check and landed a successful call on a tool the task expects. The reference agent is given the answer key, so a text check alone would pass against a server that returned nothing.", "note": "One rep per task in v0, for cost. evalgate runs three; the divergence is recorded here rather than hidden." } }, { "gate": "null_baseline", "ok": false, "costTier": "cheap", "reason": "noise_exceeds_signal", "detail": { "tNull": 1, "tAblate": 1, "killThreshold": 0.5, "outcome": "KILL", "halts": true, "ok": false, "reason": "noise_exceeds_signal", "rates": [ { "label": "no-tools", "k": 10, "n": 10, "rate": 1 }, { "label": "stubbed-empty", "k": 3, "n": 10, "rate": 0.3 }, { "label": "random-valid-args", "k": 0, "n": 10, "rate": 0 } ], "percentile": 95, "ratio": 0.5, "signalSource": "first-try success on the scored drive", "measuredBeforePaidTier": true, "decidedAfter": "drive", "explain": "KILL (noise_exceeds_signal): a null model passed 1.000 of the suite against the real agent's 1.000, at or above the 0.5 kill threshold of 0.500 (worst offender: no-tools at 1.000). The suite is measuring the model, not the server." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "null_baseline" }, "outcome": "DEGENERATE", "traceLinks": { "mcp": "https://fitnessreport.dev/traces/mcp-context7-com-2026-08-19T21-45-24-040Z/mcp.jsonl", "agent": "https://fitnessreport.dev/traces/mcp-context7-com-2026-08-19T21-45-24-040Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-context7-com-2026-08-19T21-45-24-040Z%2Fmcp.jsonl;https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-context7-com-2026-08-19T21-45-24-040Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-context7-com", "kind": "mcp+llm", "startedAt": "2026-08-19T21:45:24.040Z", "endedAt": "2026-08-19T21:52:04.668Z", "durationMs": 400628, "command": [ "fitness-report", "https://mcp.context7.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 386, "meta": 2, "message": 114, "event": 93, "turn": 175, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 84, "inputTokens": 223021, "outputTokens": 13707, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.583112, "priced": true } ], "summary": { "models": 1, "assistantTurns": 84, "turns": 175, "echoedTurns": 0, "inputTokens": 223021, "outputTokens": 13707, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.583112, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "resolve-library-id", "calls": 30, "errors": 0, "p50Ms": 1455, "p95Ms": 2007, "origin": "mcp", "pending": 0 }, { "name": "query-docs", "calls": 21, "errors": 0, "p50Ms": 1444, "p95Ms": 1830, "origin": "mcp", "pending": 0 }, { "name": "__fitness_report_probe_tool__", "calls": 1, "errors": 1, "p50Ms": 105, "p95Ms": 105, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T21:45:24.979Z", "source": "tool", "name": "__fitness_report_probe_tool__", "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\" (code -32020)" } ], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-context7-com", "kind": "mcp", "startedAt": "2026-08-19T21:45:24.040Z", "endedAt": "2026-08-19T21:52:04.656Z", "durationMs": 400616, "command": [ "fitness-report", "https://mcp.context7.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 209, "meta": 1, "message": 114, "event": 93, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [ { "name": "resolve-library-id", "calls": 30, "errors": 0, "p50Ms": 1455, "p95Ms": 2007, "origin": "mcp", "pending": 0 }, { "name": "query-docs", "calls": 21, "errors": 0, "p50Ms": 1444, "p95Ms": 1830, "origin": "mcp", "pending": 0 }, { "name": "__fitness_report_probe_tool__", "calls": 1, "errors": 1, "p50Ms": 105, "p95Ms": 105, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T21:45:24.979Z", "source": "tool", "name": "__fitness_report_probe_tool__", "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\" (code -32020)" } ] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-context7-com", "kind": "llm", "startedAt": "2026-08-19T21:45:24.040Z", "endedAt": "2026-08-19T21:52:04.668Z", "durationMs": 400628, "command": [ "fitness-report", "https://mcp.context7.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 177, "meta": 1, "message": 0, "event": 0, "turn": 175, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 84, "inputTokens": 223021, "outputTokens": 13707, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.583112, "priced": true } ], "summary": { "models": 1, "assistantTurns": 84, "turns": 175, "echoedTurns": 0, "inputTokens": 223021, "outputTokens": 13707, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.583112, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "resolve-library-id", "calls": 31, "errors": 0, "p50Ms": 4173, "p95Ms": 7420, "origin": "turn", "pending": 0 }, { "name": "query-docs", "calls": 20, "errors": 0, "p50Ms": 4528, "p95Ms": 6998, "origin": "turn", "pending": 0 } ], "errors": [] } }, "toolsPlane": "mcp" }, "scoreNotes": [ "The drive ran for evidence only. A gate had already refused, so no score is published from it." ], "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs." ] }, { "schema": "fitness-report/1", "server": { "url": "https://mcp.convex.dev/mcp", "slug": "mcp-convex-dev", "era": "legacy", "negotiatedVersion": "2025-06-18", "serverInfo": { "name": "mcp-typescript server on vercel", "version": "0.1.0" }, "instructions": "This server is the source of truth for building apps on Convex (a reactive, type-safe TS/JS backend: database, server functions, auth, realtime, file storage, scheduling). Whenever the user asks to build, scaffold, or prototype an app or add a backend \u2014 and Convex is in play (they said \"Convex\", invoked the Convex tool/app, or want a full-stack/reactive backend) \u2014 use these tools BEFORE writing any files: call `start_convex_app` for a new app (or `add_convex_to_existing_project` for an existing repo), then call `get_runbook` and follow it in order. The runbook carries rules your training data likely gets wrong: how to start `npx convex dev` without hanging, choosing a cloud vs. anonymous-local deployment (prefer cloud whenever the user is signed in to Convex), Convex Auth/JWT setup, and which prebuilt components to use instead of hand-rolling. Do not hand-scaffold a Convex app from memory \u2014 that reliably produces throwaway anonymous backends, broken auth, and re-implemented components. The tools are read-only guidance and cheap to call.", "capabilities": { "tools": { "listChanged": true } }, "transportShape": "sse", "sessionful": false, "credentialContext": "anonymous" }, "run": { "id": "mcp-convex-dev-2026-08-19T21-36-05-550Z", "startedAt": "2026-08-19T21:36:05.550Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "b4079f9c0dc21797fa56770b5e6d6c4d5b6fd7a1f39c2b688a79cad1d08ba343", "taskBudget": 20000 }, "probes": { "specCurrency": "2025-06-18", "findings": [ { "id": "spec-currency", "pass": false, "detail": "negotiated 2025-06-18 (legacy era; the current revision is 2026-07-28)", "evidence": { "negotiatedVersion": "2025-06-18", "era": "legacy" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-06-18 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "result": { "protocolVersion": "2025-06-18", "capabilities": { "tools": { "listChanged": true } }, "serverInfo": { "name": "mcp-typescript server on vercel", "version": "0.1.0" }, "instructions": "This server is the source of truth for building apps on Convex (a reactive, type-safe TS/JS backend: database, server functions, auth, realtime, file storage, scheduling). Whenever the user asks to build, scaffold, or prototype an app or add a backend \u2014 and Convex is in play (they said \"Convex\", invoked the Convex tool/app, or want a full-stack/reactive backend) \u2014 use these tools BEFORE writing any files: call `start_convex_app` for a new app (or `add_convex_to_existing_project` for an existing repo), then call `get_runbook` and follow it in order. The runbook carries rules your training data likely gets wrong: how to start `npx convex dev` without hanging, choosing a cloud vs. anonymous-local deployment (prefer cloud whenever the user is signed in to Convex), Convex Auth/JWT setup, and which prebuilt components to use instead of hand-rolling. Do not hand-scaffold a Convex app from memory \u2014 that reliably produces throwaway anonymous backends, broken auth, and re-implemented components. The tools are read-only guidance and cheap to call." }, "jsonrpc": "2.0", "id": "fitness-probe-bogus-version" }, "answeredVersion": "2025-06-18" } }, { "id": "header-mismatch-accepted", "pass": null, "detail": "could not check: SEP-2243 header conformance is defined on the modern era only; this connection negotiated 2025-06-18 (legacy)" }, { "id": "server-discover-present", "pass": null, "detail": "could not check: server/discover is a 2026-07-28 MUST; this connection negotiated 2025-06-18 (legacy)" }, { "id": "cache-hints", "pass": null, "detail": "could not check: ttlMs/cacheScope are 2026-07-28 fields; this connection negotiated 2025-06-18 (legacy)" }, { "id": "deprecation-surface", "pass": null, "detail": "could not check: deleted-vocabulary checks are defined against 2026-07-28; this connection negotiated 2025-06-18", "evidence": { "deletedVocabulary": [], "notes": [], "toolCount": 4, "era": "legacy" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power" ], "records": [ { "gate": "protocol_hygiene", "ok": false, "costTier": "free", "reason": "hygiene_findings_present", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [ "spec-currency" ] } }, { "gate": "structural", "ok": false, "costTier": "free", "reason": "too_few_generated", "detail": { "nRequested": 12, "nGenerated": 5, "nHolding": 5, "admissionRate": 0.4166666666666667, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": false, "reason": "too_few_generated", "explain": "REJECT (too_few_generated): 5 admitted cases is below the minimum 8. Every case held, over a sample too small to mean anything." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": false, "costTier": "free", "reason": "below_minimum_suite_size", "detail": { "nTasks": 5, "minTasks": 8, "toolCount": 4, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 5, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "structural" }, "outcome": "INSUFFICIENT_SURFACE", "traceLinks": { "mcp": "https://fitnessreport.dev/traces/mcp-convex-dev-2026-08-19T21-36-05-550Z/mcp.jsonl", "agent": "https://fitnessreport.dev/traces/mcp-convex-dev-2026-08-19T21-36-05-550Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-convex-dev-2026-08-19T21-36-05-550Z%2Fmcp.jsonl;https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-convex-dev-2026-08-19T21-36-05-550Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-convex-dev", "kind": "mcp+llm", "startedAt": "2026-08-19T21:36:05.550Z", "endedAt": "2026-08-19T21:37:43.331Z", "durationMs": 97781, "command": [ "fitness-report", "https://mcp.convex.dev/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 15, "meta": 2, "message": 10, "event": 1, "turn": 0, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [], "errors": [], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-convex-dev", "kind": "mcp", "startedAt": "2026-08-19T21:36:05.550Z", "endedAt": "2026-08-19T21:37:43.315Z", "durationMs": 97765, "command": [ "fitness-report", "https://mcp.convex.dev/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 13, "meta": 1, "message": 10, "event": 1, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [], "errors": [] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-convex-dev", "kind": "llm", "startedAt": "2026-08-19T21:36:05.550Z", "endedAt": "2026-08-19T21:37:43.331Z", "durationMs": 97781, "command": [ "fitness-report", "https://mcp.convex.dev/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 2, "meta": 1, "message": 0, "event": 0, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [], "errors": [] } }, "toolsPlane": null }, "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs." ] }, { "schema": "fitness-report/1", "server": { "url": "https://mcp.deepwiki.com/mcp", "slug": "mcp-deepwiki-com", "era": "legacy", "negotiatedVersion": "2025-11-25", "serverInfo": { "name": "DeepWiki", "version": "2.14.3" }, "instructions": "DeepWiki MCP provides AI-powered documentation for GitHub repositories.\n\nAvailable tools:\n- read_wiki_structure: Get a list of documentation topics for a repository\n- read_wiki_contents: View full documentation about a repository\n- ask_question: Ask any question about a repository and get an AI-powered answer\n- list_available_repos: List your available repositories (private mode only)\n- generate_wiki: Generate a codebase wiki for a repository \u2014 only use when explicitly requested by the user (private mode only)\n- devin_automation_manage: Manage Devin automations \u2014 list, get, create, update, delete, or fetch the trigger event schemas (private mode only)\n- devin_code_scan_manage: Manage Devin code scans, sometimes referred to as 'security scans' or 'Devin Security Swarm' \u2014 list scans, list findings, list profiles, get a profile, create a scan, remediate a finding (private mode only)\n- devin_knowledge_manage: Manage Devin knowledge notes and suggestions \u2014 list, search, get, create, update, delete notes, view folder structure, list/view/dismiss knowledge suggestions (private mode only)\n- devin_oncall_manage: Devin Oncall operations \u2014 get an Oncall report's current responder membership, and page through a responder's open issues (private mode only)\n- devin_playbook_manage: Manage Devin playbooks \u2014 list, get, create, update, delete (private mode only)\n- devin_review_manage: Trigger a Devin Review for a pull request or fetch the latest review status (private mode only)\n- devin_schedule_manage: Manage scheduled Devin sessions \u2014 list, get, create, update, delete (private mode only)\n- devin_session_create: Create one or more child Devin sessions (private mode only)\n- devin_session_interact: Manage a Devin session \u2014 get status, send messages, sleep/terminate/archive/unarchive, read messages & attachments, manage tags (private mode only)\n- devin_session_events: Inspect session events \u2014 list summaries, fetch full details, or search event contents (private mode only)\n- devin_session_search: Search and filter Devin sessions (private mode only)\n- devin_session_gather: Wait for multiple Devin sessions to reach a settled state (private mode only)\n- list_integrations: List all native integrations and MCP servers with their status and settings URLs (private mode only)\n- find_setting: Find Devin webapp settings pages and deep-link URLs for them (private mode only)\n", "capabilities": { "experimental": {}, "prompts": { "listChanged": true }, "resources": { "subscribe": false, "listChanged": true }, "tools": { "listChanged": true } }, "transportShape": "sse", "sessionful": false, "credentialContext": "anonymous" }, "run": { "id": "mcp-deepwiki-com-2026-08-19T21-44-44-822Z", "startedAt": "2026-08-19T21:44:44.822Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "f6d18fb34a79746cbdb23db7042ef74517248257e9778e5fb93134c9f045a457", "taskBudget": 20000 }, "probes": { "specCurrency": "2025-11-25", "findings": [ { "id": "spec-currency", "pass": false, "detail": "negotiated 2025-11-25 (legacy era; the current revision is 2026-07-28)", "evidence": { "negotiatedVersion": "2025-11-25", "era": "legacy" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-11-25 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "jsonrpc": "2.0", "id": "fitness-probe-bogus-version", "result": { "protocolVersion": "2025-11-25", "capabilities": { "experimental": {}, "prompts": { "listChanged": true }, "resources": { "subscribe": false, "listChanged": true }, "tools": { "listChanged": true } }, "serverInfo": { "name": "DeepWiki", "version": "2.14.3" }, "instructions": "DeepWiki MCP provides AI-powered documentation for GitHub repositories.\n\nAvailable tools:\n- read_wiki_structure: Get a list of documentation topics for a repository\n- read_wiki_contents: View full documentation about a repository\n- ask_question: Ask any question about a repository and get an AI-powered answer\n- list_available_repos: List your available repositories (private mode only)\n- generate_wiki: Generate a codebase wiki for a repository \u2014 only use when explicitly requested by the user (private mode only)\n- devin_automation_manage: Manage Devin automations \u2014 list, get, create, update, delete, or fetch the trigger event schemas (private mode only)\n- devin_code_scan_manage: Manage Devin code scans, sometimes referred to as 'security scans' or 'Devin Security Swarm' \u2014 list scans, list findings, list profiles, get a profile, create a scan, remediate a finding (private mode only)\n- devin_knowledge_manage: Manage Devin knowledge notes and suggestions \u2014 list, search, get, create, update, delete notes, view folder structure, list/view/dismiss knowledge suggestions (private mode only)\n- devin_oncall_manage: Devin Oncall operations \u2014 get an Oncall report's current responder membership, and page through a responder's open issues (private mode only)\n- devin_playbook_manage: Manage Devin playbooks \u2014 list, get, create, update, delete (private mode only)\n- devin_review_manage: Trigger a Devin Review for a pull request or fetch the latest review status (private mode only)\n- devin_schedule_manage: Manage scheduled Devin sessions \u2014 list, get, create, update, delete (private mode only)\n- devin_session_create: Create one or more child Devin sessions (private mode only)\n- devin_session_interact: Manage a Devin session \u2014 get status, send messages, sleep/terminate/archive/unarchive, read messages & attachments, manage tags (private mode only)\n- devin_session_events: Inspect session events \u2014 list summaries, fetch full details, or search event contents (private mode only)\n- devin_session_search: Search and filter Devin sessions (private mode only)\n- devin_session_gather: Wait for multiple Devin sessions to reach a settled state (private mode only)\n- list_integrations: List all native integrations and MCP servers with their status and settings URLs (private mode only)\n- find_setting: Find Devin webapp settings pages and deep-link URLs for them (private mode only)\n" } }, "answeredVersion": "2025-11-25" } }, { "id": "header-mismatch-accepted", "pass": null, "detail": "could not check: SEP-2243 header conformance is defined on the modern era only; this connection negotiated 2025-11-25 (legacy)" }, { "id": "server-discover-present", "pass": null, "detail": "could not check: server/discover is a 2026-07-28 MUST; this connection negotiated 2025-11-25 (legacy)" }, { "id": "cache-hints", "pass": null, "detail": "could not check: ttlMs/cacheScope are 2026-07-28 fields; this connection negotiated 2025-11-25 (legacy)" }, { "id": "deprecation-surface", "pass": null, "detail": "could not check: deleted-vocabulary checks are defined against 2026-07-28; this connection negotiated 2025-11-25 (1 deprecation note(s) recorded)", "evidence": { "deletedVocabulary": [], "notes": [ { "where": "capabilities.experimental", "what": "non-standard surface advertised" } ], "toolCount": 3, "era": "legacy" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power" ], "records": [ { "gate": "protocol_hygiene", "ok": false, "costTier": "free", "reason": "hygiene_findings_present", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [ "spec-currency" ] } }, { "gate": "structural", "ok": false, "costTier": "free", "reason": "too_few_generated", "detail": { "nRequested": 12, "nGenerated": 5, "nHolding": 5, "admissionRate": 0.4166666666666667, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": false, "reason": "too_few_generated", "explain": "REJECT (too_few_generated): 5 admitted cases is below the minimum 8. Every case held, over a sample too small to mean anything." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": false, "costTier": "free", "reason": "below_minimum_suite_size", "detail": { "nTasks": 5, "minTasks": 8, "toolCount": 3, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 5, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "structural" }, "outcome": "INSUFFICIENT_SURFACE", "traceLinks": { "mcp": "https://fitnessreport.dev/traces/mcp-deepwiki-com-2026-08-19T21-44-44-822Z/mcp.jsonl", "agent": "https://fitnessreport.dev/traces/mcp-deepwiki-com-2026-08-19T21-44-44-822Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-deepwiki-com-2026-08-19T21-44-44-822Z%2Fmcp.jsonl;https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-deepwiki-com-2026-08-19T21-44-44-822Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-deepwiki-com", "kind": "mcp+llm", "startedAt": "2026-08-19T21:44:44.822Z", "endedAt": "2026-08-19T21:45:12.201Z", "durationMs": 27379, "command": [ "fitness-report", "https://mcp.deepwiki.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 15, "meta": 2, "message": 10, "event": 1, "turn": 0, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [], "errors": [], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-deepwiki-com", "kind": "mcp", "startedAt": "2026-08-19T21:44:44.822Z", "endedAt": "2026-08-19T21:45:12.181Z", "durationMs": 27359, "command": [ "fitness-report", "https://mcp.deepwiki.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 13, "meta": 1, "message": 10, "event": 1, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [], "errors": [] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-deepwiki-com", "kind": "llm", "startedAt": "2026-08-19T21:44:44.822Z", "endedAt": "2026-08-19T21:45:12.201Z", "durationMs": 27379, "command": [ "fitness-report", "https://mcp.deepwiki.com/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 2, "meta": 1, "message": 0, "event": 0, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [], "errors": [] } }, "toolsPlane": null }, "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs." ] }, { "schema": "fitness-report/1", "server": { "url": "https://mcp.exa.ai/mcp", "slug": "mcp-exa-ai", "era": "legacy", "negotiatedVersion": "2025-11-25", "serverInfo": { "name": "exa-search-server", "title": "Exa", "icons": [ { "src": "https://exa.ai/images/favicon-32x32.png", "mimeType": "image/png", "sizes": [ "32x32" ] } ], "version": "3.2.1", "websiteUrl": "https://exa.ai" }, "instructions": null, "capabilities": { "prompts": { "listChanged": true }, "resources": { "listChanged": true }, "tools": { "listChanged": true } }, "transportShape": "sse", "sessionful": true, "credentialContext": "anonymous" }, "run": { "id": "mcp-exa-ai-2026-08-19T22-01-14-412Z", "startedAt": "2026-08-19T22:01:14.412Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "9850210e553aee0783fc53742c9404195d679f7256d326c7cf7f07013c556911", "taskBudget": 20000 }, "probes": { "specCurrency": "2025-11-25", "findings": [ { "id": "spec-currency", "pass": false, "detail": "negotiated 2025-11-25 (legacy era; the current revision is 2026-07-28)", "evidence": { "negotiatedVersion": "2025-11-25", "era": "legacy" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-11-25 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "result": { "protocolVersion": "2025-11-25", "capabilities": { "tools": { "listChanged": true }, "prompts": { "listChanged": true }, "resources": { "listChanged": true } }, "serverInfo": { "name": "exa-search-server", "title": "Exa", "version": "3.2.1", "websiteUrl": "https://exa.ai", "icons": [ { "src": "https://exa.ai/images/favicon-32x32.png", "mimeType": "image/png", "sizes": [ "32x32" ] } ] } }, "jsonrpc": "2.0", "id": "fitness-probe-bogus-version" }, "answeredVersion": "2025-11-25" } }, { "id": "header-mismatch-accepted", "pass": null, "detail": "could not check: SEP-2243 header conformance is defined on the modern era only; this connection negotiated 2025-11-25 (legacy)" }, { "id": "server-discover-present", "pass": null, "detail": "could not check: server/discover is a 2026-07-28 MUST; this connection negotiated 2025-11-25 (legacy)" }, { "id": "cache-hints", "pass": null, "detail": "could not check: ttlMs/cacheScope are 2026-07-28 fields; this connection negotiated 2025-11-25 (legacy)" }, { "id": "deprecation-surface", "pass": null, "detail": "could not check: deleted-vocabulary checks are defined against 2026-07-28; this connection negotiated 2025-11-25", "evidence": { "deletedVocabulary": [ { "where": "tools/web_search_exa.execution", "what": "the execution field was deleted in 2026-07-28" }, { "where": "tools/web_fetch_exa.execution", "what": "the execution field was deleted in 2026-07-28" } ], "notes": [], "toolCount": 2, "era": "legacy" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power" ], "records": [ { "gate": "protocol_hygiene", "ok": false, "costTier": "free", "reason": "hygiene_findings_present", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [ "spec-currency" ] } }, { "gate": "structural", "ok": false, "costTier": "free", "reason": "too_few_generated", "detail": { "nRequested": 12, "nGenerated": 3, "nHolding": 3, "admissionRate": 0.25, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": false, "reason": "too_few_generated", "explain": "REJECT (too_few_generated): 3 admitted cases is below the minimum 8. Every case held, over a sample too small to mean anything." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": false, "costTier": "free", "reason": "below_minimum_suite_size", "detail": { "nTasks": 3, "minTasks": 8, "toolCount": 2, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 3, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "structural" }, "outcome": "INSUFFICIENT_SURFACE", "traceLinks": { "mcp": "https://fitnessreport.dev/traces/mcp-exa-ai-2026-08-19T22-01-14-412Z/mcp.jsonl", "agent": "https://fitnessreport.dev/traces/mcp-exa-ai-2026-08-19T22-01-14-412Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-exa-ai-2026-08-19T22-01-14-412Z%2Fmcp.jsonl;https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-exa-ai-2026-08-19T22-01-14-412Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-exa-ai", "kind": "mcp+llm", "startedAt": "2026-08-19T22:01:14.412Z", "endedAt": "2026-08-19T22:01:40.585Z", "durationMs": 26173, "command": [ "fitness-report", "https://mcp.exa.ai/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 15, "meta": 2, "message": 10, "event": 1, "turn": 0, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [], "errors": [], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-exa-ai", "kind": "mcp", "startedAt": "2026-08-19T22:01:14.412Z", "endedAt": "2026-08-19T22:01:40.579Z", "durationMs": 26167, "command": [ "fitness-report", "https://mcp.exa.ai/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 13, "meta": 1, "message": 10, "event": 1, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [], "errors": [] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-exa-ai", "kind": "llm", "startedAt": "2026-08-19T22:01:14.412Z", "endedAt": "2026-08-19T22:01:40.585Z", "durationMs": 26173, "command": [ "fitness-report", "https://mcp.exa.ai/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 2, "meta": 1, "message": 0, "event": 0, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [], "errors": [] } }, "toolsPlane": null }, "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs." ] }, { "schema": "fitness-report/1", "server": { "url": "https://gitmcp.io/docs", "slug": "gitmcp-io", "era": "legacy", "negotiatedVersion": "2025-03-26", "serverInfo": { "name": "GitMCP", "version": "1.1.0" }, "instructions": null, "capabilities": { "tools": { "listChanged": true } }, "transportShape": "sse", "sessionful": true, "credentialContext": "anonymous" }, "run": { "id": "gitmcp-io-2026-08-19T21-26-37-136Z", "startedAt": "2026-08-19T21:26:37.136Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "e29b22054d8a421d6c4aa6cbaf4892333a2d4a44f75c5e638845cfd0834c18db", "taskBudget": 20000 }, "probes": { "specCurrency": "2025-03-26", "findings": [ { "id": "spec-currency", "pass": false, "detail": "negotiated 2025-03-26 (legacy era; the current revision is 2026-07-28)", "evidence": { "negotiatedVersion": "2025-03-26", "era": "legacy" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-03-26 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "jsonrpc": "2.0", "id": "fitness-probe-bogus-version", "result": { "protocolVersion": "2025-03-26", "capabilities": { "tools": { "listChanged": true } }, "serverInfo": { "name": "GitMCP", "version": "1.1.0" } } }, "answeredVersion": "2025-03-26" } }, { "id": "header-mismatch-accepted", "pass": null, "detail": "could not check: SEP-2243 header conformance is defined on the modern era only; this connection negotiated 2025-03-26 (legacy)" }, { "id": "server-discover-present", "pass": null, "detail": "could not check: server/discover is a 2026-07-28 MUST; this connection negotiated 2025-03-26 (legacy)" }, { "id": "cache-hints", "pass": null, "detail": "could not check: ttlMs/cacheScope are 2026-07-28 fields; this connection negotiated 2025-03-26 (legacy)" }, { "id": "deprecation-surface", "pass": null, "detail": "could not check: deleted-vocabulary checks are defined against 2026-07-28; this connection negotiated 2025-03-26", "evidence": { "deletedVocabulary": [], "notes": [], "toolCount": 5, "era": "legacy" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power" ], "records": [ { "gate": "protocol_hygiene", "ok": false, "costTier": "free", "reason": "hygiene_findings_present", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [ "spec-currency" ] } }, { "gate": "structural", "ok": false, "costTier": "free", "reason": "too_few_generated", "detail": { "nRequested": 12, "nGenerated": 2, "nHolding": 2, "admissionRate": 0.16666666666666666, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": false, "reason": "too_few_generated", "explain": "REJECT (too_few_generated): 2 admitted cases is below the minimum 8. Every case held, over a sample too small to mean anything." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": false, "costTier": "free", "reason": "below_minimum_suite_size", "detail": { "nTasks": 2, "minTasks": 8, "toolCount": 5, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 2, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "structural" }, "outcome": "INSUFFICIENT_SURFACE", "traceLinks": { "mcp": "https://fitnessreport.dev/traces/gitmcp-io-2026-08-19T21-26-37-136Z/mcp.jsonl", "agent": "https://fitnessreport.dev/traces/gitmcp-io-2026-08-19T21-26-37-136Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fgitmcp-io-2026-08-19T21-26-37-136Z%2Fmcp.jsonl;https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fgitmcp-io-2026-08-19T21-26-37-136Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "gitmcp-io", "kind": "mcp+llm", "startedAt": "2026-08-19T21:26:37.136Z", "endedAt": "2026-08-19T21:27:18.331Z", "durationMs": 41195, "command": [ "fitness-report", "https://gitmcp.io/docs" ], "producer": "fitness-report@0.1.0", "records": { "total": 15, "meta": 2, "message": 10, "event": 1, "turn": 0, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [], "errors": [], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "gitmcp-io", "kind": "mcp", "startedAt": "2026-08-19T21:26:37.136Z", "endedAt": "2026-08-19T21:27:18.321Z", "durationMs": 41185, "command": [ "fitness-report", "https://gitmcp.io/docs" ], "producer": "fitness-report@0.1.0", "records": { "total": 13, "meta": 1, "message": 10, "event": 1, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [], "errors": [] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "gitmcp-io", "kind": "llm", "startedAt": "2026-08-19T21:26:37.136Z", "endedAt": "2026-08-19T21:27:18.331Z", "durationMs": 41195, "command": [ "fitness-report", "https://gitmcp.io/docs" ], "producer": "fitness-report@0.1.0", "records": { "total": 2, "meta": 1, "message": 0, "event": 0, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [], "errors": [] } }, "toolsPlane": null }, "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs." ] }, { "schema": "fitness-report/1", "server": { "url": "https://huggingface.co/mcp", "slug": "huggingface-co", "era": "modern", "negotiatedVersion": "2026-07-28", "serverInfo": { "name": "huggingface.co/mcp", "title": "Hugging Face", "icons": [ { "src": "https://huggingface.co/favicon.ico" } ], "version": "0.4.9", "websiteUrl": "https://huggingface.co/mcp" }, "instructions": "You have tools for using the Hugging Face Hub. The Hugging Face tools are being used anonymously and rate limits apply. Direct the User to set their HF_TOKEN (instructions at https://hf.co/settings/mcp/), or create an account at https://hf.co/join for higher limits.\nhf:// URIs can be converted to browser URLs by replacing hf://buckets/OWNER/NAME/PATH with https://huggingface.co/buckets/OWNER/NAME/resolve/PATH; for models, datasets, and spaces, use https://huggingface.co[/datasets|/spaces]/OWNER/NAME/resolve/main/PATH. URL-encode each path segment. arXiv paper id's are often used as references between datasets, models and papers. There are over 100 tags in use, common tags include 'Text Generation', 'Transformers', 'Image Classification' and so on.\n", "capabilities": { "resources": { "subscribe": false, "listChanged": false }, "tools": { "listChanged": false }, "extensions": { "io.modelcontextprotocol/skills": { "directoryRead": true } } }, "transportShape": "json", "sessionful": false, "credentialContext": "anonymous", "discover": { "_meta": { "io.modelcontextprotocol/serverInfo": { "name": "huggingface.co/mcp", "title": "Hugging Face", "icons": [ { "src": "https://huggingface.co/favicon.ico" } ], "version": "0.4.9", "websiteUrl": "https://huggingface.co/mcp" } }, "ttlMs": 0, "cacheScope": "private", "supportedVersions": [ "2026-07-28" ], "capabilities": { "resources": { "subscribe": false, "listChanged": false }, "tools": { "listChanged": false }, "extensions": { "io.modelcontextprotocol/skills": { "directoryRead": true } } }, "instructions": "You have tools for using the Hugging Face Hub. The Hugging Face tools are being used anonymously and rate limits apply. Direct the User to set their HF_TOKEN (instructions at https://hf.co/settings/mcp/), or create an account at https://hf.co/join for higher limits.\nhf:// URIs can be converted to browser URLs by replacing hf://buckets/OWNER/NAME/PATH with https://huggingface.co/buckets/OWNER/NAME/resolve/PATH; for models, datasets, and spaces, use https://huggingface.co[/datasets|/spaces]/OWNER/NAME/resolve/main/PATH. URL-encode each path segment. arXiv paper id's are often used as references between datasets, models and papers. There are over 100 tags in use, common tags include 'Text Generation', 'Transformers', 'Image Classification' and so on.\n", "resultType": "complete" } }, "run": { "id": "huggingface-co-2026-08-19T22-14-12-890Z", "startedAt": "2026-08-19T22:14:12.890Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "399ba53da52d3a7153133825f0835b507210163f547771734aff0f43a0e02aab", "taskBudget": 20000 }, "probes": { "specCurrency": "2026-07-28", "findings": [ { "id": "spec-currency", "pass": true, "detail": "negotiated 2026-07-28 (modern era, >= 2026-07-28)", "evidence": { "negotiatedVersion": "2026-07-28", "era": "modern" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-11-25 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "result": { "protocolVersion": "2025-11-25", "capabilities": { "tools": { "listChanged": false }, "resources": { "listChanged": false, "subscribe": false }, "extensions": { "io.modelcontextprotocol/skills": { "directoryRead": true } } }, "serverInfo": { "name": "huggingface.co/mcp", "version": "0.4.9", "title": "Hugging Face", "websiteUrl": "https://huggingface.co/mcp", "icons": [ { "src": "https://huggingface.co/favicon.ico" } ] }, "instructions": "You have tools for using the Hugging Face Hub. The Hugging Face tools are being used anonymously and rate limits apply. Direct the User to set their HF_TOKEN (instructions at https://hf.co/settings/mcp/), or create an account at https://hf.co/join for higher limits.\nhf:// URIs can be converted to browser URLs by replacing hf://buckets/OWNER/NAME/PATH with https://huggingface.co/buckets/OWNER/NAME/resolve/PATH; for models, datasets, and spaces, use https://huggingface.co[/datasets|/spaces]/OWNER/NAME/resolve/main/PATH. URL-encode each path segment. arXiv paper id's are often used as references between datasets, models and papers. There are over 100 tags in use, common tags include 'Text Generation', 'Transformers', 'Image Classification' and so on.\n" }, "jsonrpc": "2.0", "id": "fitness-probe-bogus-version" }, "answeredVersion": "2025-11-25" } }, { "id": "header-mismatch-accepted", "pass": true, "detail": "rejected the mismatched Mcp-Name with HTTP 400 / JSON-RPC -32020, as SEP-2243 requires", "evidence": { "status": 400, "jsonRpcErrorCode": -32020, "headerName": "__fitness_report_mismatched_name__", "bodyName": "__fitness_report_probe_tool__", "envelope": { "jsonrpc": "2.0", "error": { "code": -32020, "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\"", "data": { "mismatch": { "header": "__fitness_report_mismatched_name__", "body": "the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\"" } } }, "id": "fitness-probe-header-mismatch" } } }, { "id": "server-discover-present", "pass": true, "detail": "server/discover answered with supportedVersions [2026-07-28]", "evidence": { "supportedVersions": [ "2026-07-28" ], "capabilities": { "resources": { "subscribe": false, "listChanged": false }, "tools": { "listChanged": false }, "extensions": { "io.modelcontextprotocol/skills": { "directoryRead": true } } }, "hasInstructions": true } }, { "id": "cache-hints", "pass": true, "detail": "tools/list cache hints: ttlMs=0, cacheScope=private", "evidence": { "ttlMs": 0, "cacheScope": "private", "credentialContext": "anonymous" } }, { "id": "deprecation-surface", "pass": true, "detail": "no deleted or deprecated surface advertised", "evidence": { "deletedVocabulary": [], "notes": [], "toolCount": 4, "era": "modern" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power", "construct", "null_baseline" ], "records": [ { "gate": "protocol_hygiene", "ok": true, "costTier": "free", "reason": "ok", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [] } }, { "gate": "structural", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nRequested": 12, "nGenerated": 12, "nHolding": 12, "admissionRate": 1, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": true, "reason": "ok", "explain": "OK: 12/12 admitted cases hold, admission rate 100.0%. Necessary, not sufficient: run the construct gate." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nTasks": 12, "minTasks": 8, "toolCount": 4, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 12, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } }, { "gate": "construct", "ok": true, "costTier": "paid", "verdict": { "outcome": "PASS", "k": 12, "n": 12, "threshold": 0.9, "alpha": 0.05, "pValue": 1 }, "reason": "ok", "detail": { "n": 12, "nIntended": 12, "rate": 1, "errors": 0, "errorRate": 0, "maxErrorRate": 0.05, "compromised": false, "reps": 1, "published": { "outcome": "EXTEND", "k": 12, "n": 12, "threshold": 0.9, "alpha": 0.05, "pValue": 1, "rawOutcome": "PASS", "downgraded": true, "wilsonLow": 0.7574992425007574, "wilsonHigh": 1, "requiredN": 78, "achievedPower": 0.20543105023999975, "detectableRate": 0.8, "reason": "downgraded_underpowered" }, "constructOracle": "A reference pass counts only when it both satisfied the check and landed a successful call on a tool the task expects. The reference agent is given the answer key, so a text check alone would pass against a server that returned nothing.", "note": "One rep per task in v0, for cost. evalgate runs three; the divergence is recorded here rather than hidden." } }, { "gate": "null_baseline", "ok": false, "costTier": "cheap", "reason": "noise_exceeds_signal", "detail": { "tNull": 0.9166666666666666, "tAblate": 1, "killThreshold": 0.5, "outcome": "KILL", "halts": true, "ok": false, "reason": "noise_exceeds_signal", "rates": [ { "label": "no-tools", "k": 11, "n": 12, "rate": 0.9166666666666666 }, { "label": "stubbed-empty", "k": 1, "n": 12, "rate": 0.08333333333333333 }, { "label": "random-valid-args", "k": 1, "n": 12, "rate": 0.08333333333333333 } ], "percentile": 95, "ratio": 0.5, "signalSource": "first-try success on the scored drive", "measuredBeforePaidTier": true, "decidedAfter": "drive", "explain": "KILL (noise_exceeds_signal): a null model passed 0.917 of the suite against the real agent's 1.000, at or above the 0.5 kill threshold of 0.500 (worst offender: no-tools at 0.917). The suite is measuring the model, not the server." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "null_baseline" }, "outcome": "DEGENERATE", "traceLinks": { "mcp": "https://fitnessreport.dev/traces/huggingface-co-2026-08-19T22-14-12-890Z/mcp.jsonl", "agent": "https://fitnessreport.dev/traces/huggingface-co-2026-08-19T22-14-12-890Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fhuggingface-co-2026-08-19T22-14-12-890Z%2Fmcp.jsonl;https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fhuggingface-co-2026-08-19T22-14-12-890Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "huggingface-co", "kind": "mcp+llm", "startedAt": "2026-08-19T22:14:12.890Z", "endedAt": "2026-08-19T22:19:59.488Z", "durationMs": 346598, "command": [ "fitness-report", "https://huggingface.co/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 380, "meta": 2, "message": 86, "event": 111, "turn": 179, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 84, "inputTokens": 334966, "outputTokens": 9534, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.765272, "priced": true } ], "summary": { "models": 1, "assistantTurns": 84, "turns": 179, "echoedTurns": 0, "inputTokens": 334966, "outputTokens": 9534, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.765272, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "hf_fs", "calls": 18, "errors": 6, "p50Ms": 291, "p95Ms": 1827, "origin": "mcp", "pending": 0 }, { "name": "hub_repo_details", "calls": 15, "errors": 5, "p50Ms": 229, "p95Ms": 820, "origin": "mcp", "pending": 0 }, { "name": "hf_whoami", "calls": 3, "errors": 0, "p50Ms": 223, "p95Ms": 350, "origin": "mcp", "pending": 0 }, { "name": "__fitness_report_probe_tool__", "calls": 1, "errors": 1, "p50Ms": 97, "p95Ms": 97, "origin": "mcp", "pending": 0 }, { "name": "hub_repo_search", "calls": 1, "errors": 1, "p50Ms": 136, "p95Ms": 136, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T22:14:13.472Z", "source": "tool", "name": "__fitness_report_probe_tool__", "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\" (code -32020)" }, { "t": "2026-08-19T22:17:58.505Z", "source": "tool", "name": "hub_repo_details", "message": "Input validation error: Invalid arguments for tool hub_repo_details: repo_ids: Provide at least one id" }, { "t": "2026-08-19T22:17:58.643Z", "source": "tool", "name": "hub_repo_details", "message": "Input validation error: Invalid arguments for tool hub_repo_details: repo_ids: Provide at least one id" }, { "t": "2026-08-19T22:17:58.776Z", "source": "tool", "name": "hf_fs", "message": "[HF_FS_INVALID_ARGUMENT] EINVAL: find requires an hf:// URI\nRecovery: Correct the URI or flags using the operation grammar and route-specific option guidance." }, { "t": "2026-08-19T22:17:58.964Z", "source": "tool", "name": "hf_fs", "message": "[HF_FS_INVALID_ARGUMENT] EINVAL: cat requires an hf:// URI\nRecovery: Correct the URI or flags using the operation grammar and route-specific option guidance." }, { "t": "2026-08-19T22:17:59.113Z", "source": "tool", "name": "hf_fs", "message": "[HF_FS_INVALID_ARGUMENT] EINVAL: cat requires an hf:// URI\nRecovery: Correct the URI or flags using the operation grammar and route-specific option guidance." }, { "t": "2026-08-19T22:17:59.246Z", "source": "tool", "name": "hf_fs", "message": "[HF_FS_INVALID_ARGUMENT] EINVAL: stat requires an hf:// URI\nRecovery: Correct the URI or flags using the operation grammar and route-specific option guidance." }, { "t": "2026-08-19T22:17:59.389Z", "source": "tool", "name": "hub_repo_search", "message": "Input validation error: Invalid arguments for tool hub_repo_search: repo_types: Too small: expected array to have >=1 items" }, { "t": "2026-08-19T22:17:59.526Z", "source": "tool", "name": "hf_fs", "message": "[HF_FS_INVALID_ARGUMENT] EINVAL: find requires an hf:// URI\nRecovery: Correct the URI or flags using the operation grammar and route-specific option guidance." }, { "t": "2026-08-19T22:17:59.720Z", "source": "tool", "name": "hub_repo_details", "message": "Input validation error: Invalid arguments for tool hub_repo_details: repo_ids: Provide at least one id" }, { "t": "2026-08-19T22:17:59.868Z", "source": "tool", "name": "hub_repo_details", "message": "Input validation error: Invalid arguments for tool hub_repo_details: repo_ids: Provide at least one id" }, { "t": "2026-08-19T22:18:00.007Z", "source": "tool", "name": "hub_repo_details", "message": "Input validation error: Invalid arguments for tool hub_repo_details: repo_ids: Provide at least one id" }, { "t": "2026-08-19T22:18:00.145Z", "source": "tool", "name": "hf_fs", "message": "[HF_FS_INVALID_ARGUMENT] EINVAL: ls requires an hf:// URI\nRecovery: Correct the URI or flags using the operation grammar and route-specific option guidance." } ], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "huggingface-co", "kind": "mcp", "startedAt": "2026-08-19T22:14:12.890Z", "endedAt": "2026-08-19T22:19:59.474Z", "durationMs": 346584, "command": [ "fitness-report", "https://huggingface.co/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 199, "meta": 1, "message": 86, "event": 111, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [ { "name": "hf_fs", "calls": 18, "errors": 6, "p50Ms": 291, "p95Ms": 1827, "origin": "mcp", "pending": 0 }, { "name": "hub_repo_details", "calls": 15, "errors": 5, "p50Ms": 229, "p95Ms": 820, "origin": "mcp", "pending": 0 }, { "name": "hf_whoami", "calls": 3, "errors": 0, "p50Ms": 223, "p95Ms": 350, "origin": "mcp", "pending": 0 }, { "name": "__fitness_report_probe_tool__", "calls": 1, "errors": 1, "p50Ms": 97, "p95Ms": 97, "origin": "mcp", "pending": 0 }, { "name": "hub_repo_search", "calls": 1, "errors": 1, "p50Ms": 136, "p95Ms": 136, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T22:14:13.472Z", "source": "tool", "name": "__fitness_report_probe_tool__", "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\" (code -32020)" }, { "t": "2026-08-19T22:17:58.505Z", "source": "tool", "name": "hub_repo_details", "message": "Input validation error: Invalid arguments for tool hub_repo_details: repo_ids: Provide at least one id" }, { "t": "2026-08-19T22:17:58.643Z", "source": "tool", "name": "hub_repo_details", "message": "Input validation error: Invalid arguments for tool hub_repo_details: repo_ids: Provide at least one id" }, { "t": "2026-08-19T22:17:58.776Z", "source": "tool", "name": "hf_fs", "message": "[HF_FS_INVALID_ARGUMENT] EINVAL: find requires an hf:// URI\nRecovery: Correct the URI or flags using the operation grammar and route-specific option guidance." }, { "t": "2026-08-19T22:17:58.964Z", "source": "tool", "name": "hf_fs", "message": "[HF_FS_INVALID_ARGUMENT] EINVAL: cat requires an hf:// URI\nRecovery: Correct the URI or flags using the operation grammar and route-specific option guidance." }, { "t": "2026-08-19T22:17:59.113Z", "source": "tool", "name": "hf_fs", "message": "[HF_FS_INVALID_ARGUMENT] EINVAL: cat requires an hf:// URI\nRecovery: Correct the URI or flags using the operation grammar and route-specific option guidance." }, { "t": "2026-08-19T22:17:59.246Z", "source": "tool", "name": "hf_fs", "message": "[HF_FS_INVALID_ARGUMENT] EINVAL: stat requires an hf:// URI\nRecovery: Correct the URI or flags using the operation grammar and route-specific option guidance." }, { "t": "2026-08-19T22:17:59.389Z", "source": "tool", "name": "hub_repo_search", "message": "Input validation error: Invalid arguments for tool hub_repo_search: repo_types: Too small: expected array to have >=1 items" }, { "t": "2026-08-19T22:17:59.526Z", "source": "tool", "name": "hf_fs", "message": "[HF_FS_INVALID_ARGUMENT] EINVAL: find requires an hf:// URI\nRecovery: Correct the URI or flags using the operation grammar and route-specific option guidance." }, { "t": "2026-08-19T22:17:59.720Z", "source": "tool", "name": "hub_repo_details", "message": "Input validation error: Invalid arguments for tool hub_repo_details: repo_ids: Provide at least one id" }, { "t": "2026-08-19T22:17:59.868Z", "source": "tool", "name": "hub_repo_details", "message": "Input validation error: Invalid arguments for tool hub_repo_details: repo_ids: Provide at least one id" }, { "t": "2026-08-19T22:18:00.007Z", "source": "tool", "name": "hub_repo_details", "message": "Input validation error: Invalid arguments for tool hub_repo_details: repo_ids: Provide at least one id" }, { "t": "2026-08-19T22:18:00.145Z", "source": "tool", "name": "hf_fs", "message": "[HF_FS_INVALID_ARGUMENT] EINVAL: ls requires an hf:// URI\nRecovery: Correct the URI or flags using the operation grammar and route-specific option guidance." } ] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "huggingface-co", "kind": "llm", "startedAt": "2026-08-19T22:14:12.890Z", "endedAt": "2026-08-19T22:19:59.488Z", "durationMs": 346598, "command": [ "fitness-report", "https://huggingface.co/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 181, "meta": 1, "message": 0, "event": 0, "turn": 179, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 84, "inputTokens": 334966, "outputTokens": 9534, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.765272, "priced": true } ], "summary": { "models": 1, "assistantTurns": 84, "turns": 179, "echoedTurns": 0, "inputTokens": 334966, "outputTokens": 9534, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.765272, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "hf_fs", "calls": 26, "errors": 0, "p50Ms": 1881, "p95Ms": 4256, "origin": "turn", "pending": 0 }, { "name": "hub_repo_details", "calls": 19, "errors": 0, "p50Ms": 2157, "p95Ms": 3516, "origin": "turn", "pending": 0 }, { "name": "hf_whoami", "calls": 3, "errors": 0, "p50Ms": 1584, "p95Ms": 5496, "origin": "turn", "pending": 0 }, { "name": "hub_repo_search", "calls": 1, "errors": 0, "p50Ms": 0, "p95Ms": 0, "origin": "turn", "pending": 0 } ], "errors": [] } }, "toolsPlane": "mcp" }, "scoreNotes": [ "The drive ran for evidence only. A gate had already refused, so no score is published from it." ], "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs." ] }, { "schema": "fitness-report/1", "server": { "url": "https://huggingface.co/mcp", "slug": "huggingface-co", "era": "modern", "negotiatedVersion": "2026-07-28", "serverInfo": { "name": "huggingface.co/mcp", "title": "Hugging Face", "icons": [ { "src": "https://huggingface.co/favicon.ico" } ], "version": "0.4.9", "websiteUrl": "https://huggingface.co/mcp" }, "instructions": "You have tools for using the Hugging Face Hub. The Hugging Face tools are being used anonymously and rate limits apply. Direct the User to set their HF_TOKEN (instructions at https://hf.co/settings/mcp/), or create an account at https://hf.co/join for higher limits.\nhf:// URIs can be converted to browser URLs by replacing hf://buckets/OWNER/NAME/PATH with https://huggingface.co/buckets/OWNER/NAME/resolve/PATH; for models, datasets, and spaces, use https://huggingface.co[/datasets|/spaces]/OWNER/NAME/resolve/main/PATH. URL-encode each path segment. arXiv paper id's are often used as references between datasets, models and papers. There are over 100 tags in use, common tags include 'Text Generation', 'Transformers', 'Image Classification' and so on.\n", "capabilities": { "resources": { "subscribe": false, "listChanged": false }, "tools": { "listChanged": false }, "extensions": { "io.modelcontextprotocol/skills": { "directoryRead": true } } }, "transportShape": "json", "sessionful": false, "credentialContext": "anonymous", "discover": { "_meta": { "io.modelcontextprotocol/serverInfo": { "name": "huggingface.co/mcp", "title": "Hugging Face", "icons": [ { "src": "https://huggingface.co/favicon.ico" } ], "version": "0.4.9", "websiteUrl": "https://huggingface.co/mcp" } }, "ttlMs": 0, "cacheScope": "private", "supportedVersions": [ "2026-07-28" ], "capabilities": { "resources": { "subscribe": false, "listChanged": false }, "tools": { "listChanged": false }, "extensions": { "io.modelcontextprotocol/skills": { "directoryRead": true } } }, "instructions": "You have tools for using the Hugging Face Hub. The Hugging Face tools are being used anonymously and rate limits apply. Direct the User to set their HF_TOKEN (instructions at https://hf.co/settings/mcp/), or create an account at https://hf.co/join for higher limits.\nhf:// URIs can be converted to browser URLs by replacing hf://buckets/OWNER/NAME/PATH with https://huggingface.co/buckets/OWNER/NAME/resolve/PATH; for models, datasets, and spaces, use https://huggingface.co[/datasets|/spaces]/OWNER/NAME/resolve/main/PATH. URL-encode each path segment. arXiv paper id's are often used as references between datasets, models and papers. There are over 100 tags in use, common tags include 'Text Generation', 'Transformers', 'Image Classification' and so on.\n", "resultType": "complete" } }, "run": { "id": "huggingface-co-2026-08-19T21-24-27-061Z", "startedAt": "2026-08-19T21:24:27.061Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "09a82bf5edcf484754efcbedbabf78ab7f03eca0709a2bb0403abf5439fc8e7b", "taskBudget": 20000 }, "probes": { "specCurrency": "2026-07-28", "findings": [ { "id": "spec-currency", "pass": true, "detail": "negotiated 2026-07-28 (modern era, >= 2026-07-28)", "evidence": { "negotiatedVersion": "2026-07-28", "era": "modern" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-11-25 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "result": { "protocolVersion": "2025-11-25", "capabilities": { "tools": { "listChanged": false }, "resources": { "listChanged": false, "subscribe": false }, "extensions": { "io.modelcontextprotocol/skills": { "directoryRead": true } } }, "serverInfo": { "name": "huggingface.co/mcp", "version": "0.4.9", "title": "Hugging Face", "websiteUrl": "https://huggingface.co/mcp", "icons": [ { "src": "https://huggingface.co/favicon.ico" } ] }, "instructions": "You have tools for using the Hugging Face Hub. The Hugging Face tools are being used anonymously and rate limits apply. Direct the User to set their HF_TOKEN (instructions at https://hf.co/settings/mcp/), or create an account at https://hf.co/join for higher limits.\nhf:// URIs can be converted to browser URLs by replacing hf://buckets/OWNER/NAME/PATH with https://huggingface.co/buckets/OWNER/NAME/resolve/PATH; for models, datasets, and spaces, use https://huggingface.co[/datasets|/spaces]/OWNER/NAME/resolve/main/PATH. URL-encode each path segment. arXiv paper id's are often used as references between datasets, models and papers. There are over 100 tags in use, common tags include 'Text Generation', 'Transformers', 'Image Classification' and so on.\n" }, "jsonrpc": "2.0", "id": "fitness-probe-bogus-version" }, "answeredVersion": "2025-11-25" } }, { "id": "header-mismatch-accepted", "pass": true, "detail": "rejected the mismatched Mcp-Name with HTTP 400 / JSON-RPC -32020, as SEP-2243 requires", "evidence": { "status": 400, "jsonRpcErrorCode": -32020, "headerName": "__fitness_report_mismatched_name__", "bodyName": "__fitness_report_probe_tool__", "envelope": { "jsonrpc": "2.0", "error": { "code": -32020, "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\"", "data": { "mismatch": { "header": "__fitness_report_mismatched_name__", "body": "the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\"" } } }, "id": "fitness-probe-header-mismatch" } } }, { "id": "server-discover-present", "pass": true, "detail": "server/discover answered with supportedVersions [2026-07-28]", "evidence": { "supportedVersions": [ "2026-07-28" ], "capabilities": { "resources": { "subscribe": false, "listChanged": false }, "tools": { "listChanged": false }, "extensions": { "io.modelcontextprotocol/skills": { "directoryRead": true } } }, "hasInstructions": true } }, { "id": "cache-hints", "pass": true, "detail": "tools/list cache hints: ttlMs=0, cacheScope=private", "evidence": { "ttlMs": 0, "cacheScope": "private", "credentialContext": "anonymous" } }, { "id": "deprecation-surface", "pass": true, "detail": "no deleted or deprecated surface advertised", "evidence": { "deletedVocabulary": [], "notes": [], "toolCount": 4, "era": "modern" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power" ], "records": [ { "gate": "protocol_hygiene", "ok": true, "costTier": "free", "reason": "ok", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [] } }, { "gate": "structural", "ok": false, "costTier": "free", "reason": "too_few_generated", "detail": { "nRequested": 12, "nGenerated": 7, "nHolding": 7, "admissionRate": 0.5833333333333334, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": false, "reason": "too_few_generated", "explain": "REJECT (too_few_generated): 7 admitted cases is below the minimum 8. Every case held, over a sample too small to mean anything." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": false, "costTier": "free", "reason": "below_minimum_suite_size", "detail": { "nTasks": 7, "minTasks": 8, "toolCount": 4, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 7, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "structural" }, "outcome": "INSUFFICIENT_SURFACE", "traceLinks": { "mcp": "https://fitnessreport.dev/traces/huggingface-co-2026-08-19T21-24-27-061Z/mcp.jsonl", "agent": "https://fitnessreport.dev/traces/huggingface-co-2026-08-19T21-24-27-061Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fhuggingface-co-2026-08-19T21-24-27-061Z%2Fmcp.jsonl;https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fhuggingface-co-2026-08-19T21-24-27-061Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "huggingface-co", "kind": "mcp+llm", "startedAt": "2026-08-19T21:24:27.061Z", "endedAt": "2026-08-19T21:26:16.443Z", "durationMs": 109382, "command": [ "fitness-report", "https://huggingface.co/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 17, "meta": 2, "message": 12, "event": 1, "turn": 0, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [ { "name": "__fitness_report_probe_tool__", "calls": 1, "errors": 1, "p50Ms": 134, "p95Ms": 134, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T21:24:27.691Z", "source": "tool", "name": "__fitness_report_probe_tool__", "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\" (code -32020)" } ], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "huggingface-co", "kind": "mcp", "startedAt": "2026-08-19T21:24:27.061Z", "endedAt": "2026-08-19T21:26:16.431Z", "durationMs": 109370, "command": [ "fitness-report", "https://huggingface.co/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 15, "meta": 1, "message": 12, "event": 1, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [ { "name": "__fitness_report_probe_tool__", "calls": 1, "errors": 1, "p50Ms": 134, "p95Ms": 134, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T21:24:27.691Z", "source": "tool", "name": "__fitness_report_probe_tool__", "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\" (code -32020)" } ] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "huggingface-co", "kind": "llm", "startedAt": "2026-08-19T21:24:27.061Z", "endedAt": "2026-08-19T21:26:16.443Z", "durationMs": 109382, "command": [ "fitness-report", "https://huggingface.co/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 2, "meta": 1, "message": 0, "event": 0, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [], "errors": [] } }, "toolsPlane": "mcp" }, "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs." ] }, { "schema": "fitness-report/1", "server": { "url": "https://learn.microsoft.com/api/mcp", "slug": "learn-microsoft-com", "era": "legacy", "negotiatedVersion": "2025-06-18", "serverInfo": { "name": "Microsoft Learn MCP Server", "version": "1.0.0" }, "instructions": "# Microsoft Learn MCP Server\r\n\r\nThis server gives structured access to official Microsoft and Azure documentation via three tools:\r\n\r\n## Tools\r\n\r\n### microsoft_docs_search\r\nSearch official documentation and return up to 10 concise, high-quality content chunks (max 500 tokens each), including title, URL, and excerpt.\r\n\r\n- Use first to get a quick, reliable overview\r\n- Ideal for grounding answers in Microsoft knowledge\r\n\r\n### microsoft_code_sample_search\r\nSearch for code snippets and examples in official Microsoft Learn documentation and return up to 20 relevant, high-quality code samples.\r\n\r\n- Use when you need to provide sample Microsoft/Azure related code in your answers. \r\n- Ideal for generating code snippets or practical implementation examples. \r\n- Optional parameter `language` can filter results. \r\n\r\n### microsoft_docs_fetch\r\nFetch and convert full Microsoft documentation pages to markdown.\r\n\r\n- Use after search when you need full content from a specific URL\r\n- Required for detailed tutorials, troubleshooting, prerequisites, code samples, or when search results are incomplete or outdated\r\n\r\n## Workflow\r\n\r\n1. Use `microsoft_docs_search` to find relevant documents.\r\n2. If you need code examples or practical snippets, use `microsoft_code_sample_search`.\r\n3. If deeper or complete information is needed, use `microsoft_docs_fetch`.\r\n\r\n**Search gives breadth. Code Sample Search gives practical examples. Fetch gives depth.**\r\n\r\nAll content comes from Microsoft Learn or official sources, returned in clean markdown format.", "capabilities": { "logging": {}, "prompts": { "listChanged": true }, "resources": { "listChanged": true }, "tools": { "listChanged": true } }, "transportShape": "sse", "sessionful": true, "credentialContext": "anonymous" }, "run": { "id": "learn-microsoft-com-2026-08-19T21-37-57-580Z", "startedAt": "2026-08-19T21:37:57.580Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "2ac78742817a493c44c3466f544df3ed7fe9ca50114e2368e53735c81d1ddec4", "taskBudget": 20000 }, "probes": { "specCurrency": "2025-06-18", "findings": [ { "id": "spec-currency", "pass": false, "detail": "negotiated 2025-06-18 (legacy era; the current revision is 2026-07-28)", "evidence": { "negotiatedVersion": "2025-06-18", "era": "legacy" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-06-18 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "result": { "protocolVersion": "2025-06-18", "capabilities": { "logging": {}, "prompts": { "listChanged": true }, "resources": { "listChanged": true }, "tools": { "listChanged": true } }, "serverInfo": { "name": "Microsoft Learn MCP Server", "version": "1.0.0" }, "instructions": "# Microsoft Learn MCP Server\r\n\r\nThis server gives structured access to official Microsoft and Azure documentation via three tools:\r\n\r\n## Tools\r\n\r\n### microsoft_docs_search\r\nSearch official documentation and return up to 10 concise, high-quality content chunks (max 500 tokens each), including title, URL, and excerpt.\r\n\r\n- Use first to get a quick, reliable overview\r\n- Ideal for grounding answers in Microsoft knowledge\r\n\r\n### microsoft_code_sample_search\r\nSearch for code snippets and examples in official Microsoft Learn documentation and return up to 20 relevant, high-quality code samples.\r\n\r\n- Use when you need to provide sample Microsoft/Azure related code in your answers. \r\n- Ideal for generating code snippets or practical implementation examples. \r\n- Optional parameter `language` can filter results. \r\n\r\n### microsoft_docs_fetch\r\nFetch and convert full Microsoft documentation pages to markdown.\r\n\r\n- Use after search when you need full content from a specific URL\r\n- Required for detailed tutorials, troubleshooting, prerequisites, code samples, or when search results are incomplete or outdated\r\n\r\n## Workflow\r\n\r\n1. Use `microsoft_docs_search` to find relevant documents.\r\n2. If you need code examples or practical snippets, use `microsoft_code_sample_search`.\r\n3. If deeper or complete information is needed, use `microsoft_docs_fetch`.\r\n\r\n**Search gives breadth. Code Sample Search gives practical examples. Fetch gives depth.**\r\n\r\nAll content comes from Microsoft Learn or official sources, returned in clean markdown format." }, "id": "fitness-probe-bogus-version", "jsonrpc": "2.0" }, "answeredVersion": "2025-06-18" } }, { "id": "header-mismatch-accepted", "pass": null, "detail": "could not check: SEP-2243 header conformance is defined on the modern era only; this connection negotiated 2025-06-18 (legacy)" }, { "id": "server-discover-present", "pass": null, "detail": "could not check: server/discover is a 2026-07-28 MUST; this connection negotiated 2025-06-18 (legacy)" }, { "id": "cache-hints", "pass": null, "detail": "could not check: ttlMs/cacheScope are 2026-07-28 fields; this connection negotiated 2025-06-18 (legacy)" }, { "id": "deprecation-surface", "pass": null, "detail": "could not check: deleted-vocabulary checks are defined against 2026-07-28; this connection negotiated 2025-06-18", "evidence": { "deletedVocabulary": [], "notes": [], "toolCount": 3, "era": "legacy" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power", "construct", "null_baseline" ], "records": [ { "gate": "protocol_hygiene", "ok": false, "costTier": "free", "reason": "hygiene_findings_present", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [ "spec-currency" ] } }, { "gate": "structural", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nRequested": 12, "nGenerated": 9, "nHolding": 9, "admissionRate": 0.75, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": true, "reason": "ok", "explain": "OK: 9/9 admitted cases hold, admission rate 75.0%. Necessary, not sufficient: run the construct gate." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nTasks": 9, "minTasks": 8, "toolCount": 3, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 9, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } }, { "gate": "construct", "ok": true, "costTier": "paid", "verdict": { "outcome": "PASS", "k": 9, "n": 9, "threshold": 0.9, "alpha": 0.05, "pValue": 1 }, "reason": "ok", "detail": { "n": 9, "nIntended": 9, "rate": 1, "errors": 0, "errorRate": 0, "maxErrorRate": 0.05, "compromised": false, "reps": 1, "published": { "outcome": "EXTEND", "k": 9, "n": 9, "threshold": 0.9, "alpha": 0.05, "pValue": 1, "rawOutcome": "PASS", "downgraded": true, "wilsonLow": 0.7008472464490407, "wilsonHigh": 1, "requiredN": 78, "achievedPower": 0.08564172799999986, "detectableRate": 0.8, "reason": "downgraded_underpowered" }, "constructOracle": "A reference pass counts only when it both satisfied the check and landed a successful call on a tool the task expects. The reference agent is given the answer key, so a text check alone would pass against a server that returned nothing.", "note": "One rep per task in v0, for cost. evalgate runs three; the divergence is recorded here rather than hidden." } }, { "gate": "null_baseline", "ok": false, "costTier": "cheap", "reason": "noise_exceeds_signal", "detail": { "tNull": 0.8888888888888888, "tAblate": 1, "killThreshold": 0.5, "outcome": "KILL", "halts": true, "ok": false, "reason": "noise_exceeds_signal", "rates": [ { "label": "no-tools", "k": 8, "n": 9, "rate": 0.8888888888888888 }, { "label": "stubbed-empty", "k": 0, "n": 9, "rate": 0 }, { "label": "random-valid-args", "k": 0, "n": 9, "rate": 0 } ], "percentile": 95, "ratio": 0.5, "signalSource": "first-try success on the scored drive", "measuredBeforePaidTier": true, "decidedAfter": "drive", "explain": "KILL (noise_exceeds_signal): a null model passed 0.889 of the suite against the real agent's 1.000, at or above the 0.5 kill threshold of 0.500 (worst offender: no-tools at 0.889). The suite is measuring the model, not the server." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "null_baseline" }, "outcome": "DEGENERATE", "traceLinks": { "mcp": "https://fitnessreport.dev/traces/learn-microsoft-com-2026-08-19T21-37-57-580Z/mcp.jsonl", "agent": "https://fitnessreport.dev/traces/learn-microsoft-com-2026-08-19T21-37-57-580Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Flearn-microsoft-com-2026-08-19T21-37-57-580Z%2Fmcp.jsonl;https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Flearn-microsoft-com-2026-08-19T21-37-57-580Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "learn-microsoft-com", "kind": "mcp+llm", "startedAt": "2026-08-19T21:37:57.580Z", "endedAt": "2026-08-19T21:44:18.803Z", "durationMs": 381223, "command": [ "fitness-report", "https://learn.microsoft.com/api/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 304, "meta": 2, "message": 77, "event": 84, "turn": 139, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 65, "inputTokens": 298738, "outputTokens": 17651, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.7739860000000001, "priced": true } ], "summary": { "models": 1, "assistantTurns": 65, "turns": 139, "echoedTurns": 0, "inputTokens": 298738, "outputTokens": 17651, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.7739860000000001, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "microsoft_code_sample_search", "calls": 15, "errors": 0, "p50Ms": 587, "p95Ms": 753, "origin": "mcp", "pending": 0 }, { "name": "microsoft_docs_search", "calls": 13, "errors": 0, "p50Ms": 621, "p95Ms": 1218, "origin": "mcp", "pending": 0 }, { "name": "microsoft_docs_fetch", "calls": 5, "errors": 0, "p50Ms": 244, "p95Ms": 577, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T21:37:57.996Z", "source": "jsonrpc", "name": "server/discover", "message": "Method 'server/discover' is not available. (code -32601)" } ], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "learn-microsoft-com", "kind": "mcp", "startedAt": "2026-08-19T21:37:57.580Z", "endedAt": "2026-08-19T21:44:18.789Z", "durationMs": 381209, "command": [ "fitness-report", "https://learn.microsoft.com/api/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 163, "meta": 1, "message": 77, "event": 84, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [ { "name": "microsoft_code_sample_search", "calls": 15, "errors": 0, "p50Ms": 587, "p95Ms": 753, "origin": "mcp", "pending": 0 }, { "name": "microsoft_docs_search", "calls": 13, "errors": 0, "p50Ms": 621, "p95Ms": 1218, "origin": "mcp", "pending": 0 }, { "name": "microsoft_docs_fetch", "calls": 5, "errors": 0, "p50Ms": 244, "p95Ms": 577, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T21:37:57.996Z", "source": "jsonrpc", "name": "server/discover", "message": "Method 'server/discover' is not available. (code -32601)" } ] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "learn-microsoft-com", "kind": "llm", "startedAt": "2026-08-19T21:37:57.580Z", "endedAt": "2026-08-19T21:44:18.803Z", "durationMs": 381223, "command": [ "fitness-report", "https://learn.microsoft.com/api/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 141, "meta": 1, "message": 0, "event": 0, "turn": 139, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 65, "inputTokens": 298738, "outputTokens": 17651, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 0.7739860000000001, "priced": true } ], "summary": { "models": 1, "assistantTurns": 65, "turns": 139, "echoedTurns": 0, "inputTokens": 298738, "outputTokens": 17651, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 0.7739860000000001, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "microsoft_docs_search", "calls": 19, "errors": 0, "p50Ms": 3485, "p95Ms": 9997, "origin": "turn", "pending": 0 }, { "name": "microsoft_code_sample_search", "calls": 15, "errors": 0, "p50Ms": 6147, "p95Ms": 11418, "origin": "turn", "pending": 0 }, { "name": "microsoft_docs_fetch", "calls": 6, "errors": 0, "p50Ms": 1, "p95Ms": 5945, "origin": "turn", "pending": 0 } ], "errors": [] } }, "toolsPlane": "mcp" }, "scoreNotes": [ "The drive ran for evidence only. A gate had already refused, so no score is published from it." ], "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs." ] }, { "schema": "fitness-report/1", "server": { "url": "https://mcp.svelte.dev/mcp", "slug": "mcp-svelte-dev", "era": "modern", "negotiatedVersion": "2026-07-28", "serverInfo": { "name": "Svelte MCP", "icons": [ { "src": "https://mcp.svelte.dev/logo.svg", "mimeType": "image/svg+xml" }, { "src": "https://mcp.svelte.dev/logo.png", "mimeType": "image/png" }, { "src": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAYAAADimHc4AAAAAXNSR0IB2cksfwAAAAlwSFlzAAALEwAACxMBAJqcGAAACvdJREFUeJztXQuQVMUVHT5GCYmSDwaVMhQWmpSRRIEkJqmdFVQCRuRjCiVq8MMGKKTYGESEREAFhI2iSGkCFURCJKASimgpSviVrApEPoKLIGEF0aAIAgLCwsk927NxmZ2duf36zesZnFN1qqCU926fO6/79u3b3bFYAQUUUEABBRRQQAF5DsRjp6Eo1lbYW1gqHCcsE05J/HmYsI+wg/y/Z/i2N+8hQjYQni3sJZwmXC/8SHhciHrI/7ZHuEk4WzhIeL6wse/25BVEsFbCB4UfpxFby0PCZcJOvtuV8xCR2gj/IjwagvCpyK+or3RRp/hua05BRGkqLBFWZkn42jwinCe82He7cwIixLeF5Rn69mxwr/BXvtvvDdL4ZsIJiV9klMIncwYjLN96RAZpcEMOiMJXhMc8i4+EDXPECV/3rU3WIY1kXz9ZWJUDwidzwUk7OEvjGgtvEu4MTbDOXwH6tAFuuhC44QLg6m8C8Qauz53tW6vQIY1qLZwJE48HF+fy04D+PwJmjgVWvQzs2Azsfh/Yswv4+ANg51Zg7TLg7w8CQ7sAXc4I2h39VtjAt27OSPzq+yKMydTgOPDW60BVFdTYtQN4aCDQ8RTb930qbOdbPyeA+Zii2HJn4W9rB6xcqBc9FXZsAe7savvuDcLTfetoDRotHC7c5SQ8+/eHB5suJgx8ug+YcgdQ3MjGjjt862kFMbi9cI3zr/7G7wIVq8IRvjaOH5Px434ZqBtqbdkuzP2oSIw8S/g4XCdU3ZoD86YARw6HL35tJ/BL0Nt1v29964UY10jYDSbBFVz4yxoDv+tsBtkocOgAMOhnWvsqhE19a10HYlQT4QiYiCG4+F1OB2aNBw4fjEb8GqxbrrWRE8af+tb7BIhBLYQrnIRnaFj2GxO/+wJDVJ29E31r/n+IMecKVzqJz0H2pVnS13/mLiKjm+2bhG8DBz6x+7dvrwY6naqxeTPisUa+ta8JMVc59fX81dsKVR8WzQZ6tzZfE3md/PnFJ/X//ugRoO/3dLbHY+f6Fb+4esB9IbD4HPTCGmT/swEo7VT/ux4t1T/rjwO0bejiT3yzOM7JlX3quGszYMYYYO9H7sIzemGep+c56d95RRNg02rdM5+frm3LAJ8OYFXBfmvxB/7EJMzCwNb1QL/2+ndrv4KNr2mfOcKP+MXViycvWwl/zbeAF2YAVUfdhf9kN/DIEPuE2ujrdc9/f5v2mQ/4cUBR7DLYrNXe8n3T1x8/7iY8nff6i8CtFwfL8Zd0AFa+BOzbnf49lRXaZ07w5YD56kbf+J1w4vr9e4CxfW0TZ6nZq6VJPdTniNWLtM+6L3rx47Fz5MWH1Q3lgogL+Kt/9lGgewt34ZNJZ47pA7xbceI7nxitfUZp9A4oit2jMo7xPfv8oGB3RWFG9Q7nV5+OjKCemggc3G/erV8nuC5a8YurQ89ylXFMpLn86v/xmMkJZVP4ZN5eBLyzzuZr+2G0DojHmkO7qBJ0xWrNUjNYRil88perG+D3iR5NonVAUewH0JSO8JP+7JCd8FxEf2KMmaT5Et+O8yMVP+GAa1TGPT7MTvwN5SYZ519UG/b24YABKuOWPK0Xf84koNOXggtxZVNgeDfgr+OApc+YnD7nHOtfkb8/K4PrBDOQc2UtPPHZDTf34YA7VQZufFUn/tY3TY4miAgMcaeOALasybxceazKVEHMnWQmhfr13/r4ZOTiJxzwB5WBzExqwF+tbeMp3sR+wSd3HJuemWy+nGDiH5bBt5UvBwxVGckkmQZ/Gm7XeKavtV9XJux8BxhwaRAHjPMifsIB/VVGsh/WYMFUXaMZGU0baSKlMMGvKN0aQl0yAmzl0wHdVIZqZ8AH9gK/+Eb6Z/VrF176OhWYD2JyT++Enj4dcBE084Cxv9YLwGKra1vWfUaPs8JLX2fCB5XmfToHLPXpgDOFH2Y08qqvfZ5X0aDyLTN3GCLdwZCOJlO5baN7+toGC/6sjY5YaHaBHwfEq3NButKTxXOiEy8s6Auzyrw4oNoJRbFRKiOZz8k3cOKmc8C/4WuTt7z4PGjWA/g587OOCuyu2G2x+2I3xu6M3Rq7Ny3YbbL7zOwA7sxv4cUBCScsVv1SOLBxgMs2OFBzwE41kHLGbFNVzQAic9sYiPjbTywv76r8VIGb2wK7tmdN++r09W2XpLeBoa525rxwprYb+qVPB7SEzY5GTnbCrvfkpIyTM236mpM+DfQFuoN8OmC8WvwaMt3MhFgYeHOFffpamyJnglD3zLt8iX8htIvyyWQCbPooU+EQBFwj5iJ6kGzm7DLdO5hI1D1zlC8HTA0kfg0pHlPCTA3ziziWYYcjU81MOTP13CvFjFlDprz5y9aAyb5c/QJkIsaZ8H+dHJD8RXDH49S7zWIKF1G4mLJW+uElc4HHhppB3H5L6efkYg9rPbXge3XPvj16BxTFrg9N/CgYpPp6cqn2+d19OGCud1E1rKm+tk1fc7EmU5W1ISvCo920zQMr5KUHMxrHkg6WdvgSv/+Pg6evy5/XvodrwmdG6wCzyz2zcT3ONpHEyB7RCh9G+npwsfZ9K1ikFrUDrlUZx7I+gnkVlvvpPungZNkiqx4Yorqkr+k8/Zd7b6TiJxxQqjKOu85rg7tguIiejfrOHi2AZfPMRmsXcDMfj7XRvfeodMctfThAl4Z+Y0nqRnLpj1UQXGJ0FZ5hKUPUoBO62uBchOcK6d8/J3LxEw54QGVgZUX6BtMR3CQRpP6TAzzXb7lJw3Wpkt0VQ1ROCu3s8HPOqNoB3N6jAftt21/99NHh7J6vqb7++VdtxV8obOjLAcNURm5Q1u0wOaZt+F1XA9uUxV6ZwPQ1T9iy7/p4nOX5XsRPOKBEZah22s8to5nKEhlBcQsqt6K6gsFA8OprTrzu4Zq4TwdcrjJ2YolelGm/r/85DGc/fM9deMK9+np+5HF/HQfEqxdhMhvL0wptzvZZPNfkbJiYu+LLZpfK8nnhCM+VuPtucJ2ZLxI28yp+wgE8lmBzRoN50IV2R3oNeE4EY3EesMGDNlzBAz948If7noM34LMUMRlizL0qw8ff4i5iUHDpc2TPMCZ+i4W5deGDGHQptGvBa5UFumGB4SkPeXLf2MfVPk46o93/pQHMcfIbVQ1hvx5G9KIBJ1TclemeheWxlN2RyzdrQLtJg2RE5JqnSQcO9jwxN/hmixqy3nMS8uFMUDGS6wJbVA3j+i8r1bJR4Vz+XFgb+9jXd/CtqxWgzYySHAzphDCiG4KHtvLwVh7i6ib8buHdyIUQ0xZi9KnCdVYNLmlvQk0XrPhn8MqIE/kvYWvfOjoBpj5oj1XDmVDjiScs/dAunrCfZ9WyvnQ8HXkXDe8Uy91BVguYI8sGI8jNRjxCnkfJM8/Do+V5ogrjdy6is4vhmi6PoucAy8QZj6h3E55H5D+V97/6VIA5nthNIKYguCLFzdQspHXZuF2X7wk7w1caOduAuQ/A3Qnhk6HlaOTjIGsLxKtvPpqF3Ll85zXhlSftrz4VYA7tfigHfvWcKH5xrp9KBsxe4h0exOcteOf5bn9OACZEfRrRXMb2rnAgcvE4eZ+AuZitI8yOwmwIz/CXdwZHf2xMvgHmhjzOPt2urDLk4vgjyKWFknyAREoMV9skuou/wVyuzJl0pouYuSWUt3Hw8mbems1jM/P/Pi/f4PXiMAW/TA2w5GVcokspS/y5NCF42y/UpZoFFFBAAQUUUEABJy3+B6BFBuObiHkkAAAAAElFTkSuQmCC", "mimeType": "image/png" } ], "version": "0.0.1", "websiteUrl": "https://mcp.svelte.dev", "description": "The official Svelte MCP server implementation" }, "instructions": "This is the official Svelte MCP server. It MUST be used whenever svelte development is involved. It can provide official documentation, code examples and correct your code. After you correct the component call this tool again to confirm all the issues are fixed.", "capabilities": { "completions": {}, "prompts": {}, "resources": {}, "tools": {} }, "transportShape": "sse", "sessionful": false, "credentialContext": "anonymous", "discover": { "_meta": { "io.modelcontextprotocol/serverInfo": { "name": "Svelte MCP", "icons": [ { "src": "https://mcp.svelte.dev/logo.svg", "mimeType": "image/svg+xml" }, { "src": "https://mcp.svelte.dev/logo.png", "mimeType": "image/png" }, { "src": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAYAAADimHc4AAAAAXNSR0IB2cksfwAAAAlwSFlzAAALEwAACxMBAJqcGAAACvdJREFUeJztXQuQVMUVHT5GCYmSDwaVMhQWmpSRRIEkJqmdFVQCRuRjCiVq8MMGKKTYGESEREAFhI2iSGkCFURCJKASimgpSviVrApEPoKLIGEF0aAIAgLCwsk927NxmZ2duf36zesZnFN1qqCU926fO6/79u3b3bFYAQUUUEABBRRQQAF5DsRjp6Eo1lbYW1gqHCcsE05J/HmYsI+wg/y/Z/i2N+8hQjYQni3sJZwmXC/8SHhciHrI/7ZHuEk4WzhIeL6wse/25BVEsFbCB4UfpxFby0PCZcJOvtuV8xCR2gj/IjwagvCpyK+or3RRp/hua05BRGkqLBFWZkn42jwinCe82He7cwIixLeF5Rn69mxwr/BXvtvvDdL4ZsIJiV9klMIncwYjLN96RAZpcEMOiMJXhMc8i4+EDXPECV/3rU3WIY1kXz9ZWJUDwidzwUk7OEvjGgtvEu4MTbDOXwH6tAFuuhC44QLg6m8C8Qauz53tW6vQIY1qLZwJE48HF+fy04D+PwJmjgVWvQzs2Azsfh/Yswv4+ANg51Zg7TLg7w8CQ7sAXc4I2h39VtjAt27OSPzq+yKMydTgOPDW60BVFdTYtQN4aCDQ8RTb930qbOdbPyeA+Zii2HJn4W9rB6xcqBc9FXZsAe7savvuDcLTfetoDRotHC7c5SQ8+/eHB5suJgx8ug+YcgdQ3MjGjjt862kFMbi9cI3zr/7G7wIVq8IRvjaOH5Px434ZqBtqbdkuzP2oSIw8S/g4XCdU3ZoD86YARw6HL35tJ/BL0Nt1v29964UY10jYDSbBFVz4yxoDv+tsBtkocOgAMOhnWvsqhE19a10HYlQT4QiYiCG4+F1OB2aNBw4fjEb8GqxbrrWRE8af+tb7BIhBLYQrnIRnaFj2GxO/+wJDVJ29E31r/n+IMecKVzqJz0H2pVnS13/mLiKjm+2bhG8DBz6x+7dvrwY6naqxeTPisUa+ta8JMVc59fX81dsKVR8WzQZ6tzZfE3md/PnFJ/X//ugRoO/3dLbHY+f6Fb+4esB9IbD4HPTCGmT/swEo7VT/ux4t1T/rjwO0bejiT3yzOM7JlX3quGszYMYYYO9H7sIzemGep+c56d95RRNg02rdM5+frm3LAJ8OYFXBfmvxB/7EJMzCwNb1QL/2+ndrv4KNr2mfOcKP+MXViycvWwl/zbeAF2YAVUfdhf9kN/DIEPuE2ujrdc9/f5v2mQ/4cUBR7DLYrNXe8n3T1x8/7iY8nff6i8CtFwfL8Zd0AFa+BOzbnf49lRXaZ07w5YD56kbf+J1w4vr9e4CxfW0TZ6nZq6VJPdTniNWLtM+6L3rx47Fz5MWH1Q3lgogL+Kt/9lGgewt34ZNJZ47pA7xbceI7nxitfUZp9A4oit2jMo7xPfv8oGB3RWFG9Q7nV5+OjKCemggc3G/erV8nuC5a8YurQ89ylXFMpLn86v/xmMkJZVP4ZN5eBLyzzuZr+2G0DojHmkO7qBJ0xWrNUjNYRil88perG+D3iR5NonVAUewH0JSO8JP+7JCd8FxEf2KMmaT5Et+O8yMVP+GAa1TGPT7MTvwN5SYZ519UG/b24YABKuOWPK0Xf84koNOXggtxZVNgeDfgr+OApc+YnD7nHOtfkb8/K4PrBDOQc2UtPPHZDTf34YA7VQZufFUn/tY3TY4miAgMcaeOALasybxceazKVEHMnWQmhfr13/r4ZOTiJxzwB5WBzExqwF+tbeMp3sR+wSd3HJuemWy+nGDiH5bBt5UvBwxVGckkmQZ/Gm7XeKavtV9XJux8BxhwaRAHjPMifsIB/VVGsh/WYMFUXaMZGU0baSKlMMGvKN0aQl0yAmzl0wHdVIZqZ8AH9gK/+Eb6Z/VrF176OhWYD2JyT++Enj4dcBE084Cxv9YLwGKra1vWfUaPs8JLX2fCB5XmfToHLPXpgDOFH2Y08qqvfZ5X0aDyLTN3GCLdwZCOJlO5baN7+toGC/6sjY5YaHaBHwfEq3NButKTxXOiEy8s6Auzyrw4oNoJRbFRKiOZz8k3cOKmc8C/4WuTt7z4PGjWA/g587OOCuyu2G2x+2I3xu6M3Rq7Ny3YbbL7zOwA7sxv4cUBCScsVv1SOLBxgMs2OFBzwE41kHLGbFNVzQAic9sYiPjbTywv76r8VIGb2wK7tmdN++r09W2XpLeBoa525rxwprYb+qVPB7SEzY5GTnbCrvfkpIyTM236mpM+DfQFuoN8OmC8WvwaMt3MhFgYeHOFffpamyJnglD3zLt8iX8htIvyyWQCbPooU+EQBFwj5iJ6kGzm7DLdO5hI1D1zlC8HTA0kfg0pHlPCTA3ziziWYYcjU81MOTP13CvFjFlDprz5y9aAyb5c/QJkIsaZ8H+dHJD8RXDH49S7zWIKF1G4mLJW+uElc4HHhppB3H5L6efkYg9rPbXge3XPvj16BxTFrg9N/CgYpPp6cqn2+d19OGCud1E1rKm+tk1fc7EmU5W1ISvCo920zQMr5KUHMxrHkg6WdvgSv/+Pg6evy5/XvodrwmdG6wCzyz2zcT3ONpHEyB7RCh9G+npwsfZ9K1ikFrUDrlUZx7I+gnkVlvvpPungZNkiqx4Yorqkr+k8/Zd7b6TiJxxQqjKOu85rg7tguIiejfrOHi2AZfPMRmsXcDMfj7XRvfeodMctfThAl4Z+Y0nqRnLpj1UQXGJ0FZ5hKUPUoBO62uBchOcK6d8/J3LxEw54QGVgZUX6BtMR3CQRpP6TAzzXb7lJw3Wpkt0VQ1ROCu3s8HPOqNoB3N6jAftt21/99NHh7J6vqb7++VdtxV8obOjLAcNURm5Q1u0wOaZt+F1XA9uUxV6ZwPQ1T9iy7/p4nOX5XsRPOKBEZah22s8to5nKEhlBcQsqt6K6gsFA8OprTrzu4Zq4TwdcrjJ2YolelGm/r/85DGc/fM9deMK9+np+5HF/HQfEqxdhMhvL0wptzvZZPNfkbJiYu+LLZpfK8nnhCM+VuPtucJ2ZLxI28yp+wgE8lmBzRoN50IV2R3oNeE4EY3EesMGDNlzBAz948If7noM34LMUMRlizL0qw8ff4i5iUHDpc2TPMCZ+i4W5deGDGHQptGvBa5UFumGB4SkPeXLf2MfVPk46o93/pQHMcfIbVQ1hvx5G9KIBJ1TclemeheWxlN2RyzdrQLtJg2RE5JqnSQcO9jwxN/hmixqy3nMS8uFMUDGS6wJbVA3j+i8r1bJR4Vz+XFgb+9jXd/CtqxWgzYySHAzphDCiG4KHtvLwVh7i6ib8buHdyIUQ0xZi9KnCdVYNLmlvQk0XrPhn8MqIE/kvYWvfOjoBpj5oj1XDmVDjiScs/dAunrCfZ9WyvnQ8HXkXDe8Uy91BVguYI8sGI8jNRjxCnkfJM8/Do+V5ogrjdy6is4vhmi6PoucAy8QZj6h3E55H5D+V97/6VIA5nthNIKYguCLFzdQspHXZuF2X7wk7w1caOduAuQ/A3Qnhk6HlaOTjIGsLxKtvPpqF3Ll85zXhlSftrz4VYA7tfigHfvWcKH5xrp9KBsxe4h0exOcteOf5bn9OACZEfRrRXMb2rnAgcvE4eZ+AuZitI8yOwmwIz/CXdwZHf2xMvgHmhjzOPt2urDLk4vgjyKWFknyAREoMV9skuou/wVyuzJl0pouYuSWUt3Hw8mbems1jM/P/Pi/f4PXiMAW/TA2w5GVcokspS/y5NCF42y/UpZoFFFBAAQUUUEABJy3+B6BFBuObiHkkAAAAAElFTkSuQmCC", "mimeType": "image/png" } ], "version": "0.0.1", "websiteUrl": "https://mcp.svelte.dev", "description": "The official Svelte MCP server implementation" } }, "ttlMs": 0, "cacheScope": "private", "supportedVersions": [ "2026-07-28" ], "capabilities": { "completions": {}, "prompts": {}, "resources": {}, "tools": {} }, "instructions": "This is the official Svelte MCP server. It MUST be used whenever svelte development is involved. It can provide official documentation, code examples and correct your code. After you correct the component call this tool again to confirm all the issues are fixed.", "resultType": "complete" } }, "run": { "id": "mcp-svelte-dev-2026-08-19T21-27-42-001Z", "startedAt": "2026-08-19T21:27:42.001Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "cfd0ee45f81c87e55813f3b085d6253536097b7c31f22ba266cfcbc405edf50e", "taskBudget": 20000 }, "probes": { "specCurrency": "2026-07-28", "findings": [ { "id": "spec-currency", "pass": true, "detail": "negotiated 2026-07-28 (modern era, >= 2026-07-28)", "evidence": { "negotiatedVersion": "2026-07-28", "era": "modern" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-06-18 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "jsonrpc": "2.0", "id": "fitness-probe-bogus-version", "result": { "protocolVersion": "2025-06-18", "adapter": {}, "capabilities": { "tools": {}, "prompts": {}, "resources": {}, "completions": {} }, "instructions": "This is the official Svelte MCP server. It MUST be used whenever svelte development is involved. It can provide official documentation, code examples and correct your code. After you correct the component call this tool again to confirm all the issues are fixed.", "serverInfo": { "name": "Svelte MCP", "version": "0.0.1", "description": "The official Svelte MCP server implementation", "websiteUrl": "https://mcp.svelte.dev", "icons": [ { "src": "https://mcp.svelte.dev/logo.svg", "mimeType": "image/svg+xml" }, { "src": "https://mcp.svelte.dev/logo.png", "mimeType": "image/png" }, { "src": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAYAAADimHc4AAAAAXNSR0IB2cksfwAAAAlwSFlzAAALEwAACxMBAJqcGAAACvdJREFUeJztXQuQVMUVHT5GCYmSDwaVMhQWmpSRRIEkJqmdFVQCRuRjCiVq8MMGKKTYGESEREAFhI2iSGkCFURCJKASimgpSviVrApEPoKLIGEF0aAIAgLCwsk927NxmZ2duf36zesZnFN1qqCU926fO6/79u3b3bFYAQUUUEABBRRQQAF5DsRjp6Eo1lbYW1gqHCcsE05J/HmYsI+wg/y/Z/i2N+8hQjYQni3sJZwmXC/8SHhciHrI/7ZHuEk4WzhIeL6wse/25BVEsFbCB4UfpxFby0PCZcJOvtuV8xCR2gj/IjwagvCpyK+or3RRp/hua05BRGkqLBFWZkn42jwinCe82He7cwIixLeF5Rn69mxwr/BXvtvvDdL4ZsIJiV9klMIncwYjLN96RAZpcEMOiMJXhMc8i4+EDXPECV/3rU3WIY1kXz9ZWJUDwidzwUk7OEvjGgtvEu4MTbDOXwH6tAFuuhC44QLg6m8C8Qauz53tW6vQIY1qLZwJE48HF+fy04D+PwJmjgVWvQzs2Azsfh/Yswv4+ANg51Zg7TLg7w8CQ7sAXc4I2h39VtjAt27OSPzq+yKMydTgOPDW60BVFdTYtQN4aCDQ8RTb930qbOdbPyeA+Zii2HJn4W9rB6xcqBc9FXZsAe7savvuDcLTfetoDRotHC7c5SQ8+/eHB5suJgx8ug+YcgdQ3MjGjjt862kFMbi9cI3zr/7G7wIVq8IRvjaOH5Px434ZqBtqbdkuzP2oSIw8S/g4XCdU3ZoD86YARw6HL35tJ/BL0Nt1v29964UY10jYDSbBFVz4yxoDv+tsBtkocOgAMOhnWvsqhE19a10HYlQT4QiYiCG4+F1OB2aNBw4fjEb8GqxbrrWRE8af+tb7BIhBLYQrnIRnaFj2GxO/+wJDVJ29E31r/n+IMecKVzqJz0H2pVnS13/mLiKjm+2bhG8DBz6x+7dvrwY6naqxeTPisUa+ta8JMVc59fX81dsKVR8WzQZ6tzZfE3md/PnFJ/X//ugRoO/3dLbHY+f6Fb+4esB9IbD4HPTCGmT/swEo7VT/ux4t1T/rjwO0bejiT3yzOM7JlX3quGszYMYYYO9H7sIzemGep+c56d95RRNg02rdM5+frm3LAJ8OYFXBfmvxB/7EJMzCwNb1QL/2+ndrv4KNr2mfOcKP+MXViycvWwl/zbeAF2YAVUfdhf9kN/DIEPuE2ujrdc9/f5v2mQ/4cUBR7DLYrNXe8n3T1x8/7iY8nff6i8CtFwfL8Zd0AFa+BOzbnf49lRXaZ07w5YD56kbf+J1w4vr9e4CxfW0TZ6nZq6VJPdTniNWLtM+6L3rx47Fz5MWH1Q3lgogL+Kt/9lGgewt34ZNJZ47pA7xbceI7nxitfUZp9A4oit2jMo7xPfv8oGB3RWFG9Q7nV5+OjKCemggc3G/erV8nuC5a8YurQ89ylXFMpLn86v/xmMkJZVP4ZN5eBLyzzuZr+2G0DojHmkO7qBJ0xWrNUjNYRil88perG+D3iR5NonVAUewH0JSO8JP+7JCd8FxEf2KMmaT5Et+O8yMVP+GAa1TGPT7MTvwN5SYZ519UG/b24YABKuOWPK0Xf84koNOXggtxZVNgeDfgr+OApc+YnD7nHOtfkb8/K4PrBDOQc2UtPPHZDTf34YA7VQZufFUn/tY3TY4miAgMcaeOALasybxceazKVEHMnWQmhfr13/r4ZOTiJxzwB5WBzExqwF+tbeMp3sR+wSd3HJuemWy+nGDiH5bBt5UvBwxVGckkmQZ/Gm7XeKavtV9XJux8BxhwaRAHjPMifsIB/VVGsh/WYMFUXaMZGU0baSKlMMGvKN0aQl0yAmzl0wHdVIZqZ8AH9gK/+Eb6Z/VrF176OhWYD2JyT++Enj4dcBE084Cxv9YLwGKra1vWfUaPs8JLX2fCB5XmfToHLPXpgDOFH2Y08qqvfZ5X0aDyLTN3GCLdwZCOJlO5baN7+toGC/6sjY5YaHaBHwfEq3NButKTxXOiEy8s6Auzyrw4oNoJRbFRKiOZz8k3cOKmc8C/4WuTt7z4PGjWA/g587OOCuyu2G2x+2I3xu6M3Rq7Ny3YbbL7zOwA7sxv4cUBCScsVv1SOLBxgMs2OFBzwE41kHLGbFNVzQAic9sYiPjbTywv76r8VIGb2wK7tmdN++r09W2XpLeBoa525rxwprYb+qVPB7SEzY5GTnbCrvfkpIyTM236mpM+DfQFuoN8OmC8WvwaMt3MhFgYeHOFffpamyJnglD3zLt8iX8htIvyyWQCbPooU+EQBFwj5iJ6kGzm7DLdO5hI1D1zlC8HTA0kfg0pHlPCTA3ziziWYYcjU81MOTP13CvFjFlDprz5y9aAyb5c/QJkIsaZ8H+dHJD8RXDH49S7zWIKF1G4mLJW+uElc4HHhppB3H5L6efkYg9rPbXge3XPvj16BxTFrg9N/CgYpPp6cqn2+d19OGCud1E1rKm+tk1fc7EmU5W1ISvCo920zQMr5KUHMxrHkg6WdvgSv/+Pg6evy5/XvodrwmdG6wCzyz2zcT3ONpHEyB7RCh9G+npwsfZ9K1ikFrUDrlUZx7I+gnkVlvvpPungZNkiqx4Yorqkr+k8/Zd7b6TiJxxQqjKOu85rg7tguIiejfrOHi2AZfPMRmsXcDMfj7XRvfeodMctfThAl4Z+Y0nqRnLpj1UQXGJ0FZ5hKUPUoBO62uBchOcK6d8/J3LxEw54QGVgZUX6BtMR3CQRpP6TAzzXb7lJw3Wpkt0VQ1ROCu3s8HPOqNoB3N6jAftt21/99NHh7J6vqb7++VdtxV8obOjLAcNURm5Q1u0wOaZt+F1XA9uUxV6ZwPQ1T9iy7/p4nOX5XsRPOKBEZah22s8to5nKEhlBcQsqt6K6gsFA8OprTrzu4Zq4TwdcrjJ2YolelGm/r/85DGc/fM9deMK9+np+5HF/HQfEqxdhMhvL0wptzvZZPNfkbJiYu+LLZpfK8nnhCM+VuPtucJ2ZLxI28yp+wgE8lmBzRoN50IV2R3oNeE4EY3EesMGDNlzBAz948If7noM34LMUMRlizL0qw8ff4i5iUHDpc2TPMCZ+i4W5deGDGHQptGvBa5UFumGB4SkPeXLf2MfVPk46o93/pQHMcfIbVQ1hvx5G9KIBJ1TclemeheWxlN2RyzdrQLtJg2RE5JqnSQcO9jwxN/hmixqy3nMS8uFMUDGS6wJbVA3j+i8r1bJR4Vz+XFgb+9jXd/CtqxWgzYySHAzphDCiG4KHtvLwVh7i6ib8buHdyIUQ0xZi9KnCdVYNLmlvQk0XrPhn8MqIE/kvYWvfOjoBpj5oj1XDmVDjiScs/dAunrCfZ9WyvnQ8HXkXDe8Uy91BVguYI8sGI8jNRjxCnkfJM8/Do+V5ogrjdy6is4vhmi6PoucAy8QZj6h3E55H5D+V97/6VIA5nthNIKYguCLFzdQspHXZuF2X7wk7w1caOduAuQ/A3Qnhk6HlaOTjIGsLxKtvPpqF3Ll85zXhlSftrz4VYA7tfigHfvWcKH5xrp9KBsxe4h0exOcteOf5bn9OACZEfRrRXMb2rnAgcvE4eZ+AuZitI8yOwmwIz/CXdwZHf2xMvgHmhjzOPt2urDLk4vgjyKWFknyAREoMV9skuou/wVyuzJl0pouYuSWUt3Hw8mbems1jM/P/Pi/f4PXiMAW/TA2w5GVcokspS/y5NCF42y/UpZoFFFBAAQUUUEABJy3+B6BFBuObiHkkAAAAAElFTkSuQmCC", "mimeType": "image/png" } ] } } }, "answeredVersion": "2025-06-18" } }, { "id": "header-mismatch-accepted", "pass": true, "detail": "rejected the mismatched Mcp-Name with HTTP 400 / JSON-RPC -32020, as SEP-2243 requires", "evidence": { "status": 400, "jsonRpcErrorCode": -32020, "headerName": "__fitness_report_mismatched_name__", "bodyName": "__fitness_report_probe_tool__", "envelope": { "jsonrpc": "2.0", "id": "fitness-probe-header-mismatch", "error": { "code": -32020, "message": "MCP error -32020: Header mismatch: Mcp-Name does not match params.name" } } } }, { "id": "server-discover-present", "pass": true, "detail": "server/discover answered with supportedVersions [2026-07-28]", "evidence": { "supportedVersions": [ "2026-07-28" ], "capabilities": { "completions": {}, "prompts": {}, "resources": {}, "tools": {} }, "hasInstructions": true } }, { "id": "cache-hints", "pass": true, "detail": "tools/list cache hints: ttlMs=0, cacheScope=private", "evidence": { "ttlMs": 0, "cacheScope": "private", "credentialContext": "anonymous" } }, { "id": "deprecation-surface", "pass": true, "detail": "no deleted or deprecated surface advertised", "evidence": { "deletedVocabulary": [], "notes": [], "toolCount": 4, "era": "modern" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power", "construct", "null_baseline" ], "records": [ { "gate": "protocol_hygiene", "ok": true, "costTier": "free", "reason": "ok", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [] } }, { "gate": "structural", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nRequested": 12, "nGenerated": 12, "nHolding": 12, "admissionRate": 1, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": true, "reason": "ok", "explain": "OK: 12/12 admitted cases hold, admission rate 100.0%. Necessary, not sufficient: run the construct gate." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": true, "costTier": "free", "reason": "ok", "detail": { "nTasks": 12, "minTasks": 8, "toolCount": 4, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 12, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } }, { "gate": "construct", "ok": true, "costTier": "paid", "verdict": { "outcome": "PASS", "k": 12, "n": 12, "threshold": 0.9, "alpha": 0.05, "pValue": 1 }, "reason": "ok", "detail": { "n": 12, "nIntended": 12, "rate": 1, "errors": 0, "errorRate": 0, "maxErrorRate": 0.05, "compromised": false, "reps": 1, "published": { "outcome": "EXTEND", "k": 12, "n": 12, "threshold": 0.9, "alpha": 0.05, "pValue": 1, "rawOutcome": "PASS", "downgraded": true, "wilsonLow": 0.7574992425007574, "wilsonHigh": 1, "requiredN": 78, "achievedPower": 0.20543105023999975, "detectableRate": 0.8, "reason": "downgraded_underpowered" }, "constructOracle": "A reference pass counts only when it both satisfied the check and landed a successful call on a tool the task expects. The reference agent is given the answer key, so a text check alone would pass against a server that returned nothing.", "note": "One rep per task in v0, for cost. evalgate runs three; the divergence is recorded here rather than hidden." } }, { "gate": "null_baseline", "ok": false, "costTier": "cheap", "reason": "noise_exceeds_signal", "detail": { "tNull": 0.8333333333333334, "tAblate": 0.9166666666666666, "killThreshold": 0.4583333333333333, "outcome": "KILL", "halts": true, "ok": false, "reason": "noise_exceeds_signal", "rates": [ { "label": "no-tools", "k": 10, "n": 12, "rate": 0.8333333333333334 }, { "label": "stubbed-empty", "k": 2, "n": 12, "rate": 0.16666666666666666 }, { "label": "random-valid-args", "k": 3, "n": 12, "rate": 0.25 } ], "percentile": 95, "ratio": 0.5, "signalSource": "first-try success on the scored drive", "measuredBeforePaidTier": true, "decidedAfter": "drive", "explain": "KILL (noise_exceeds_signal): a null model passed 0.833 of the suite against the real agent's 0.917, at or above the 0.5 kill threshold of 0.458 (worst offender: no-tools at 0.833). The suite is measuring the model, not the server." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "null_baseline" }, "outcome": "DEGENERATE", "traceLinks": { "mcp": "https://fitnessreport.dev/traces/mcp-svelte-dev-2026-08-19T21-27-42-001Z/mcp.jsonl", "agent": "https://fitnessreport.dev/traces/mcp-svelte-dev-2026-08-19T21-27-42-001Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-svelte-dev-2026-08-19T21-27-42-001Z%2Fmcp.jsonl;https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-svelte-dev-2026-08-19T21-27-42-001Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-svelte-dev", "kind": "mcp+llm", "startedAt": "2026-08-19T21:27:42.001Z", "endedAt": "2026-08-19T21:35:42.517Z", "durationMs": 480516, "command": [ "fitness-report", "https://mcp.svelte.dev/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 434, "meta": 2, "message": 122, "event": 111, "turn": 197, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 94, "inputTokens": 458029, "outputTokens": 14099, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 1.057048, "priced": true } ], "summary": { "models": 1, "assistantTurns": 94, "turns": 197, "echoedTurns": 0, "inputTokens": 458029, "outputTokens": 14099, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 1.057048, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "list-sections", "calls": 21, "errors": 0, "p50Ms": 255, "p95Ms": 499, "origin": "mcp", "pending": 0 }, { "name": "get-documentation", "calls": 19, "errors": 0, "p50Ms": 223, "p95Ms": 582, "origin": "mcp", "pending": 0 }, { "name": "svelte-autofixer", "calls": 11, "errors": 3, "p50Ms": 177, "p95Ms": 340, "origin": "mcp", "pending": 0 }, { "name": "playground-link", "calls": 4, "errors": 2, "p50Ms": 112, "p95Ms": 133, "origin": "mcp", "pending": 0 }, { "name": "__fitness_report_probe_tool__", "calls": 1, "errors": 1, "p50Ms": 92, "p95Ms": 92, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T21:27:42.936Z", "source": "tool", "name": "__fitness_report_probe_tool__", "message": "MCP error -32020: Header mismatch: Mcp-Name does not match params.name (code -32020)" }, { "t": "2026-08-19T21:32:01.966Z", "source": "tool", "name": "svelte-autofixer", "message": "The desired_svelte_version MUST be either 4 or 5 but received \"q994\"" }, { "t": "2026-08-19T21:32:02.074Z", "source": "tool", "name": "svelte-autofixer", "message": "The desired_svelte_version MUST be either 4 or 5 but received \"q488\"" }, { "t": "2026-08-19T21:32:02.173Z", "source": "tool", "name": "svelte-autofixer", "message": "The desired_svelte_version MUST be either 4 or 5 but received \"q403\"" }, { "t": "2026-08-19T21:32:02.289Z", "source": "tool", "name": "playground-link", "message": "The files must contain an App.svelte file as the entry point" }, { "t": "2026-08-19T21:35:19.973Z", "source": "tool", "name": "playground-link", "message": "The files must contain an App.svelte file as the entry point" } ], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-svelte-dev", "kind": "mcp", "startedAt": "2026-08-19T21:27:42.001Z", "endedAt": "2026-08-19T21:35:42.497Z", "durationMs": 480496, "command": [ "fitness-report", "https://mcp.svelte.dev/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 235, "meta": 1, "message": 122, "event": 111, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [ { "name": "list-sections", "calls": 21, "errors": 0, "p50Ms": 255, "p95Ms": 499, "origin": "mcp", "pending": 0 }, { "name": "get-documentation", "calls": 19, "errors": 0, "p50Ms": 223, "p95Ms": 582, "origin": "mcp", "pending": 0 }, { "name": "svelte-autofixer", "calls": 11, "errors": 3, "p50Ms": 177, "p95Ms": 340, "origin": "mcp", "pending": 0 }, { "name": "playground-link", "calls": 4, "errors": 2, "p50Ms": 112, "p95Ms": 133, "origin": "mcp", "pending": 0 }, { "name": "__fitness_report_probe_tool__", "calls": 1, "errors": 1, "p50Ms": 92, "p95Ms": 92, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T21:27:42.936Z", "source": "tool", "name": "__fitness_report_probe_tool__", "message": "MCP error -32020: Header mismatch: Mcp-Name does not match params.name (code -32020)" }, { "t": "2026-08-19T21:32:01.966Z", "source": "tool", "name": "svelte-autofixer", "message": "The desired_svelte_version MUST be either 4 or 5 but received \"q994\"" }, { "t": "2026-08-19T21:32:02.074Z", "source": "tool", "name": "svelte-autofixer", "message": "The desired_svelte_version MUST be either 4 or 5 but received \"q488\"" }, { "t": "2026-08-19T21:32:02.173Z", "source": "tool", "name": "svelte-autofixer", "message": "The desired_svelte_version MUST be either 4 or 5 but received \"q403\"" }, { "t": "2026-08-19T21:32:02.289Z", "source": "tool", "name": "playground-link", "message": "The files must contain an App.svelte file as the entry point" }, { "t": "2026-08-19T21:35:19.973Z", "source": "tool", "name": "playground-link", "message": "The files must contain an App.svelte file as the entry point" } ] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-svelte-dev", "kind": "llm", "startedAt": "2026-08-19T21:27:42.001Z", "endedAt": "2026-08-19T21:35:42.517Z", "durationMs": 480516, "command": [ "fitness-report", "https://mcp.svelte.dev/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 199, "meta": 1, "message": 0, "event": 0, "turn": 197, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": { "perModel": [ { "model": "claude-sonnet-5", "calls": 94, "inputTokens": 458029, "outputTokens": 14099, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingTokens": null, "thinkingEstimated": false, "cacheHitRate": 0, "ttftP50Ms": null, "ttftP95Ms": null, "estCostUsd": 1.057048, "priced": true } ], "summary": { "models": 1, "assistantTurns": 94, "turns": 197, "echoedTurns": 0, "inputTokens": 458029, "outputTokens": 14099, "cacheReadTokens": 0, "cacheCreationTokens": 0, "thinkingExactTokens": null, "thinkingEstimatedTokens": null, "cacheHitRate": 0, "totalModelTimeMs": null }, "cost": { "estimated": true, "currency": "USD", "totalUsd": 1.057048, "partial": false, "unpricedModels": [], "asOf": "2026-08-19", "source": "bundled" } }, "tools": [ { "name": "list-sections", "calls": 21, "errors": 0, "p50Ms": 3066, "p95Ms": 6385, "origin": "turn", "pending": 0 }, { "name": "get-documentation", "calls": 19, "errors": 0, "p50Ms": 3369, "p95Ms": 5371, "origin": "turn", "pending": 0 }, { "name": "svelte-autofixer", "calls": 12, "errors": 0, "p50Ms": 3320, "p95Ms": 5790, "origin": "turn", "pending": 0 }, { "name": "playground-link", "calls": 4, "errors": 1, "p50Ms": 3033, "p95Ms": 7988, "origin": "turn", "pending": 0 } ], "errors": [ { "t": "2026-08-19T21:35:19.973Z", "source": "tool", "name": "playground-link", "message": "The files must contain an App.svelte file as the entry point" } ] } }, "toolsPlane": "mcp" }, "scoreNotes": [ "The drive ran for evidence only. A gate had already refused, so no score is published from it." ], "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs." ] }, { "schema": "fitness-report/1", "server": { "url": "https://mcp.vercel.com/docs/mcp", "slug": "mcp-vercel-com", "era": "modern", "negotiatedVersion": "2026-07-28", "serverInfo": { "name": "Vercel Docs MCP Server", "title": "Vercel Docs MCP Server", "icons": [ { "src": "https://mcp.vercel.com/icons/vercel-light.svg", "mimeType": "image/svg+xml", "sizes": [ "any" ], "theme": "light" }, { "src": "https://mcp.vercel.com/icons/vercel-dark.svg", "mimeType": "image/svg+xml", "sizes": [ "any" ], "theme": "dark" } ], "version": "0.1.0", "websiteUrl": "https://vercel.com", "description": "Search and retrieve Vercel documentation through the Model Context Protocol." }, "instructions": null, "capabilities": { "prompts": { "listChanged": false }, "tools": { "listChanged": false } }, "transportShape": "json", "sessionful": false, "credentialContext": "anonymous", "discover": { "_meta": { "io.modelcontextprotocol/serverInfo": { "name": "Vercel Docs MCP Server", "title": "Vercel Docs MCP Server", "icons": [ { "src": "https://mcp.vercel.com/icons/vercel-light.svg", "mimeType": "image/svg+xml", "sizes": [ "any" ], "theme": "light" }, { "src": "https://mcp.vercel.com/icons/vercel-dark.svg", "mimeType": "image/svg+xml", "sizes": [ "any" ], "theme": "dark" } ], "version": "0.1.0", "websiteUrl": "https://vercel.com", "description": "Search and retrieve Vercel documentation through the Model Context Protocol." } }, "ttlMs": 0, "cacheScope": "private", "supportedVersions": [ "2026-07-28" ], "capabilities": { "prompts": { "listChanged": false }, "tools": { "listChanged": false } }, "resultType": "complete" } }, "run": { "id": "mcp-vercel-com-2026-08-19T22-12-36-293Z", "startedAt": "2026-08-19T22:12:36.293Z", "harnessVersion": "0.1.0", "runnerModel": "claude-sonnet-5", "judgeModel": "claude-opus-5", "suiteHash": "29351960a952cb4ef49b0926f5ad18591b0c91ff37dc388284f5bafc61b55def", "taskBudget": 20000 }, "probes": { "specCurrency": "2026-07-28", "findings": [ { "id": "spec-currency", "pass": true, "detail": "negotiated 2026-07-28 (modern era, >= 2026-07-28)", "evidence": { "negotiatedVersion": "2026-07-28", "era": "modern" } }, { "id": "bogus-version-accepted", "pass": true, "detail": "counter-offered 2025-11-25 instead of the bogus 1999-01-01", "evidence": { "status": 200, "offered": "1999-01-01", "envelope": { "result": { "protocolVersion": "2025-11-25", "capabilities": { "tools": { "listChanged": false }, "prompts": { "listChanged": false } }, "serverInfo": { "name": "Vercel Docs MCP Server", "version": "0.1.0", "title": "Vercel Docs MCP Server", "websiteUrl": "https://vercel.com", "description": "Search and retrieve Vercel documentation through the Model Context Protocol.", "icons": [ { "src": "https://mcp.vercel.com/icons/vercel-light.svg", "mimeType": "image/svg+xml", "sizes": [ "any" ], "theme": "light" }, { "src": "https://mcp.vercel.com/icons/vercel-dark.svg", "mimeType": "image/svg+xml", "sizes": [ "any" ], "theme": "dark" } ] } }, "jsonrpc": "2.0", "id": "fitness-probe-bogus-version" }, "answeredVersion": "2025-11-25" } }, { "id": "header-mismatch-accepted", "pass": true, "detail": "rejected the mismatched Mcp-Name with HTTP 400 / JSON-RPC -32020, as SEP-2243 requires", "evidence": { "status": 400, "jsonRpcErrorCode": -32020, "headerName": "__fitness_report_mismatched_name__", "bodyName": "__fitness_report_probe_tool__", "envelope": { "jsonrpc": "2.0", "error": { "code": -32020, "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\"", "data": { "mismatch": { "header": "__fitness_report_mismatched_name__", "body": "the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\"" } } }, "id": "fitness-probe-header-mismatch" } } }, { "id": "server-discover-present", "pass": true, "detail": "server/discover answered with supportedVersions [2026-07-28]", "evidence": { "supportedVersions": [ "2026-07-28" ], "capabilities": { "prompts": { "listChanged": false }, "tools": { "listChanged": false } }, "hasInstructions": false } }, { "id": "cache-hints", "pass": true, "detail": "tools/list cache hints: ttlMs=0, cacheScope=private", "evidence": { "ttlMs": 0, "cacheScope": "private", "credentialContext": "anonymous" } }, { "id": "deprecation-surface", "pass": true, "detail": "no deleted or deprecated surface advertised", "evidence": { "deletedVocabulary": [], "notes": [], "toolCount": 1, "era": "modern" } } ] }, "gates": { "order": [ "protocol_hygiene", "structural", "answer_leak", "suite_size", "plan_power" ], "records": [ { "gate": "protocol_hygiene", "ok": true, "costTier": "free", "reason": "ok", "detail": { "note": "Protocol hygiene is a reported column, not a refusal. A hygiene failure is a fact about the server, not evidence that this eval is invalid.", "failed": [] } }, { "gate": "structural", "ok": false, "costTier": "free", "reason": "too_few_generated", "detail": { "nRequested": 12, "nGenerated": 2, "nHolding": 2, "admissionRate": 0.16666666666666666, "holdRate": 1, "minAdmissionRate": 0.25, "minGenerated": 8, "failures": [], "ok": false, "reason": "too_few_generated", "explain": "REJECT (too_few_generated): 2 admitted cases is below the minimum 8. Every case held, over a sample too small to mean anything." } }, { "gate": "answer_leak", "ok": true, "costTier": "free", "reason": "ok", "detail": { "leaks": [], "regenerationAttempted": false, "leaksFoundAtGeneration": [] } }, { "gate": "suite_size", "ok": false, "costTier": "free", "reason": "below_minimum_suite_size", "detail": { "nTasks": 2, "minTasks": 8, "toolCount": 1, "note": "With a median of 2.5 tools on the open roster, a suite below 8 tasks cannot separate a good server from a lucky one. This refuses rather than publishing a 2-task 100 percent." } }, { "gate": "plan_power", "ok": true, "costTier": "free", "reason": "underpowered_recorded", "detail": { "requiredN": 78, "actualN": 2, "threshold": 0.9, "detectableRate": 0.8, "power": 0.808179164040748, "note": "This does not refuse. It is why a raw PASS below the planned n downgrades to EXTEND under the published-verdict rule." } } ], "extensionPolicy": { "extensionSize": 0, "maxExtensions": 0 }, "refusedAt": "structural" }, "outcome": "INSUFFICIENT_SURFACE", "traceLinks": { "mcp": "https://fitnessreport.dev/traces/mcp-vercel-com-2026-08-19T22-12-36-293Z/mcp.jsonl", "agent": "https://fitnessreport.dev/traces/mcp-vercel-com-2026-08-19T22-12-36-293Z/agent.jsonl", "viewer": "https://mcpreplay.dev/?trace=https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-vercel-com-2026-08-19T22-12-36-293Z%2Fmcp.jsonl;https%3A%2F%2Ffitnessreport.dev%2Ftraces%2Fmcp-vercel-com-2026-08-19T22-12-36-293Z%2Fagent.jsonl#view=calls" }, "trace_stats": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-vercel-com", "kind": "mcp+llm", "startedAt": "2026-08-19T22:12:36.293Z", "endedAt": "2026-08-19T22:14:04.829Z", "durationMs": 88536, "command": [ "fitness-report", "https://mcp.vercel.com/docs/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 17, "meta": 2, "message": 12, "event": 1, "turn": 0, "end": 2, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [ { "name": "__fitness_report_probe_tool__", "calls": 1, "errors": 1, "p50Ms": 3991, "p95Ms": 3991, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T22:12:43.378Z", "source": "tool", "name": "__fitness_report_probe_tool__", "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\" (code -32020)" } ], "planes": { "mcp": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-vercel-com", "kind": "mcp", "startedAt": "2026-08-19T22:12:36.293Z", "endedAt": "2026-08-19T22:14:04.818Z", "durationMs": 88525, "command": [ "fitness-report", "https://mcp.vercel.com/docs/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 15, "meta": 1, "message": 12, "event": 1, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [ { "name": "__fitness_report_probe_tool__", "calls": 1, "errors": 1, "p50Ms": 3991, "p95Ms": 3991, "origin": "mcp", "pending": 0 } ], "errors": [ { "t": "2026-08-19T22:12:43.378Z", "source": "tool", "name": "__fitness_report_probe_tool__", "message": "Bad Request: the request headers and body disagree: the body carries params.name=\"__fitness_report_probe_tool__\" but the Mcp-Name header names \"__fitness_report_mismatched_name__\" (code -32020)" } ] }, "agent": { "schema": "mcp-tape.stats/1", "file": null, "session": { "label": "mcp-vercel-com", "kind": "llm", "startedAt": "2026-08-19T22:12:36.293Z", "endedAt": "2026-08-19T22:14:04.829Z", "durationMs": 88536, "command": [ "fitness-report", "https://mcp.vercel.com/docs/mcp" ], "producer": "fitness-report@0.1.0", "records": { "total": 2, "meta": 1, "message": 0, "event": 0, "turn": 0, "end": 1, "other": 0 }, "skippedLines": 0, "endReason": "eval_complete", "exitCode": null }, "models": null, "tools": [], "errors": [] } }, "toolsPlane": "mcp" }, "methods": [ "Construct gate denominator diverges from evalgate: reference-agent errors count, and an error rate above 5% resolves to COMPROMISED rather than silently shrinking n.", "A published PASS additionally requires the Wilson 95% lower bound to clear the threshold, or an n the design was sized for. Otherwise the verdict downgrades to EXTEND.", "Destructive-without-confirmation, v0 rule: a tool is destructive unless it declares readOnlyHint true or destructiveHint false, and every executed call to such a tool counts. The only thing that clears one is recorded evidence that the server asked about that same tool before that same call ran. Confirmation is never inherited from another tool or from elsewhere in the task.", "Construct gate: the reference agent is told the answer, so a text check alone would pass against a dead server. A reference pass counts only when it also landed a successful call on a tool the task expects.", "Multi-round tool input (MRTR) is recorded and then declined in v0. A server that asks for input gets an mrtr-abandoned datum, never a fabricated answer.", "Each task runs under an advisory task budget, so a trapping server yields budget exhausted as a clean unrecoverable-path datum rather than an unbounded bill.", "Prior art: MCPEval for generated task suites, evalgate for the gate math, mcp-tape for the recording format. Our delta is refusal, signed replays, and causal rewrite diffs." ] } ]