test(mcp): prove a switched-off surface is actually gone

Every role-local MCP spec gates itself on the service flag and skips when the surface is off, so nothing ever checked that switching it off removes it. A surface still serving after the flag went false was indistinguishable from one that was never deployed: both read as a green skip. The gating helper had no inverse at all, so there was no way to assert what a disabled service must not do.

A shared assertion now probes the endpoint with both a GET and a POST while the flag is false and fails on any answer that shows something listening. It counts 401 and 403 as listening, which is the point: a refused credential still means the surface is there. The check is transport-agnostic because the roles differ in their enabled-state probes, over SSE, Streamable HTTP or plain JSON-RPC, while an endpoint that is gone answers none of them. Each role passes its endpoint URL as a thunk, since some resolve it only once a test is running.

The coverage lint requires the registration, and derives the role set from meta/mcp.yml rather than from a filename convention, which is how the tenth role surfaced: its spec is not named like the other nine.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

roles/test-e2e-playwright/files/mcp-endpoint.js

@@ -0,0 +1,66 @@
+/**
+ * Shared disabled-state assertion for MCP endpoints.
+ *
+ * Every role-local MCP spec gates itself on `MCP_SERVICE_ENABLED` and skips
+ * when the surface is off, so nothing ever checked that switching it off
+ * actually removes it. A surface still serving after the flag went false is
+ * indistinguishable from one that was never deployed, which is the failure
+ * this registers a test for.
+ *
+ * The check is deliberately transport-agnostic. Roles serve over classic SSE,
+ * Streamable HTTP or plain JSON-RPC and their enabled-state specs differ
+ * accordingly, but an endpoint that is gone answers none of them, so both a
+ * GET and a POST have to come back unserved.
+ *
+ * Contract:
+ *   registerMcpDisabledState(() => endpointUrl) -> registers one Playwright test
+ *
+ * The URL is passed as a thunk because the roles build it differently and some
+ * of them resolve it from fixtures that only exist once a test is running.
+ */
+
+const { test, expect } = require("@playwright/test");
+const { skipUnlessServiceDisabled } = require("./service-gating");
+
+const SERVED = new Set([200, 201, 202, 401, 403, 405, 406, 415, 429]);
+
+function registerMcpDisabledState(resolveEndpointUrl) {
+  test("guest: the MCP endpoint is gone while the service is switched off", async ({
+    page,
+  }) => {
+    skipUnlessServiceDisabled("mcp");
+
+    const endpointUrl = resolveEndpointUrl();
+    expect(endpointUrl, "the MCP endpoint URL must resolve even while disabled").toBeTruthy();
+
+    const attempts = [
+      await page.request.get(endpointUrl, {
+        failOnStatusCode: false,
+        maxRedirects: 0,
+        headers: { accept: "text/event-stream" },
+      }),
+      await page.request.post(endpointUrl, {
+        failOnStatusCode: false,
+        maxRedirects: 0,
+        headers: { Accept: "application/json", "Content-Type": "application/json" },
+        data: { jsonrpc: "2.0", id: 1, method: "tools/list", params: {} },
+      }),
+    ];
+
+    for (const response of attempts) {
+      const status = response.status();
+      const excerpt = (await response.text()).slice(0, 200);
+      expect(
+        SERVED.has(status),
+        `${endpointUrl} answered ${status} with ${excerpt} while MCP is disabled; ` +
+          `a refused credential still means the surface is listening`,
+      ).toBe(false);
+      expect(
+        response.headers()["content-type"] || "",
+        "a disabled endpoint must not open the SSE channel",
+      ).not.toContain("text/event-stream");
+    }
+  });
+}
+
+module.exports = { registerMcpDisabledState };

roles/test-e2e-playwright/files/service-gating.js

@@ -121,6 +121,25 @@ function skipUnlessServiceEnabled(name) {
   }
 }
 
+/**
+ * Skip the current test unless the named service is disabled, so a spec can
+ * assert what a switched-off service must NOT do:
+ *
+ *   test("the endpoint is gone while mcp is off", async ({ page }) => {
+ *     skipUnlessServiceDisabled("mcp");
+ *     // assert absence below
+ *   });
+ *
+ * Without this the disabled state is never asserted at all: every gated spec
+ * skips itself when the flag is false, so a surface that kept serving after
+ * being switched off looks exactly like one that was never deployed.
+ */
+function skipUnlessServiceDisabled(name) {
+  if (isServiceEnabled(name)) {
+    test.skip(true, `${envKey(name)}=true`);
+  }
+}
+
 /**
  * Legacy wrapper kept for compatibility with callers that were written
  * against the wrap-style API. New code SHOULD use
@@ -142,5 +161,6 @@ module.exports = {
   isServiceEnabled,
   isServiceDisabledReason,
   requireService,
+  skipUnlessServiceDisabled,
   skipUnlessServiceEnabled,
 };

roles/web-app-baserow/files/playwright/test-mcp.js

@@ -1,6 +1,7 @@
 const { test, expect } = require("@playwright/test");
 const { skipUnlessServiceEnabled } = require("./service-gating");
 const { normalizeBaseUrl } = require("./personas");
+const { registerMcpDisabledState } = require("./mcp-endpoint");
 
 const UNKNOWN_ENDPOINT_KEY = "0".repeat(32);
 
@@ -25,3 +26,8 @@ test("mcp: an unauthenticated probe of the MCP endpoint is rejected", async ({ p
     "a probe carrying no valid endpoint key must never receive an MCP session endpoint",
   ).not.toContain("event: endpoint");
 });
+
+registerMcpDisabledState(
+  () =>
+    `${normalizeBaseUrl(process.env.BASEROW_BASE_URL || "")}/mcp/${UNKNOWN_ENDPOINT_KEY}/sse`,
+);

roles/web-app-gitea/files/playwright/test-mcp-guest.js

@@ -1,6 +1,7 @@
 const { test, expect } = require("@playwright/test");
 
 const { skipUnlessServiceEnabled } = require("./service-gating");
+const { registerMcpDisabledState } = require("./mcp-endpoint");
 
 exports.register = function (shared) {
   test("mcp: an unauthenticated probe of the MCP endpoint is rejected", async ({ page }) => {
@@ -27,4 +28,6 @@ exports.register = function (shared) {
       "the public vhost must not return an MCP protocol response to a bearerless probe",
     ).not.toContain('"jsonrpc"');
   });
+
+  registerMcpDisabledState(() => shared.mcpEndpointUrl());
 };

roles/web-app-gitlab/files/playwright/test-mcp-guest.js

@@ -1,6 +1,7 @@
 const { test, expect } = require("@playwright/test");
 const { skipUnlessServiceEnabled } = require("./service-gating");
 const { decodeDotenvQuotedValue, normalizeBaseUrl } = require("./personas");
+const { registerMcpDisabledState } = require("./mcp-endpoint");
 
 const appBaseUrl = normalizeBaseUrl(process.env.APP_BASE_URL || "");
 const mcpEndpointPath = decodeDotenvQuotedValue(process.env.MCP_ENDPOINT_PATH || "");
@@ -28,3 +29,5 @@ test("guest: the MCP endpoint rejects unauthenticated access", async ({ page })
     "a refused probe must not return an MCP protocol response",
   ).not.toContain('"jsonrpc"');
 });
+
+registerMcpDisabledState(() => `${appBaseUrl}${mcpEndpointPath}`);

roles/web-app-homeassistant/files/playwright/test-mcp-guest.js

@@ -2,6 +2,7 @@ const { test, expect } = require("@playwright/test");
 const { resolveTimeout } = require("./timeouts");
 
 const { skipUnlessServiceEnabled } = require("./service-gating");
+const { registerMcpDisabledState } = require("./mcp-endpoint");
 
 exports.register = function (shared) {
   test("mcp: an unauthenticated probe of the MCP endpoint is rejected", async ({ request }) => {
@@ -25,4 +26,8 @@ exports.register = function (shared) {
       "a refused probe must not return an MCP protocol response",
     ).not.toContain('"jsonrpc"');
   });
+
+  registerMcpDisabledState(
+    () => `${shared.env.baseUrl.replace(/\/+$/, "")}/api/mcp`,
+  );
 };

roles/web-app-jenkins/files/playwright/test-mcp-guest.js

@@ -1,6 +1,7 @@
 const { test, expect } = require("@playwright/test");
 const { skipUnlessServiceEnabled } = require("./service-gating");
 const { decodeDotenvQuotedValue, normalizeBaseUrl } = require("./personas");
+const { registerMcpDisabledState } = require("./mcp-endpoint");
 
 const baseUrl = normalizeBaseUrl(process.env.JENKINS_BASE_URL || "");
 const mcpEndpointPath = decodeDotenvQuotedValue(process.env.MCP_ENDPOINT_PATH || "");
@@ -27,3 +28,7 @@ test("MCP: the plugin endpoint serves no tool without credentials", async ({ pag
     "a refused tools/list must not leak a tool inventory",
   ).not.toContain('"tools"');
 });
+
+registerMcpDisabledState(
+  () => `${baseUrl.replace(/\/$/, "")}${mcpEndpointPath}`,
+);

roles/web-app-mattermost/files/playwright/test-mcp-guest.js

@@ -2,6 +2,7 @@ const { test, expect } = require("@playwright/test");
 
 const { decodeDotenvQuotedValue } = require("./personas");
 const { skipUnlessServiceEnabled } = require("./service-gating");
+const { registerMcpDisabledState } = require("./mcp-endpoint");
 
 const MCP_ENDPOINT_PATH = decodeDotenvQuotedValue(process.env.MCP_ENDPOINT_PATH);
 
@@ -33,4 +34,8 @@ exports.register = function (shared) {
       "an unauthenticated MCP initialize must not be served a 2xx; the endpoint is bearer-guarded",
     ).toBeGreaterThanOrEqual(400);
   });
+
+  registerMcpDisabledState(() =>
+    new URL(MCP_ENDPOINT_PATH, shared.expectedMattermostBaseUrl()).toString(),
+  );
 };

roles/web-app-moodle/files/playwright/test-mcp-guest.js

@@ -1,6 +1,7 @@
 const { test, expect } = require("@playwright/test");
 const { decodeDotenvQuotedValue } = require("./personas");
 const { skipUnlessServiceEnabled } = require("./service-gating");
+const { registerMcpDisabledState } = require("./mcp-endpoint");
 
 const mcpEndpointPath = decodeDotenvQuotedValue(process.env.MCP_ENDPOINT_PATH || "");
 
@@ -34,4 +35,8 @@ exports.register = function (shared) {
       `an unauthenticated MCP probe must be rejected, got ${status} with ${excerpt}`,
     ).toBe(true);
   });
+
+  registerMcpDisabledState(() =>
+    new URL(mcpEndpointPath, shared.env.moodleBaseUrl).toString(),
+  );
 };

roles/web-app-n8n/files/playwright/test-mcp-guest.js

@@ -1,5 +1,6 @@
 const { test, expect } = require("@playwright/test");
 const { skipUnlessServiceEnabled } = require("./service-gating");
+const { registerMcpDisabledState } = require("./mcp-endpoint");
 const { decodeDotenvQuotedValue, normalizeBaseUrl } = require("./personas");
 
 const appBaseUrl = normalizeBaseUrl(process.env.APP_BASE_URL || "");
@@ -27,3 +28,5 @@ test("guest: the managed MCP trigger answers nobody without its bearer", async (
     "a refused probe must not open the SSE channel",
   ).not.toContain("text/event-stream");
 });
+
+registerMcpDisabledState(() => `${appBaseUrl}${mcpEndpointPath}`);

roles/web-app-nextcloud/files/playwright/test-mcp-guest.js

@@ -1,6 +1,7 @@
 const { test, expect } = require("@playwright/test");
 const { decodeDotenvQuotedValue } = require("./personas");
 const { skipUnlessServiceEnabled } = require("./service-gating");
+const { registerMcpDisabledState } = require("./mcp-endpoint");
 
 const mcpEndpointPath = decodeDotenvQuotedValue(process.env.NEXTCLOUD_MCP_ENDPOINT_PATH);
 
@@ -30,4 +31,8 @@ exports.register = function (shared) {
       "an unauthenticated MCP probe must not be served a 2xx; the endpoint is app-password guarded",
     ).toBeGreaterThanOrEqual(400);
   });
+
+  registerMcpDisabledState(() =>
+    new URL(mcpEndpointPath, shared.env.nextcloudBaseUrl).toString(),
+  );
 };

roles/web-app-wordpress/files/playwright/test-mcp-guest.js

@@ -1,6 +1,7 @@
 const { test, expect } = require("@playwright/test");
 const { skipUnlessServiceEnabled } = require("./service-gating");
 const shared = require("./_shared");
+const { registerMcpDisabledState } = require("./mcp-endpoint");
 
 const MCP_ENDPOINT = "/wp-json/mcp/mcp-adapter-default-server";
 
@@ -26,3 +27,5 @@ test("guest: the MCP endpoint rejects unauthenticated access", async ({ page })
     "a refused probe must not return an MCP protocol response",
   ).not.toContain('"jsonrpc"');
 });
+
+registerMcpDisabledState(() => `${shared.env.wpBaseUrl}${MCP_ENDPOINT}`);

tests/lint/ansible/services/test_mcp_playwright_coverage.py

@@ -0,0 +1,171 @@
+"""Lint: an enabled MCP surface is covered by a Playwright spec of its own.
+
+The deploy-time provider probe proves the endpoint answers *its* credential. It
+says nothing about what the endpoint answers a stranger, because it never asks
+without one. That question belongs to a browser-side spec, and a role that
+turns its MCP surface on without one ships an endpoint nobody ever asked the
+hostile question of.
+
+Two relations per role whose ``meta/mcp.yml`` is enabled:
+
+* it ships at least one file under ``files/playwright/`` whose name carries
+  ``mcp``, so the surface has a spec rather than being covered incidentally by
+  a persona test;
+* a role serving MCP additionally asserts the unauthenticated case somewhere in
+  those files. The anchor is vocabulary — ``unauthenticated``, ``bearerless``,
+  ``without a bearer``, ``refus…`` or ``reject…`` — so name the assertion in
+  those words rather than leaving a reader to infer it.
+
+Disabled surfaces are exempt: nothing is served until an operator turns them on.
+
+Suppression (see ``docs/contributing/actions/testing/suppression.md``):
+
+* ``# nocheck: mcp-playwright-coverage`` in the head of the role's
+  ``meta/mcp.yml``.
+"""
+
+from __future__ import annotations
+
+import re
+import unittest
+from collections.abc import Mapping
+from pathlib import Path
+
+from utils.annotations.suppress import is_suppressed_in_head
+from utils.cache.files import read_text
+from utils.cache.yaml import load_yaml_any
+from utils.roles.mapping import ROLE_FILE_META_MCP
+
+from . import PROJECT_ROOT
+
+_RULE = "mcp-playwright-coverage"
+_SERVER_DIRECTIONS = frozenset({"server", "both"})
+_CLIENT_DIRECTIONS = frozenset({"client", "both"})
+_UNAUTHENTICATED = re.compile(
+    r"unauthenticated|bearerless|without a bearer|refus|reject", re.IGNORECASE
+)
+_CLIENT_SURFACE = re.compile(
+    r"unauthenticated|refus|reject|server list|servers\b", re.IGNORECASE
+)
+
+
+def _enabled_mcp_roles() -> dict[str, str]:
+    """Return ``{role: direction}`` for every role serving or consuming MCP."""
+    roles: dict[str, str] = {}
+    for mcp_path in sorted(Path(PROJECT_ROOT, "roles").glob(f"*/{ROLE_FILE_META_MCP}")):
+        mcp = load_yaml_any(str(mcp_path), default_if_missing={})
+        if not isinstance(mcp, Mapping) or not mcp.get("enabled"):
+            continue
+        if is_suppressed_in_head(read_text(str(mcp_path)).splitlines(), _RULE):
+            continue
+        roles[mcp_path.parent.parent.name] = (
+            str(mcp.get("direction") or "").strip().lower()
+        )
+    return roles
+
+
+def _mcp_specs(role: str) -> list[Path]:
+    directory = PROJECT_ROOT / "roles" / role / "files" / "playwright"
+    return sorted(p for p in directory.glob("*.js") if "mcp" in p.name)
+
+
+class TestMcpPlaywrightCoverage(unittest.TestCase):
+    def test_every_enabled_surface_has_a_spec(self) -> None:
+        missing = [
+            f"{role}: its MCP surface is enabled but files/playwright/ holds no "
+            f"mcp spec, so nothing exercises it from outside the deploy"
+            for role in sorted(_enabled_mcp_roles())
+            if not _mcp_specs(role)
+        ]
+        self.assertEqual(
+            [],
+            missing,
+            f"enabled MCP surface(s) without a spec ({len(missing)}):\n"
+            + "\n".join(f"  - {m}" for m in missing),
+        )
+
+    def test_every_served_surface_asserts_the_unauthenticated_case(self) -> None:
+        silent = []
+        for role, direction in sorted(_enabled_mcp_roles().items()):
+            if direction not in _SERVER_DIRECTIONS:
+                continue
+            specs = _mcp_specs(role)
+            if specs and not any(
+                _UNAUTHENTICATED.search(read_text(str(spec))) for spec in specs
+            ):
+                silent.append(
+                    f"{role}: serves MCP but no spec names the unauthenticated "
+                    f"case, so the endpoint is only ever asked with a valid "
+                    f"credential"
+                )
+        self.assertEqual(
+            [],
+            silent,
+            f"served MCP surface(s) never probed anonymously ({len(silent)}):\n"
+            + "\n".join(f"  - {s}" for s in silent),
+        )
+
+    def test_every_served_surface_asserts_the_disabled_case(self) -> None:
+        silent = []
+        for role, direction in sorted(_enabled_mcp_roles().items()):
+            if direction not in _SERVER_DIRECTIONS:
+                continue
+            specs = _mcp_specs(role)
+            if specs and not any(
+                "registerMcpDisabledState" in read_text(str(spec)) for spec in specs
+            ):
+                silent.append(
+                    f"{role}: every MCP spec here skips itself when the surface "
+                    f"is switched off, so one that kept serving afterwards looks "
+                    f"exactly like one that was never deployed"
+                )
+        self.assertEqual(
+            [],
+            silent,
+            f"served MCP surface(s) never probed while disabled ({len(silent)}):\n"
+            + "\n".join(f"  - {s}" for s in silent),
+        )
+
+    def test_every_consumed_surface_asserts_what_the_client_exposes(self) -> None:
+        silent = []
+        for role, direction in sorted(_enabled_mcp_roles().items()):
+            if direction not in _CLIENT_DIRECTIONS:
+                continue
+            specs = _mcp_specs(role)
+            if specs and not any(
+                _CLIENT_SURFACE.search(read_text(str(spec))) for spec in specs
+            ):
+                silent.append(
+                    f"{role}: consumes MCP but no spec asserts either the "
+                    f"configured server list or that its own API refuses an "
+                    f"unauthenticated caller, so the credential it holds is "
+                    f"never shown to be out of reach"
+                )
+        self.assertEqual(
+            [],
+            silent,
+            f"MCP client(s) with an unexercised surface ({len(silent)}):\n"
+            + "\n".join(f"  - {s}" for s in silent),
+        )
+
+    def test_the_scan_finds_enabled_surfaces(self) -> None:
+        roles = _enabled_mcp_roles()
+        self.assertTrue(
+            roles,
+            "no role enables an MCP surface, so both rules would pass "
+            "vacuously; check that the scan still reads the right topic",
+        )
+        self.assertTrue(
+            any(d in _SERVER_DIRECTIONS for d in roles.values()),
+            "no enabled role serves MCP, so the anonymous-probe rule would "
+            "pass vacuously",
+        )
+        self.assertTrue(
+            any(d in _CLIENT_DIRECTIONS for d in roles.values()),
+            "no enabled role consumes MCP, so the client-surface rule would "
+            "pass vacuously",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()