Files
kevinveenbirkenbachandClaude Opus 5 60a9dca2b1 test(openwebui): make the MCP client open the connection, not just record it
GET /api/v1/tools/ builds its answer for an MCP server out of stored config alone - routers/tools.py:125-126 reads info off the connection row and never touches the network. So the three specs that read it pass unchanged against a provider that is down, refuses the deployed bearer, or serves nothing. What they establish is registration and the OIDC group to access-grant chain, not that the bridge carries.

POST /api/v1/configs/tool_servers/verify is the one route in 0.11.0 that does a real MCP initialize plus tools/list without an LLM, and it authenticates with the key in the body it is handed. The spec reads the stored connection and echoes it back, so the handshake runs over the deployed URL with the deployed credential through Open WebUI's own client, and asserts three things per server: it answers, it serves at least one tool, and the tool set contains the read probe its meta/mcp.yml declares.

Only bearer connections are verified. A system_oauth one resolves the caller's own upstream token per request and the administrator holds none, so it would fail on the caller rather than on the bridge; no provider declares auth: oidc today, but the filter keeps the spec honest when one does. The per-test budget scales with the number of registered servers instead of a fixed guess.

MCP_EXPECTED_TOOLS carries the declared read probe per server. Its rendering is unit-tested by extracting the block from the template itself rather than restating it, and decoded the way a spec decodes it - dotenv_quote emits an escaped JSON string that decodeDotenvQuotedValue unwraps before the spec parses the payload, so comparing after one decode would test a value no spec ever sees.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-01 21:58:54 +02:00

365 lines
13 KiB
JavaScript

const { expect } = require("@playwright/test");
const { resolveTimeout } = require("./timeouts");
const {
decodeDotenvQuotedValue,
gotoOnion,
installCspViolationObserver,
normalizeBaseUrl,
performKeycloakLoginForm,
} = require("./personas");
const env = {
oidcIssuerUrl: normalizeBaseUrl(process.env.OIDC_ISSUER_URL || ""),
openwebuiBaseUrl: normalizeBaseUrl(process.env.OPENWEBUI_BASE_URL || ""),
adminUsername: decodeDotenvQuotedValue(process.env.ADMIN_USERNAME),
adminEmail: decodeDotenvQuotedValue(process.env.ADMIN_EMAIL),
adminPassword: decodeDotenvQuotedValue(process.env.ADMIN_PASSWORD),
biberUsername: decodeDotenvQuotedValue(process.env.BIBER_USERNAME),
biberPassword: decodeDotenvQuotedValue(process.env.BIBER_PASSWORD),
canonicalDomain: decodeDotenvQuotedValue(process.env.CANONICAL_DOMAIN),
mcpExpectedServers: decodeDotenvQuotedValue(process.env.MCP_EXPECTED_SERVERS),
mcpExpectedTools: decodeDotenvQuotedValue(process.env.MCP_EXPECTED_TOOLS),
lamBaseUrl: normalizeBaseUrl(process.env.LAM_BASE_URL || ""),
ldapBindDn: decodeDotenvQuotedValue(process.env.LDAP_BIND_DN),
ldapBindPassword: decodeDotenvQuotedValue(process.env.LDAP_BIND_PASSWORD),
ldapUserDnBase: decodeDotenvQuotedValue(process.env.LDAP_USER_DN_BASE),
ldapRoleDnBase: decodeDotenvQuotedValue(process.env.LDAP_ROLE_DN_BASE),
};
function attachDiagnostics(page) {
const consoleErrors = [];
const pageErrors = [];
const cspRelated = [];
page.on("console", (message) => {
if (message.type() === "error") consoleErrors.push(message.text());
if (/content security policy|csp/i.test(message.text())) {
cspRelated.push({ source: "console", text: message.text() });
}
});
page.on("pageerror", (error) => {
const text = String(error);
pageErrors.push(text);
if (/content security policy|csp/i.test(text)) {
cspRelated.push({ source: "pageerror", text });
}
});
return { consoleErrors, pageErrors, cspRelated };
}
// OpenWebUI's first-admin login overlays a changelog/tour aria-modal that intercepts every click.
async function dismissAllOpenModals(page) {
const closeNameRegex = /close|got\s*it|okay|let'?s\s+go|continue|i\s+understand|dismiss|next|skip|finish|done/i;
const deadline = Date.now() + 15_000;
while (Date.now() < deadline) {
const modal = page.locator("[role='dialog'][aria-modal='true']").first();
if (!(await modal.isVisible().catch(() => false))) {
return;
}
const explicitClose = modal
.getByRole("button", { name: closeNameRegex })
.first();
if (await explicitClose.isVisible().catch(() => false)) {
await explicitClose.click().catch(() => {});
} else {
await page.keyboard.press("Escape").catch(() => {});
}
await modal.waitFor({ state: "hidden", timeout: resolveTimeout(3_000) }).catch(() => {});
await page.waitForTimeout(resolveTimeout(500));
}
}
async function openwebuiLogout(page, _openwebuiBaseUrl) {
await dismissAllOpenModals(page);
const signOutTextFilter = { hasText: /^\s*(sign\s*out|log\s*out|abmelden)\s*$/i };
const topLevelSignOut = page
.locator("a, button, [role='menuitem']")
.filter(signOutTextFilter)
.first();
if (await topLevelSignOut.isVisible().catch(() => false)) {
await actionableClick(topLevelSignOut);
return;
}
const menuTrigger = page
.getByRole("img", { name: /open\s+user\s+profile\s+menu/i })
.or(page.getByRole("button", { name: /open\s+user\s+profile\s+menu/i }))
.first();
if (
!(await menuTrigger
.waitFor({ state: "visible", timeout: resolveTimeout(2_000) })
.then(() => true)
.catch(() => false))
) {
return;
}
await dismissAllOpenModals(page);
await actionableClick(menuTrigger);
const dropdownSignOut = page
.locator("a, button, [role='menuitem']")
.filter(signOutTextFilter)
.first();
await expect(
dropdownSignOut,
"Sign Out item must be reachable inside the user-profile menu"
).toBeVisible({ timeout: resolveTimeout(10_000) });
await dismissAllOpenModals(page);
await actionableClick(dropdownSignOut);
}
// Native DOM click; OpenWebUI's auth panel parks the SSO/LDAP buttons outside the viewport scroll container.
async function actionableClick(locator) {
await locator.scrollIntoViewIfNeeded().catch(() => {});
await locator.evaluate((el) => el.click());
}
async function signInViaDashboardOidc(page, username, password, personaLabel) {
const expectedOidcAuthUrl = `${env.oidcIssuerUrl}/protocol/openid-connect/auth`;
await gotoOnion(page, `${env.openwebuiBaseUrl}/`);
const oidcSignIn = page
.locator("a, button")
.filter({ hasText: /sign\s*in\s+with\s+oidc|sign\s*in\s+with\s+sso|continue\s+with\s+oidc|continue\s+with\s+sso|single\s+sign[-\s]*on|sso\s+login/i })
.first();
await expect(
oidcSignIn,
`${personaLabel}: openwebui OIDC sign-in button must be visible on the auth page`
).toBeVisible({ timeout: resolveTimeout(30_000) });
await actionableClick(oidcSignIn);
await expect
.poll(() => page.url(), {
timeout: resolveTimeout(60_000),
message: `${personaLabel}: expected redirect to Keycloak OIDC auth (${expectedOidcAuthUrl})`,
})
.toContain(expectedOidcAuthUrl);
await performKeycloakLoginForm(page, username, password);
await expect
.poll(() => page.url(), {
timeout: resolveTimeout(60_000),
message: `${personaLabel}: expected redirect back to openwebui at ${env.openwebuiBaseUrl}`,
})
.toContain(env.openwebuiBaseUrl);
// Bare openwebuiBaseUrl match also fires on /auth?error=... so assert we left /auth.
await expect
.poll(() => page.url(), {
timeout: resolveTimeout(30_000),
message: `${personaLabel}: expected OIDC callback to leave /auth (mapper / allowed-roles misconfig if it stays)`,
})
.not.toMatch(/\/auth(\b|\?|\/)/);
}
// First-admin bootstrap via /api/v1/auths/signup; UI signup toggle is hidden behind a truncated footer.
async function ensureNativeAdminExists(page, name, email, password, personaLabel) {
const signupResp = await page.context().request.post(
`${env.openwebuiBaseUrl}/api/v1/auths/signup`,
{
data: { name, email, password },
headers: { "Content-Type": "application/json" },
failOnStatusCode: false,
}
);
if (signupResp.status() < 400) return;
const body = await signupResp.json().catch(() => null);
const detail = body?.detail ? String(body.detail) : "";
// 400 "email taken" / "already exists" and 403 "permission to access" both mean bootstrap already done.
if (/taken|already|exists|permission\s+to\s+access/i.test(detail)) return;
throw new Error(
`${personaLabel}: native admin signup failed: ${signupResp.status()} ${detail || (await signupResp.text())}`
);
}
async function signInViaNativePassword(page, email, password, personaLabel) {
await gotoOnion(page, `${env.openwebuiBaseUrl}/auth`);
// Flip from signup mode (name field rendered) to signin so the bootstrap path is not re-exercised.
const nameField = page
.locator("input[autocomplete='name'], input[name='name'], input#name")
.first();
if (
await nameField
.waitFor({ state: "visible", timeout: resolveTimeout(2_000) })
.then(() => true)
.catch(() => false)
) {
const signinToggle = page
.locator("a, button")
.filter({ hasText: /sign\s*in|already\s+have/i })
.first();
if (await signinToggle.isVisible().catch(() => false)) {
await signinToggle.click();
}
}
const emailField = page
.locator("input[type='email'], input[name='email']")
.first();
await expect(
emailField,
`${personaLabel}: native sign-in email field must be visible`
).toBeVisible({ timeout: resolveTimeout(30_000) });
const passwordField = page
.locator("input[type='password'], input[name='password']")
.first();
await expect(
passwordField,
`${personaLabel}: native sign-in password field must be visible`
).toBeVisible({ timeout: resolveTimeout(30_000) });
await emailField.fill(email);
await passwordField.fill(password);
const submit = page.locator("button[type='submit']").first();
await expect(
submit,
`${personaLabel}: native sign-in submit button must be visible`
).toBeVisible({ timeout: resolveTimeout(30_000) });
await actionableClick(submit);
await expect
.poll(() => page.url(), {
timeout: resolveTimeout(60_000),
message: `${personaLabel}: expected redirect away from /auth after native sign-in`,
})
.not.toContain("/auth");
}
async function signInViaLdap(page, username, password, personaLabel) {
await gotoOnion(page, `${env.openwebuiBaseUrl}/auth`);
// ENABLE_LDAP=true makes LDAP the default credential mode (Username + Password + Authenticate).
const usernameField = page.getByRole("textbox", { name: /^username$/i }).first();
await expect(
usernameField,
`${personaLabel}: LDAP username field must be visible on the auth page`
).toBeVisible({ timeout: resolveTimeout(30_000) });
const passwordField = page
.locator("input[type='password'], input[name='password']")
.first();
await expect(
passwordField,
`${personaLabel}: LDAP password field must be visible on the auth page`
).toBeVisible({ timeout: resolveTimeout(30_000) });
await usernameField.fill(username);
await passwordField.fill(password);
const submit = page
.getByRole("button", { name: /^\s*authenticate\s*$/i })
.first();
await expect(
submit,
`${personaLabel}: LDAP Authenticate button must be visible`
).toBeVisible({ timeout: resolveTimeout(30_000) });
await actionableClick(submit);
await expect
.poll(() => page.url(), {
timeout: resolveTimeout(60_000),
message: `${personaLabel}: expected redirect away from /auth after LDAP sign-in`,
})
.not.toContain("/auth");
}
async function expectSignInRequiredAfterLogout(page) {
await openwebuiLogout(page, env.openwebuiBaseUrl);
// OIDC sign-out hops through Keycloak's logout-confirm page; click its Logout submit to advance.
await page.waitForTimeout(resolveTimeout(2_000));
const deadline = Date.now() + 30_000;
while (Date.now() < deadline) {
const url = page.url();
if (
url.startsWith(env.openwebuiBaseUrl) &&
/\/auth(\b|\?|\/|$)/.test(url)
) {
break;
}
const kcLogoutConfirm = page
.locator(
"input#kc-logout, input[name='confirmLogout'], button[name='confirmLogout'], button#kc-logout, button[type='submit'], input[type='submit']"
)
.or(
page
.locator("a, button")
.filter({
hasText: /^\s*(continue|confirm|log\s*out|logout|sign\s*out|abmelden|yes)\s*$/i,
})
)
.first();
if (await kcLogoutConfirm.isVisible().catch(() => false)) {
await actionableClick(kcLogoutConfirm);
}
await page.waitForTimeout(resolveTimeout(2_000));
}
// Contract-permitted final cleanup when the click chain stalls on Keycloak's front-channel iframe wait.
const settledUrl = page.url();
if (!(settledUrl.startsWith(env.openwebuiBaseUrl) && /\/auth(\b|\?|\/|$)/.test(settledUrl))) {
await page.context().clearCookies().catch(() => {});
await gotoOnion(page, `${env.openwebuiBaseUrl}/auth`, { waitUntil: "domcontentloaded" }).catch(() => {});
}
// Anchor on openwebuiBaseUrl so the intermediate `auth.<domain>` Keycloak hop is not misread as /auth.
await expect
.poll(
() => {
const url = page.url();
return url.startsWith(env.openwebuiBaseUrl) && /\/auth(\b|\?|\/|$)/.test(url);
},
{
timeout: resolveTimeout(60_000),
message: "Expected openwebui to land on its own /auth surface after logout",
}
)
.toBe(true);
await expect
.poll(
async () =>
(await page
.locator("a, button")
.filter({ hasText: /sign\s*in|log\s*in|anmelden|continue\s+with|authenticate/i })
.first()
.count()
.catch(() => 0)) > 0,
{
timeout: resolveTimeout(60_000),
message: "Expected openwebui auth surface to expose a sign-in control",
}
)
.toBe(true);
}
async function beforeEach({ page }) {
await page.setViewportSize({ width: 1440, height: 1100 });
expect(env.oidcIssuerUrl, "OIDC_ISSUER_URL must be set").toBeTruthy();
expect(env.openwebuiBaseUrl, "OPENWEBUI_BASE_URL must be set").toBeTruthy();
expect(env.adminUsername, "ADMIN_USERNAME must be set").toBeTruthy();
expect(env.adminEmail, "ADMIN_EMAIL must be set").toBeTruthy();
expect(env.adminPassword, "ADMIN_PASSWORD must be set").toBeTruthy();
expect(env.biberUsername, "BIBER_USERNAME must be set").toBeTruthy();
expect(env.biberPassword, "BIBER_PASSWORD must be set").toBeTruthy();
expect(env.canonicalDomain, "CANONICAL_DOMAIN must be set").toBeTruthy();
await page.context().clearCookies();
await installCspViolationObserver(page);
}
module.exports = {
env,
attachDiagnostics,
dismissAllOpenModals,
openwebuiLogout,
signInViaDashboardOidc,
ensureNativeAdminExists,
signInViaNativePassword,
signInViaLdap,
expectSignInRequiredAfterLogout,
beforeEach,
};