mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 10:59:59 +00:00
kevinveenbirkenbachandClaude Opus 5
a0dd564955
Each of these specs declared the service it exercises and then, as its first action, signed in through the dashboard's OIDC button. In a round that ships no identity provider that button does not exist, so the spec spent its whole timeout waiting for it and reported the missing login rather than anything about MCP. The junit already said as much, carrying the SSO_SERVICE_ENABLED flag as the skip reason on the neighbouring tests. Nine specs in the directory sign in that way; two already declared sso. The other seven now do too, through the same gate they already imported. Signing in as the deployed identity instead was not open to us: that helper authenticates as the administrator whatever username it is handed, which contradicts the non-member spec asserting the very endpoint the administrator is asserted to reach. Nor can a native member stand in, because the account is created by the OIDC login itself, so a round without one has no members to grant anything to. This buys the honest label at a price worth naming: a round that ships seaweedfs without sso now reports green with no seaweedfs coverage, where it used to report red for the wrong reason. Covering that needs a sign-in the suite does not have yet. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
63 lines
2.4 KiB
JavaScript
63 lines
2.4 KiB
JavaScript
const { test, expect } = require("@playwright/test");
|
|
const { resolveTimeout } = require("./timeouts");
|
|
const { skipUnlessServiceEnabled } = require("./service-gating");
|
|
|
|
exports.register = function (shared) {
|
|
test("litellm chat: a gateway model answers a chat completion via OpenWebUI", async ({ page }) => {
|
|
skipUnlessServiceEnabled("litellm");
|
|
skipUnlessServiceEnabled("sso");
|
|
test.setTimeout(resolveTimeout(120_000));
|
|
|
|
await shared.signInViaDashboardOidc(
|
|
page,
|
|
shared.env.adminUsername,
|
|
shared.env.adminPassword,
|
|
"administrator"
|
|
);
|
|
|
|
const token = await page.evaluate(() => window.localStorage.getItem("token"));
|
|
expect(
|
|
token,
|
|
"OpenWebUI must store a session token in localStorage after login (used to authenticate its OpenAI-compatible API)"
|
|
).toBeTruthy();
|
|
|
|
const base = shared.env.openwebuiBaseUrl.replace(/\/+$/, "");
|
|
const headers = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
|
|
const modelsResp = await page.request.get(`${base}/api/models`, { headers });
|
|
expect(
|
|
modelsResp.ok(),
|
|
`OpenWebUI /api/models must respond to the authenticated session (HTTP ${modelsResp.status()})`
|
|
).toBeTruthy();
|
|
|
|
const modelsBody = await modelsResp.json();
|
|
const modelIds = (Array.isArray(modelsBody?.data) ? modelsBody.data : [])
|
|
.map((m) => String(m?.id ?? m?.name ?? ""));
|
|
const chatModel = modelIds.find((id) => id && !/embed/i.test(id));
|
|
expect(
|
|
chatModel,
|
|
`a chat model must be served by OpenWebUI through the LiteLLM gateway, proving gateway and backend are reachable and the model pulled (got ${JSON.stringify(modelIds)})`
|
|
).toBeTruthy();
|
|
|
|
const chatResp = await page.request.post(`${base}/api/chat/completions`, {
|
|
headers,
|
|
data: {
|
|
model: chatModel,
|
|
messages: [{ role: "user", content: "Reply with exactly the word: pong" }],
|
|
stream: false,
|
|
},
|
|
});
|
|
expect(
|
|
chatResp.ok(),
|
|
`the ${chatModel} chat completion must return 200 over OpenWebUI -> LiteLLM (HTTP ${chatResp.status()})`
|
|
).toBeTruthy();
|
|
|
|
const chatBody = await chatResp.json();
|
|
const content = (chatBody?.choices?.[0]?.message?.content ?? "").trim();
|
|
expect(
|
|
content.length,
|
|
`the model must return a non-empty assistant response over OpenWebUI -> LiteLLM -> ${chatModel} (got ${JSON.stringify(chatBody).slice(0, 300)})`
|
|
).toBeGreaterThan(0);
|
|
});
|
|
};
|