computer-playbook/roles/web-app-mattermost/files/playwright/test-litellm-agent.js

const { test, expect } = require("@playwright/test");
const { resolveTimeout } = require("./timeouts");

const {
  apiFetchOnion,
  apiGetOnion,
  decodeDotenvQuotedValue,
  performKeycloakLoginForm,
} = require("./personas");
const { skipUnlessServiceEnabled } = require("./service-gating");

const LITELLM_API_URL = decodeDotenvQuotedValue(process.env.LITELLM_API_URL);
const LITELLM_BOT_NAME = decodeDotenvQuotedValue(process.env.LITELLM_BOT_NAME);

const PLUGIN_FAILURE_NOTICE = /sorry!?\s*an error|an error occurred|accessing the llm/i;

// The role rewrites the administrator to gitlab auth only when SSO is on, so the
// password grant exists exactly in the deployments that run without OIDC.
async function acquireAdminToken(context, shared, baseUrl) {
  if (!shared.oidcEnabled) {
    const login = await apiFetchOnion(context.request, `${baseUrl}/api/v4/users/login`, {
      method: "POST",
      failOnStatusCode: false,
      headers: { "Content-Type": "application/json" },
      data: { login_id: shared.env.adminUsername, password: shared.env.adminPassword },
    });
    expect(
      login.status(),
      `the administrator must be able to open a Mattermost session; without one the Agents bot cannot be prompted at all (HTTP ${login.status()})`,
    ).toBe(200);

    const token = login.headers().token;
    expect(
      token,
      "Mattermost must return the session token in the Token response header of /api/v4/users/login",
    ).toBeTruthy();
    return token;
  }

  const page = await context.newPage();
  await shared.startMattermostSsoFlow(page, baseUrl);
  await performKeycloakLoginForm(page, shared.env.adminUsername, shared.env.adminPassword);
  await expect
    .poll(() => page.url(), {
      timeout: resolveTimeout(60_000),
      message: "Expected redirect back to Mattermost after the administrator OIDC login",
    })
    .toContain(baseUrl);

  const session = (await context.cookies(baseUrl)).find((cookie) => cookie.name === "MMAUTHTOKEN");
  expect(
    session && session.value,
    "the OIDC login must leave an MMAUTHTOKEN session cookie; without it no Agents prompt can be authenticated",
  ).toBeTruthy();
  await page.close().catch(() => {});
  return session.value;
}

async function firstBotReply(context, baseUrl, headers, channelId, botId, afterPostId) {
  const response = await context.request.get(
    `${baseUrl}/api/v4/channels/${channelId}/posts?after=${afterPostId}&per_page=50`,
    { headers, failOnStatusCode: false },
  );
  if (!response.ok()) return "";

  const body = await response.json();
  const replies = Object.values((body && body.posts) || {})
    .filter((post) => post.user_id === botId && String(post.message || "").trim().length > 0)
    .sort((a, b) => a.create_at - b.create_at);
  return replies.length > 0 ? String(replies[0].message).trim() : "";
}

exports.register = function (shared) {
  test("litellm: the Agents bot answers a prompt through the in-cluster gateway", async ({ browser }) => {
    skipUnlessServiceEnabled("litellm");
    test.setTimeout(resolveTimeout(300_000));

    expect(
      LITELLM_API_URL,
      "LITELLM_API_URL must render MATTERMOST_LITELLM_BASE_URL into the Playwright env; without it the endpoint the Agents plugin was pointed at cannot be checked",
    ).toMatch(/^https?:\/\/.+/i);

    const gatewayHost = new URL(LITELLM_API_URL).hostname;
    expect(
      gatewayHost,
      `the Agents service endpoint must address the gateway by its bare in-cluster service name; "${gatewayHost}" carries a dot, so the container dns-search suffix completes it and the prompt leaves the deployment through the public ingress`,
    ).not.toContain(".");

    const baseUrl = shared.expectedMattermostBaseUrl();
    const context = await browser.newContext({ ignoreHTTPSErrors: true });

    try {
      const token = await acquireAdminToken(context, shared, baseUrl);
      await context.clearCookies();
      const headers = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };

      const meResponse = await apiGetOnion(context.request, `${baseUrl}/api/v4/users/me`, {
        headers,
        failOnStatusCode: false,
      });
      expect(
        meResponse.status(),
        `the session token must authenticate against the Mattermost API (HTTP ${meResponse.status()})`,
      ).toBe(200);
      const me = await meResponse.json();

      const botResponse = await apiGetOnion(
        context.request,
        `${baseUrl}/api/v4/users/username/${LITELLM_BOT_NAME}`,
        { headers, failOnStatusCode: false },
      );
      expect(
        botResponse.status(),
        `the Agents plugin must have provisioned the bot user "${LITELLM_BOT_NAME}"; its absence means the bot row the deploy merged into agents_confighistory was never picked up (HTTP ${botResponse.status()})`,
      ).toBe(200);
      const bot = await botResponse.json();
      expect(
        bot.id,
        `the "${LITELLM_BOT_NAME}" account must carry a user id so a direct message channel can be opened with it`,
      ).toBeTruthy();

      const dmResponse = await apiFetchOnion(context.request, `${baseUrl}/api/v4/channels/direct`, {
        method: "POST",
        headers,
        failOnStatusCode: false,
        data: [me.id, bot.id],
      });
      expect(
        dmResponse.ok(),
        `a direct message channel between the administrator and the Agents bot must open; the plugin answers prompts only inside a channel it participates in (HTTP ${dmResponse.status()})`,
      ).toBeTruthy();
      const dm = await dmResponse.json();

      const prompt = `Reply with exactly the word: pong (${Date.now()})`;
      const promptResponse = await context.request.post(`${baseUrl}/api/v4/posts`, {
        headers,
        failOnStatusCode: false,
        data: { channel_id: dm.id, message: prompt },
      });
      expect(
        promptResponse.ok(),
        `the prompt must be posted into the Agents direct message channel (HTTP ${promptResponse.status()})`,
      ).toBeTruthy();
      const prompted = await promptResponse.json();

      let reply = "";
      await expect
        .poll(
          async () => {
            reply = await firstBotReply(context, baseUrl, headers, dm.id, bot.id, prompted.id).catch(
              () => "",
            );
            return reply.length;
          },
          {
            timeout: resolveTimeout(180_000),
            intervals: [2_000],
            message: `the Agents bot "${LITELLM_BOT_NAME}" must answer the direct message with a real completion; silence means the plugin never reached ${LITELLM_API_URL}, the gateway rejected the virtual key, or no model is served behind it`,
          },
        )
        .toBeGreaterThan(0);

      expect(
        reply,
        `the Agents bot returned the plugin's LLM failure notice instead of a completion, so the prompt never got an answer out of ${LITELLM_API_URL}: ${JSON.stringify(reply).slice(0, 300)}`,
      ).not.toMatch(PLUGIN_FAILURE_NOTICE);
    } finally {
      await context.close().catch(() => {});
    }
  });
};