feat(ci): run the swarm drill on every distro

The discover job was named after the full distro list and the plan summary printed all five, but test-deploy-swarm.yml never set INFINITO_DISTROS, so every run fell back to the default.env distro and only debian was ever exercised. The job name has been advertising coverage that did not exist.

The workflow now passes the distros input into the env and hands the drill to the shared loop from 1bbe7a0eb. Bootstrap, setup, matrix deploy, seed, drain and assert move out of eleven separate steps into routine/00_one.sh, which is what the loop runs once per distro.

Collection and teardown move with them, into an EXIT trap: it replaces the removed 'if: always()' and 'if: failure()' step conditions and returns the original exit code so cleanup cannot mask the deploy failure. The always() teardown step stays as a second net for a step that times out as a whole.

Playwright reports and rescue diagnostics gain a distro segment, matching the layout the compose path already writes, because a second distro would otherwise overwrite the first one's evidence. The structure summary carries the distro in its heading for the same reason.

The README described the old shape and is corrected: the loop entries are listed, topology_summary.sh was missing, and 01_bootstrap.sh is no longer 'one CI step'.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

.github/workflows/test-deploy-swarm.yml

@@ -56,11 +56,11 @@ permissions:
   packages: read
 
 env:
+  INFINITO_DISTROS: ${{ inputs.distros }}
   NIX_CONFIG: |
     access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
   INFINITO_IMAGE_TAG: ci-${{ github.sha }}
   INFINITO_GHCR_MIRROR_PREFIX: mirror
-  INFINITO_SWARM_STEP_TIMEOUT_MINUTES: "345"
   INFINITO_APP_DISCOVERY_RUNNER: host
   INFINITO_DEPLOY_MODE: swarm
   INFINITO_LIFECYCLES: ${{ inputs.lifecycles }}
@@ -205,24 +205,11 @@ jobs:
           source scripts/meta/env/load.sh
           echo "${VENV}/bin" >> "${GITHUB_PATH}"
 
-      - name: "🥾 Bring up + bootstrap the simulated cluster (compose up, IPs, lab DNS)"
-        run: ${{ env.SCRIPTS }}/01_bootstrap.sh
-
-      - name: "🔧 Project setup (.dockerignore + dotenv + scripts/setup.sh)"
-        run: make setup
-
-      - name: "🚀 Matrix-deploy ${{ matrix.apps }}: provision/deploy/e2e/verify per round"
-        timeout-minutes: ${{ fromJSON(env.INFINITO_SWARM_STEP_TIMEOUT_MINUTES) }}
+      - name: "🚀 Drill ${{ matrix.apps }} per distro: ${{ inputs.distros }}"
         shell: bash
         env:
           variant: ${{ matrix.variant }}
-        run: |
-          source scripts/meta/env/load.sh
-          if [ "$(id -u)" -eq 0 ]; then
-            python3 -m utils.tests.swarm.matrix 2>&1 | tee "/tmp/ansible-${{ matrix.apps }}.log"
-          else
-            sudo -E env "PATH=$PATH" "HOME=$HOME" "SWARM_NAME=${SWARM_NAME:-}" "variant=${variant:-}" "disable=${disable:-}" python3 -m utils.tests.swarm.matrix 2>&1 | tee "/tmp/ansible-${{ matrix.apps }}.log"
-          fi
+        run: ./scripts/tests/deploy/distros.sh "${{ env.SCRIPTS }}/00_one.sh" 2>&1 | tee "/tmp/ansible-${{ matrix.apps }}.log"
 
       - name: "📊 Role runtimes"
         if: always()
@@ -247,24 +234,6 @@ jobs:
           path: /tmp/role-runtimes-${{ matrix.apps }}.csv
           if-no-files-found: ignore
 
-      - name: "🌱 Seed content (pre-drain marker on NFS volume)"
-        run: ${{ env.SCRIPTS }}/05_seed_content.sh
-
-      - name: "💪 Drain the worker running ${{ matrix.apps }} + force reschedule"
-        run: ${{ env.SCRIPTS }}/06_drain_worker.sh
-
-      - name: "✅ Assert state survived the reschedule"
-        run: ${{ env.SCRIPTS }}/07_assert_state.sh
-
-      - name: "🩺 Collect rescue diagnostics (recursive DiD snapshot)"
-        if: failure()
-        timeout-minutes: 25
-        shell: bash
-        run: |
-          INFINITO_RESCUE_DIAGNOSTICS_DIR="${INFINITO_RESCUE_DIAGNOSTICS_BASE}/${{ matrix.apps }}" \
-            python3 utils/diagnostics/container.py \
-            "${{ matrix.apps }}" "post-deploy failure" || true
-
       - name: "⬆️ Upload rescue diagnostics"
         if: ${{ always() && !env.ACT }}
         uses: actions/upload-artifact@v7
@@ -273,11 +242,6 @@ jobs:
           path: ${{ env.INFINITO_RESCUE_DIAGNOSTICS_BASE }}/
           if-no-files-found: ignore
 
-      - name: "📊 Collect diagnostics on failure"
-        if: failure()
-        timeout-minutes: 15
-        run: scripts/tests/deploy/swarm/utils/collect/diagnostics.sh
-
       - name: "🗜️ Zip generated inventory for debugging"
         if: always()
         shell: bash
@@ -293,19 +257,6 @@ jobs:
           path: /tmp/inventory-swarm-${{ matrix.apps }}.zip
           if-no-files-found: ignore
 
-      - name: "📊 Collect Playwright reports from the manager"
-        if: always()
-        timeout-minutes: 15
-        shell: bash
-        env:
-          APP_ID: ${{ matrix.apps }}
-        run: scripts/tests/deploy/swarm/utils/collect/playwright_reports.sh
-
-      - name: "🕸️ Swarm structure summary"
-        if: always()
-        shell: bash
-        run: scripts/tests/deploy/swarm/utils/collect/topology_summary.sh
-
       - name: "📋 Playwright step summary"
         if: always()
         shell: bash

scripts/tests/deploy/swarm/README.md

@@ -142,10 +142,12 @@ the cluster declaration (image, containers, network, DNS play) in
 
 | Stage | Script | Purpose |
 |---|---|---|
+| loop | `scripts/tests/deploy/distros.sh` | SPOT: run the drill once per distro under the shared time budget |
+| loop | `routine/00_one.sh` | the whole drill for one distro; collect + teardown run from its `EXIT` trap |
 | bring-up | `utils/topology/base.sh` | SPOT: node names + NFS export/state paths (sourced, not run) |
 | bring-up | `utils/topology/export.sh` | write the topology SPOT into `$GITHUB_ENV` |
 | bring-up | `compose/swarm/compose.yml` + `compose/swarm/Dockerfile` | declare the 5 node containers, node image + lab network (compose SPOT) |
-| bring-up | `routine/01_bootstrap.sh` | one CI step, host side: pre-clean, `compose build` + one `compose up`, sudo `.deb` build, then the play |
+| bring-up | `routine/01_bootstrap.sh` | host side: pre-clean, `compose build` + one `compose up`, sudo `.deb` build, then the play |
 | bring-up | `compose/swarm/playbook.yml` | node side over docker connection: systemd wait, NFS-export wipe, IPs into `$GITHUB_ENV`, lab DNS, repo + `.deb` install |
 | deploy | `routine/02_provision_inventory.sh` | provision the per-round inventory |
 | deploy | `routine/03_wait_converge.sh` | wait for every stack service to converge |
@@ -156,7 +158,8 @@ the cluster declaration (image, containers, network, DNS play) in
 | chaos | `routine/06_drain_worker.sh` | drain the app's worker + force reschedule |
 | chaos | `routine/07_assert_state.sh` | assert the marker + reachability survived |
 | teardown | `utils/collect/diagnostics.sh` | collect stack/service diagnostics on failure |
-| teardown | `utils/collect/playwright_reports.sh` | pull Playwright reports from the manager |
+| teardown | `utils/collect/playwright_reports.sh` | pull Playwright reports from the manager into `/tmp/playwright-artifacts/<distro>/<app>` |
+| teardown | `utils/collect/topology_summary.sh` | render the node list + service placement into `$GITHUB_STEP_SUMMARY` |
 | teardown | `utils/clean/teardown.sh` | kill the nodes + remove the lab network |
 | helper | `utils/_context.sh` | per-app facts (entity, service, NFS volume, probes) |
 | helper | `utils/unmount_nfs_mounts.sh` | best-effort NFS unmount before node removal |
@@ -165,7 +168,8 @@ the cluster declaration (image, containers, network, DNS play) in
 
 The matrix orchestrator
 (`utils/tests/swarm/matrix.py`) drives the deploy stage
-per variant round; the workflow `.github/workflows/test-deploy-swarm.yml`
-drives the surrounding stages. Run one app locally with
+per variant round; `routine/00_one.sh` drives the surrounding stages for one
+distro, and the workflow `.github/workflows/test-deploy-swarm.yml` drives the
+distro loop. Run one app locally with
 `make swarm-zombie app=<id>` (keeps the cluster for inspection) or the
 whole matrix via `make roundtrip`.

scripts/tests/deploy/swarm/routine/00_one.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Full swarm drill for ONE distro: bring the lab cluster up, matrix-deploy the
+# app, seed/drain/assert, then always collect artefacts and tear the cluster
+# down.
+#
+# Param:
+#   APP_ID                              app id under test
+#   SWARM_NAME                          cluster id
+#   INFINITO_DISTRO                     distro under test (exported by distros.sh)
+#   INFINITO_RESCUE_DIAGNOSTICS_BASE    host dir the rescue snapshots land under
+#   INFINITO_SWARM_STEP_TIMEOUT_MINUTES deploy-step ceiling
+#   variant                             optional matrix variant pin
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../../.." && pwd)"
+cd "${REPO_ROOT}"
+
+: "${APP_ID:?APP_ID is required (matrix.apps)}"
+: "${INFINITO_DISTRO:?INFINITO_DISTRO is required (exported by scripts/tests/deploy/distros.sh)}"
+: "${INFINITO_RESCUE_DIAGNOSTICS_BASE:?INFINITO_RESCUE_DIAGNOSTICS_BASE is required}"
+: "${INFINITO_SWARM_STEP_TIMEOUT_MINUTES:?INFINITO_SWARM_STEP_TIMEOUT_MINUTES is required}"
+
+collect_and_teardown() {
+	rc=$?
+
+	if [ "${rc}" -ne 0 ]; then
+		INFINITO_RESCUE_DIAGNOSTICS_DIR="${INFINITO_RESCUE_DIAGNOSTICS_BASE}/${INFINITO_DISTRO}/${APP_ID}" \
+			timeout 1500 python3 utils/diagnostics/container.py \
+			"${APP_ID}" "post-deploy failure" || true
+		timeout 900 bash "${SCRIPT_DIR}/../utils/collect/diagnostics.sh" || true
+	fi
+
+	timeout 900 bash "${SCRIPT_DIR}/../utils/collect/playwright_reports.sh" || true
+	bash "${SCRIPT_DIR}/../utils/collect/topology_summary.sh" || true
+	timeout 900 bash "${SCRIPT_DIR}/../utils/clean/teardown.sh" || true
+
+	return "${rc}"
+}
+trap collect_and_teardown EXIT
+
+INFINITO_IMAGE="$(bash "${REPO_ROOT}/scripts/meta/resolve/image/ci.sh")"
+export INFINITO_IMAGE
+
+bash "${SCRIPT_DIR}/01_bootstrap.sh"
+
+make setup
+
+# shellcheck source=scripts/meta/env/load.sh
+source scripts/meta/env/load.sh
+
+matrix_cmd=(python3 -m utils.tests.swarm.matrix)
+if [ "$(id -u)" -ne 0 ]; then
+	matrix_cmd=(
+		sudo -E env
+		"PATH=$PATH"
+		"HOME=$HOME"
+		"SWARM_NAME=${SWARM_NAME:-}"
+		"INFINITO_DISTRO=${INFINITO_DISTRO}"
+		"INFINITO_IMAGE=${INFINITO_IMAGE:-}"
+		"variant=${variant:-}"
+		"disable=${disable:-}"
+		"${matrix_cmd[@]}"
+	)
+fi
+timeout "$((INFINITO_SWARM_STEP_TIMEOUT_MINUTES * 60))" "${matrix_cmd[@]}"
+
+bash "${SCRIPT_DIR}/05_seed_content.sh"
+bash "${SCRIPT_DIR}/06_drain_worker.sh"
+bash "${SCRIPT_DIR}/07_assert_state.sh"

scripts/tests/deploy/swarm/utils/collect/playwright_reports.sh

@@ -8,8 +8,9 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 source <(grep -E '^INFINITO_PLAYWRIGHT_REPORTS_BASE_DIR=' "${SCRIPT_DIR}/../../../../../../.env")
 
 : "${APP_ID:?APP_ID required}"
+: "${INFINITO_DISTRO:?INFINITO_DISTRO required}"
 
-dest="/tmp/playwright-artifacts/${APP_ID}"
+dest="/tmp/playwright-artifacts/${INFINITO_DISTRO}/${APP_ID}"
 mkdir -p "${dest}"
 
 if ! timeout 10 docker exec "${MGR}" true 2>/dev/null; then

scripts/tests/deploy/swarm/utils/collect/topology_summary.sh

@@ -8,13 +8,14 @@ set -euo pipefail
 
 : "${GITHUB_STEP_SUMMARY:?GITHUB_STEP_SUMMARY must be set}"
 : "${MGR:?topology export.sh must run first}"
+: "${INFINITO_DISTRO:?INFINITO_DISTRO must be set}"
 
 dexec() {
 	timeout 30 docker exec "$@" 2>&1 || echo "(unreachable)"
 }
 
 {
-	echo "## 🕸️ Swarm structure — ${SWARM_NAME:-unknown}"
+	echo "## 🕸️ Swarm structure — ${SWARM_NAME:-unknown} (${INFINITO_DISTRO})"
 	echo
 	echo "| Role | Host |"
 	echo "|---|---|"