From b8a6205396563d748031a6c2caacdcd2cb0d7626 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 1 Sep 2026 12:36:10 +0200 Subject: [PATCH] fix(swarm-test): put the step guard on the same side of sudo as its payload Moodle#0 of run 33445831564 was cancelled by the 355-minute job cap with almost no artifacts. The cap did not end a deploy; it reaped a zombie. 00_one.sh wrapped `timeout` around `sudo`, so the guard ran as the unprivileged runner and the payload as root. An unprivileged SIGKILL cannot reach a root process, and the same holds for distros.sh's own `timeout -k` one level up. Probed both shapes: the payload lands in a pgid the outer governor never addresses either way, so --foreground would not have helped -- the privilege wall is what carries. The log shows the consequence directly. The governor fired on schedule: 06:18:27.884 distros.sh: line 103: 15914 Killed timeout -k 60 "${remaining}s" "$@" 06:18:27.894 [ERROR] Run failed for distro=ubuntu: rc=137 after 20460s and then ansible output kept arriving in the same tee'd log -- a TASK at 06:18:27.929, another at 06:26:00.189, elapsed still advancing -- until `##[error]The operation was canceled.` The orphan held the step's pipe open, so tee never saw EOF and the step never returned. 954b9f1ae6 added the TERM trap precisely so a timeout kill would still collect diagnostics, and its reasoning ("GNU timeout signals the whole process group") is correct. It simply never gets signalled on the swarm path, because sudo stands between. That is why this cell carries only inventory- and role-runtimes- while its compose twin also has rescue-diagnostics- and playwright-. This completes that fix rather than replacing it. The ladder was also inverted: INFINITO_CI_DISTRO_BUDGET_SECONDS=20400 (340 min) against INFINITO_SWARM_STEP_TIMEOUT_MINUTES=350, with default.env:97 asserting the order as if correct. For the trap to run, the step guard must fire BEFORE the sweep governor. Rather than re-tune two constants that start their clocks at different points -- matrix start trails drill start by 1m49s to 2m04s, measured -- 00_one.sh now derives its ceiling from the governor's deadline minus a teardown reserve, and keeps its own configured value when that is smaller. Not fixed here, and it must be said: Moodle#0 does not fit on a hosted runner at all. Its pass-2 pacing tracks pass-1 within 5%, putting the pass end at roughly 362 minutes after job start, past both the 355 cap and GitHub's own 360 ceiling. Raising the cap would be masking and is arithmetically impossible anyway. What this buys is an honest red cell at ~323 minutes WITH its diagnostics, instead of a silent cancellation with none. Co-Authored-By: Claude Opus 5 (1M context) --- default.env | 3 +++ scripts/tests/deploy/distros.sh | 1 + scripts/tests/deploy/swarm/routine/00_one.sh | 17 +++++++++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/default.env b/default.env index b4c628a0eb..23257e0f80 100644 --- a/default.env +++ b/default.env @@ -334,6 +334,9 @@ INFINITO_SWARM_NODE_PIDS_LIMIT=8192 # Ceiling in minutes for one distro's swarm matrix deploy in scripts/tests/deploy/swarm/routine/00_one.sh; swarm's step level. INFINITO_SWARM_STEP_TIMEOUT_MINUTES=350 +# Wall clock 00_one.sh leaves collect_and_teardown under the sweep budget; measured worst case 12m44s on the swarm moodle#1 rescue path. +INFINITO_SWARM_TEARDOWN_RESERVE_SECONDS=900 + # Docker label stamped on every act-swarm-test container (DinD + NFS sidecar) so cleanup selects by label, not a guessed name. INFINITO_SWARM_TEST_LABEL=infinito.swarm-test diff --git a/scripts/tests/deploy/distros.sh b/scripts/tests/deploy/distros.sh index 112eb046b9..776ad47335 100755 --- a/scripts/tests/deploy/distros.sh +++ b/scripts/tests/deploy/distros.sh @@ -43,6 +43,7 @@ echo "=== Distro execution order: ${distro_arr[*]} ===" global_start="$(date +%s)" deadline="$((global_start + INFINITO_CI_DISTRO_BUDGET_SECONDS))" +export INFINITO_CI_DISTRO_DEADLINE_EPOCH="${deadline}" echo "=== Global time budget: ${INFINITO_CI_DISTRO_BUDGET_SECONDS}s (deadline epoch=${deadline}) ===" max_seen=0 diff --git a/scripts/tests/deploy/swarm/routine/00_one.sh b/scripts/tests/deploy/swarm/routine/00_one.sh index 18247026d0..505fb24a71 100755 --- a/scripts/tests/deploy/swarm/routine/00_one.sh +++ b/scripts/tests/deploy/swarm/routine/00_one.sh @@ -42,6 +42,7 @@ export SWARM_DRILL_ENV : "${INFINITO_IMAGE_TAG:?INFINITO_IMAGE_TAG is required (source scripts/meta/env/load.sh before invoking this script)}" : "${INFINITO_RESCUE_DIAGNOSTICS_BASE:?INFINITO_RESCUE_DIAGNOSTICS_BASE is required}" : "${INFINITO_SWARM_STEP_TIMEOUT_MINUTES:?INFINITO_SWARM_STEP_TIMEOUT_MINUTES is required}" +: "${INFINITO_SWARM_TEARDOWN_RESERVE_SECONDS:?INFINITO_SWARM_TEARDOWN_RESERVE_SECONDS is required}" SWARM_REQUIRED_SERVICES="node nfs-server container_backup nfs_backup" if [ -n "${disable:-}" ]; then @@ -110,7 +111,19 @@ make setup # shellcheck source=scripts/meta/env/load.sh source scripts/meta/env/load.sh -matrix_cmd=(python3 -m utils.tests.swarm.matrix) +step_timeout="$((INFINITO_SWARM_STEP_TIMEOUT_MINUTES * 60))" +if [ -n "${INFINITO_CI_DISTRO_DEADLINE_EPOCH:-}" ]; then + under_governor="$((INFINITO_CI_DISTRO_DEADLINE_EPOCH - $(date +%s) - INFINITO_SWARM_TEARDOWN_RESERVE_SECONDS))" + if [ "${under_governor}" -le 0 ]; then + echo "[ERROR] no budget left under the sweep governor for the matrix deploy" >&2 + exit 2 + fi + if [ "${under_governor}" -lt "${step_timeout}" ]; then + step_timeout="${under_governor}" + fi +fi + +matrix_cmd=(timeout "${step_timeout}" python3 -m utils.tests.swarm.matrix) if [ "$(id -u)" -ne 0 ]; then matrix_cmd=( sudo -E env @@ -127,7 +140,7 @@ if [ "$(id -u)" -ne 0 ]; then "${matrix_cmd[@]}" ) fi -timeout "$((INFINITO_SWARM_STEP_TIMEOUT_MINUTES * 60))" "${matrix_cmd[@]}" +"${matrix_cmd[@]}" bash "${SCRIPT_DIR}/05_seed_content.sh" bash "${SCRIPT_DIR}/06_drain_worker.sh"