fix(ci): let a mid-run budget kill fail, and stop gambling on the start

508669809 booked a budget SIGKILL as skipped so the job went green. That is a
soft-skip: a kill is indistinguishable from the command hanging, so it hid a
real failure behind a green tick.

Fail on it again, and prevent it upstream instead. A distro now starts only
while the remaining budget covers the slowest run so far plus 30%. Fediwall run
31756400129 shows why the bare comparison was not enough: max_seen=6770s against
remaining=6878s let the distro start, and it took 6942s, 64s past the deadline.
max_seen is the maximum of the durations already observed in the same sweep -
a small sample over distros of genuinely different weight, drawn in randomised
order, so it underestimates systematically.

Removing the soft-skip also removed its diagnosis, so a killed distro now
reports both numbers without claiming a cause: the run only knows the exit code
and how much budget was left when it started.

The suite drops from 85s to 4.1s. The 68s case made a process ignore SIGTERM to
force the -k escalation, which exercised coreutils rather than this script; the
124 path reaches the same branch in a second.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

scripts/tests/deploy/distros.sh

@@ -2,8 +2,8 @@
 set -euo pipefail
 
 # SPOT: run one command once per distro, in random order, under a shared time
-# budget. A distro the budget drops is reported as skipped, not as a failure;
-# only the per-distro command failing fails the run. Every outcome lands in a
+# budget. A distro the budget drops BEFORE it starts is reported as skipped; a
+# distro the budget kills MID-RUN fails the job. Every outcome lands in a
 # job-summary table in execution order.
 #
 # Param:
@@ -25,6 +25,8 @@ source "scripts/meta/env/load.sh"
 : "${INFINITO_DISTROS:?INFINITO_DISTROS is required (e.g. 'arch debian ubuntu fedora centos')}"
 : "${INFINITO_CI_DISTRO_BUDGET_SECONDS:?INFINITO_CI_DISTRO_BUDGET_SECONDS is required (declared in default.env)}"
 
+BUDGET_HEADROOM_PERCENT=30
+
 if (($# == 0)); then
 	echo "[ERROR] a per-distro command is required" >&2
 	exit 2
@@ -109,10 +111,11 @@ for i in "${!distro_arr[@]}"; do
 		break
 	fi
 
-	if ((max_seen > 0 && remaining < max_seen)); then
-		echo "[WARN] Skipping distro=${distro}: remaining=${remaining}s < max_seen=${max_seen}s (fast-fail heuristic)"
+	needed="$((max_seen * (100 + BUDGET_HEADROOM_PERCENT) / 100))"
+	if ((max_seen > 0 && remaining < needed)); then
+		echo "[WARN] Skipping distro=${distro}: remaining=${remaining}s < ${needed}s (max_seen=${max_seen}s + ${BUDGET_HEADROOM_PERCENT}% headroom)"
 		skipped=$((skipped + 1))
-		notes[i]="remaining ${remaining}s < slowest run ${max_seen}s"
+		notes[i]="remaining ${remaining}s < ${needed}s needed"
 		continue
 	fi
 
@@ -141,23 +144,18 @@ for i in "${!distro_arr[@]}"; do
 
 	echo ">>> Duration: distro=${distro} took ${dur}s (max_seen=${max_seen}s)"
 
-	if [[ $rc -eq 124 || $rc -eq 137 ]] && ((dur >= remaining - 5)) && ((passed > 0)); then
-		statuses[i]="skipped"
-		notes[i]="budget exhausted mid-run after ${dur}s"
-		skipped=$((skipped + 1))
-		ran=$((ran - 1))
-		echo "[WARN] Budget exhausted while distro=${distro} was running; stopping."
-		break
-	fi
-
 	if [[ $rc -ne 0 ]]; then
 		statuses[i]="failed"
-		notes[i]="rc=${rc}"
+		if [[ $rc -eq 124 || $rc -eq 137 ]] && ((dur >= remaining)); then
+			notes[i]="rc=${rc} after ${dur}s, with ${remaining}s of the ${INFINITO_CI_DISTRO_BUDGET_SECONDS}s budget left at start"
+		else
+			notes[i]="rc=${rc}"
+		fi
 		skipped=$((skipped + ${#distro_arr[@]} - i - 1))
 		for ((j = i + 1; j < ${#distro_arr[@]}; j++)); do
 			notes[j]="not run: aborted after distro=${distro} failed"
 		done
-		echo "[ERROR] Run failed for distro=${distro} (rc=${rc})" >&2
+		echo "[ERROR] Run failed for distro=${distro}: ${notes[i]}" >&2
 		render_summary
 		exit "$rc"
 	fi

tests/unit/scripts/tests/deploy/test_distros.py

@@ -13,6 +13,7 @@ hard error rather than a guess.
 from __future__ import annotations
 
 import os
+import re
 import subprocess
 import tempfile
 import unittest
@@ -108,27 +109,30 @@ class TestDistrosLoop(unittest.TestCase):
         self.assertEqual(proc.returncode, 7, proc.stderr)
         self.assertEqual(len(record.split()), 2)
 
-    def test_budget_skips_a_distro_that_no_longer_fits(self) -> None:
+    def test_a_distro_needs_the_slowest_run_plus_headroom_to_start(self) -> None:
         proc, record = self._run(
             DISTROS,
-            'sleep 3\necho "${INFINITO_DISTRO}" >> "${RECORD}"',
-            INFINITO_CI_DISTRO_BUDGET_SECONDS="5",
+            'sleep 2\necho "${INFINITO_DISTRO}" >> "${RECORD}"',
+            INFINITO_CI_DISTRO_BUDGET_SECONDS="3",
         )
         self.assertEqual(proc.returncode, 0, proc.stderr)
         self.assertEqual(len(record.split()), 1)
-        self.assertRegex(proc.stdout, r"Skipping distro|budget exhausted")
 
-    def test_a_distro_that_ignores_sigterm_is_skipped_not_failed(self) -> None:
-        proc, record = self._run(
-            DISTROS,
-            'echo "${INFINITO_DISTRO}" >> "${RECORD}"\n'
-            'test "$(wc -l < "${RECORD}")" -gt 1 && { trap "" TERM; sleep 300; }\n'
-            "exit 0",
-            INFINITO_CI_DISTRO_BUDGET_SECONDS="8",
+        match = re.search(
+            r"< (\d+)s \(max_seen=(\d+)s \+ (\d+)% headroom\)", proc.stdout
         )
-        self.assertEqual(proc.returncode, 0, proc.stderr)
-        self.assertRegex(proc.stdout, r"Budget exhausted while distro=")
-        self.assertIn("🟦 skipped", self.summary)
+        self.assertIsNotNone(match, proc.stdout)
+        needed, max_seen, percent = (int(g) for g in match.groups())
+        self.assertGreater(percent, 0)
+        self.assertEqual(needed, max_seen * (100 + percent) // 100)
+
+    def test_a_distro_the_budget_kills_mid_run_fails_the_job(self) -> None:
+        proc, _record = self._run(
+            DISTROS, "sleep 30", INFINITO_CI_DISTRO_BUDGET_SECONDS="1"
+        )
+        self.assertEqual(proc.returncode, 124, proc.stderr)
+        self.assertIn("❌ failed", self.summary)
+        self.assertRegex(self.summary, r"rc=124 after \d+s, with \d+s of the 1s budget")
 
     def test_the_job_summary_tables_every_distro_in_execution_order(self) -> None:
         proc, _record = self._run(
@@ -153,7 +157,7 @@ class TestDistrosLoop(unittest.TestCase):
     def test_the_job_summary_marks_skipped_and_failed_distros(self) -> None:
         proc, _record = self._run(
             DISTROS,
-            'sleep 3\necho "${INFINITO_DISTRO}" >> "${RECORD}"\nexit 7',
+            'echo "${INFINITO_DISTRO}" >> "${RECORD}"\nexit 7',
             INFINITO_CI_DISTRO_BUDGET_SECONDS="5",
         )
         self.assertEqual(proc.returncode, 7, proc.stderr)