fix(ci): fit the rescue capture into the kill grace of a budget kill

distros.sh stops a distro at its budget with `timeout -k 60`: SIGTERM at
the deadline, SIGKILL 60 s later. dedicated.sh's cleanup trap then ran
the in-container capture under `timeout 1500` (266 s in Baserow#0 of run
40625679270) and the host capture under `timeout 900`, so the SIGKILL
always landed mid-capture, the snapshot was never copied out and the
rescue-diagnostics artifact was lost. The inner `timeout` also sat in its
own process group and outlived the kill, holding the step open.

- distros.sh keeps the grace in one place, KILL_GRACE_SECONDS, and
  exports INFINITO_CI_DISTRO_KILL_EPOCH next to the deadline epoch.
- scripts/tests/deploy/utils/rescue_window.sh decides per trap step:
  past the deadline, the step gets half of the time left until the
  SIGKILL (at least 1 s) and an end epoch; before it, the nominal
  timeout and no end epoch. Inside the trap `$?` is 0 on a budget kill,
  so the deadline epoch is the only reliable signal.
- dedicated.sh runs both captures through `timeout --foreground` with
  that window, keeping them in the trap's process group, and passes the
  end epoch as RESCUE_DEADLINE.
- utils/diagnostics/container.py caps every command at RESCUE_DEADLINE,
  writes `[skipped: past RESCUE_DEADLINE]` for what no longer fits and
  hands the deadline to nested captures. Without the variable nothing
  changes, so the swarm and workspace callers behave as before.

-k 60, the distro budget and every timeout-minutes are unchanged. Drill
tests replay the trap with the real helper and container.py: a budget
kill ends with rc 124 and the snapshot copied out, the old shape ends
with rc 137 and nothing copied, an early failure keeps the full capture.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/unit/python/scripts/tests/deploy/test_distros.py

@@ -149,6 +149,20 @@ class TestDistrosLoop(unittest.TestCase):
         self.assertIn("❌ failed", self.summary)
         self.assertRegex(self.summary, r"rc=124 after \d+s, with \d+s of the 5s budget")
 
+    def test_the_kill_epoch_trails_the_deadline_by_the_kill_grace(self) -> None:
+        proc, record = self._run(
+            "debian",
+            'echo "$((INFINITO_CI_DISTRO_KILL_EPOCH - INFINITO_CI_DISTRO_DEADLINE_EPOCH))"'
+            ' >> "${RECORD}"',
+            INFINITO_CI_DISTRO_BUDGET_SECONDS="600",
+        )
+        self.assertEqual(proc.returncode, 0, proc.stderr)
+        loop = read_text(str(LOOP))
+        grace = re.search(r"^KILL_GRACE_SECONDS=(\d+)$", loop, re.MULTILINE)
+        self.assertIsNotNone(grace, loop)
+        self.assertIn('timeout -k "${KILL_GRACE_SECONDS}"', loop)
+        self.assertEqual(record.split(), [grace.group(1)])
+
     def test_the_job_summary_tables_every_distro_in_execution_order(self) -> None:
         proc, _record = self._run(
             DISTROS,

tests/unit/python/scripts/tests/deploy/utils/test_rescue_window.py

@@ -0,0 +1,206 @@
+"""Budget-kill drills for the rescue window of the compose trap."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+from utils.cache.files import PROJECT_ROOT, read_text
+
+DEPLOY = PROJECT_ROOT / "scripts" / "tests" / "deploy"
+HELPER = DEPLOY / "utils" / "rescue_window.sh"
+INDEX = DEPLOY / "utils" / "rescue_index.sh"
+DEDICATED = DEPLOY / "ci" / "dedicated.sh"
+COLLECTOR = PROJECT_ROOT / "utils" / "diagnostics" / "container.py"
+DISTRO_PREFIX = "INFINITO_CI_DISTRO_"  # nocheck: prefix the drills strip, not a key
+GRACE = 8
+SKIPPED = b"[skipped: past RESCUE_DEADLINE]\n"
+
+WINDOW = """\
+set -euo pipefail
+. "$1"
+now="$(date +%s)"
+if [ -n "$2" ]; then
+	export INFINITO_CI_DISTRO_DEADLINE_EPOCH="$((now + $2))"
+	export INFINITO_CI_DISTRO_KILL_EPOCH="$((now + $3))"
+fi
+rescue_window "$4"
+printf '%s %s %s\\n' "${now}" "${rescue_secs}" "${rescue_until:--}"
+"""
+
+DRIVER = """\
+#!/usr/bin/env bash
+set -euo pipefail
+now="$(date +%s)"
+deadline="$((now + BUDGET))"
+if [ "${EPOCHS}" = true ]; then
+	export INFINITO_CI_DISTRO_DEADLINE_EPOCH="${deadline}"
+	export INFINITO_CI_DISTRO_KILL_EPOCH="$((deadline + GRACE))"
+fi
+echo "kill_epoch=$((deadline + GRACE))"
+timeout -k "${GRACE}" "$((deadline - now))s" bash "${SANDBOX}/stand-in.sh" "$@"
+"""
+
+STAND_IN = """\
+#!/usr/bin/env bash
+set -euo pipefail
+. "${HELPER}"
+cleanup() {
+	local rescue_secs rescue_until
+	rescue_window 1500
+	INFINITO_RESCUE_DIAGNOSTICS_DIR="${SANDBOX}/in-container" \\
+		INFINITO_DNS53_SAMPLER_LOG="${SANDBOX}/dns53.log" \\
+		RESCUE_DEADLINE="${rescue_until}" \\
+		PATH="${SANDBOX}/bin" \\
+		"${TIMEOUT_BIN}" --foreground "${rescue_secs}" "${PYTHON_BIN}" "${COLLECTOR}" app drill >/dev/null 2>&1 || true
+	mkdir -p "${SANDBOX}/host"
+	tar -C "${SANDBOX}/in-container" -cf - . 2>/dev/null | tar -C "${SANDBOX}/host" -xf - 2>/dev/null || true
+	bash "${INDEX}" "${SANDBOX}/host"
+	date +%s >"${SANDBOX}/rescued"
+}
+trap cleanup EXIT
+"$@"
+"""
+
+STUB_JOURNAL = """\
+#!/bin/sh
+echo "RESCUE_DEADLINE=[${RESCUE_DEADLINE}]"
+exec @SLEEP@ "${JOURNAL_SECONDS}"
+"""
+
+
+def _env(**extra: str) -> dict[str, str]:
+    env = {k: v for k, v in os.environ.items() if not k.startswith(DISTRO_PREFIX)}
+    env.update(BASH_ENV="", **extra)
+    return env
+
+
+class RescueWindowTests(unittest.TestCase):
+    def _window(self, deadline: str, kill: str, nominal: int) -> tuple[int, int, str]:
+        proc = subprocess.run(
+            ["bash", "-c", WINDOW, "_", str(HELPER), deadline, kill, str(nominal)],
+            env=_env(),
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        now, secs, until = proc.stdout.splitlines()[-1].split()
+        return int(now), int(secs), until
+
+    def test_outside_distros_sh_a_step_keeps_its_nominal_time(self) -> None:
+        _now, secs, until = self._window("", "", 1500)
+        self.assertEqual((secs, until), (1500, "-"))
+
+    def test_before_the_deadline_a_step_keeps_its_nominal_time(self) -> None:
+        _now, secs, until = self._window("30", "90", 900)
+        self.assertEqual((secs, until), (900, "-"))
+
+    def test_past_the_deadline_a_step_gets_half_the_grace_left(self) -> None:
+        now, secs, until = self._window("-1", "59", 1500)
+        self.assertIn(secs, (29, 30))
+        self.assertLessEqual(abs(int(until) - (now + secs)), 1)
+
+    def test_a_kill_already_due_still_leaves_a_timeout_that_fires(self) -> None:
+        _now, secs, until = self._window("-10", "-1", 1500)
+        self.assertEqual(secs, 1)
+        self.assertNotEqual(until, "-")
+
+
+class DedicatedWiringTests(unittest.TestCase):
+    def test_every_capture_runs_inside_the_window(self) -> None:
+        text = read_text(str(DEDICATED))
+        self.assertIn("utils/rescue_window.sh", text)
+        self.assertIn("rescue_window 1500", text)
+        self.assertIn("rescue_window 900", text)
+        commands = text.replace("\\\n", " ").replace('"', "").splitlines()
+        captures = [c for c in commands if "utils/diagnostics/container.py" in c]
+        self.assertEqual(len(captures), 2, captures)
+        for capture in captures:
+            self.assertIn("timeout --foreground ${rescue_secs}", capture)
+            self.assertIn("RESCUE_DEADLINE=${rescue_until}", capture)
+
+
+class BudgetKillDrillTests(unittest.TestCase):
+    def _drill(
+        self, *deploy: str, budget: int, journal_seconds: str, epochs: bool
+    ) -> tuple[subprocess.CompletedProcess, int, int | None, dict[str, bytes]]:
+        with tempfile.TemporaryDirectory() as tmp:
+            sandbox = Path(tmp)
+            (sandbox / "bin").mkdir()
+            journal = sandbox / "bin" / "journalctl"
+            journal.write_text(STUB_JOURNAL.replace("@SLEEP@", shutil.which("sleep")))
+            journal.chmod(0o755)
+            (sandbox / "stand-in.sh").write_text(STAND_IN)
+            (sandbox / "driver.sh").write_text(DRIVER)
+            proc = subprocess.run(
+                ["bash", str(sandbox / "driver.sh"), *deploy],
+                env=_env(
+                    SANDBOX=str(sandbox),
+                    HELPER=str(HELPER),
+                    INDEX=str(INDEX),
+                    COLLECTOR=str(COLLECTOR),
+                    PYTHON_BIN=sys.executable,
+                    TIMEOUT_BIN=shutil.which("timeout"),
+                    BUDGET=str(budget),
+                    GRACE=str(GRACE),
+                    EPOCHS="true" if epochs else "false",
+                    JOURNAL_SECONDS=journal_seconds,
+                ),
+                capture_output=True,
+                text=True,
+                check=False,
+                timeout=120,
+            )
+            kill_epoch = int(proc.stdout.split("kill_epoch=", 1)[1].split()[0])
+            marker = sandbox / "rescued"
+            rescued = (
+                int(marker.read_text())  # nocheck: cache-read - tempdir fixture
+                if marker.is_file()
+                else None
+            )
+            files = {
+                path.name: path.read_bytes()  # nocheck: cache-read - tempdir fixture
+                for path in (sandbox / "host").glob("app-*/*")
+                if path.is_file()
+            }
+            return proc, kill_epoch, rescued, files
+
+    def test_a_budget_kill_copies_a_partial_snapshot_out_inside_the_grace(self) -> None:
+        proc, kill_epoch, rescued, files = self._drill(
+            "sleep", "1000", budget=1, journal_seconds="30", epochs=True
+        )
+        self.assertEqual(proc.returncode, 124, proc.stdout)
+        self.assertIn("Budget kill:", proc.stdout)
+        self.assertIsNotNone(rescued, proc.stdout)
+        self.assertLess(rescued, kill_epoch)
+        self.assertIn(b"application_id: app", files["meta.txt"])
+        self.assertIn(b"timed out", files["journal.txt"])
+        self.assertEqual(files["journal-errors.txt"], SKIPPED)
+        self.assertIn("Rescue diagnostics:", proc.stdout)
+
+    def test_without_the_window_the_kill_lands_before_the_copy(self) -> None:
+        proc, _kill_epoch, rescued, files = self._drill(
+            "sleep", "1000", budget=1, journal_seconds="30", epochs=False
+        )
+        self.assertEqual(proc.returncode, 137, proc.stdout)
+        self.assertIsNone(rescued)
+        self.assertEqual(files, {})
+
+    def test_a_failure_before_the_deadline_keeps_the_unbounded_capture(self) -> None:
+        proc, _kill_epoch, rescued, files = self._drill(
+            "sh", "-c", "exit 3", budget=60, journal_seconds="0.2", epochs=True
+        )
+        self.assertEqual(proc.returncode, 3, proc.stdout)
+        self.assertNotIn("Budget kill:", proc.stdout)
+        self.assertIsNotNone(rescued)
+        self.assertEqual(files["journal.txt"], b"RESCUE_DEADLINE=[]\n")
+        self.assertNotIn(SKIPPED, files.values())
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/python/utils/diagnostics/test_container.py

@@ -10,6 +10,7 @@ import io
 import os
 import subprocess
 import tempfile
+import time
 import unittest
 import unittest.mock as mock
 from pathlib import Path
@@ -620,6 +621,86 @@ class RecurseTests(unittest.TestCase):
             self.assertTrue(any("rm" in c for c in calls))
 
 
+class DeadlineTests(unittest.TestCase):
+    def _run_with(self, deadline: str, **kwargs):
+        mod = _load()
+        with (
+            mock.patch.dict(os.environ, {mod._DEADLINE_ENV: deadline}),
+            mock.patch.object(mod.subprocess, "run", return_value=_cp([])) as spawn,
+        ):
+            result = mod.run(["probe"], **kwargs)
+        return mod, result, spawn
+
+    def test_without_a_deadline_the_nominal_timeout_stands(self):
+        _mod, _result, spawn = self._run_with("", timeout=600)
+        self.assertEqual(spawn.call_args.kwargs["timeout"], 600)
+
+    def test_a_command_is_capped_at_the_deadline(self):
+        _mod, _result, spawn = self._run_with(str(time.time() + 5), timeout=600)
+        self.assertGreater(spawn.call_args.kwargs["timeout"], 0)
+        self.assertLessEqual(spawn.call_args.kwargs["timeout"], 5)
+
+    def test_a_command_past_the_deadline_is_skipped_unrun(self):
+        mod, result, spawn = self._run_with(str(time.time() - 1))
+        spawn.assert_not_called()
+        self.assertEqual(result.returncode, 124)
+        self.assertEqual(result.stderr, mod._SKIPPED)
+
+    def test_a_running_command_is_cut_at_the_deadline(self):
+        mod = _load()
+        with mock.patch.dict(os.environ, {mod._DEADLINE_ENV: str(time.time() + 1)}):
+            started = time.monotonic()
+            result = mod.run(["sleep", "30"])
+        self.assertEqual(result.returncode, 124)
+        self.assertLess(time.monotonic() - started, 10)
+
+    def test_the_deadline_reaches_the_nested_capture(self):
+        mod = _load()
+        calls: list[list[str]] = []
+
+        def fake_run(cmd, **kw):
+            calls.append(cmd)
+            if cmd[-1] == "{{.Names}}":
+                return _cp(cmd, stdout=b"node1\n")
+            if "{{.Id}}" in cmd:
+                return _cp(cmd, stdout=b"d00d1234\n")
+            return _cp(cmd)
+
+        deadline = str(int(time.time()) + 3600)
+        with (
+            tempfile.TemporaryDirectory() as td,
+            mock.patch.dict(os.environ, {mod._DEADLINE_ENV: deadline}),
+            mock.patch.object(mod, "run", side_effect=fake_run),
+        ):
+            mod.recurse(Path(td), "docker", "app", "ctx", 0, [], "S")
+        nested_exec = next(c for c in calls if "python3" in c)
+        self.assertIn(f"{mod._DEADLINE_ENV}={deadline}", nested_exec)
+
+    def test_a_capture_past_the_deadline_still_leaves_a_snapshot(self):
+        mod = _load()
+        with tempfile.TemporaryDirectory() as td:
+            env = {
+                "INFINITO_RESCUE_DIAGNOSTICS_DIR": td,
+                mod._DEADLINE_ENV: str(time.time() - 1),
+            }
+            with (
+                mock.patch.dict(os.environ, env),
+                mock.patch.object(mod, "runtime_bin", return_value="docker"),
+                mock.patch.object(mod.subprocess, "run") as spawn,
+            ):
+                self.assertEqual(mod.main(["rescue.py", "app", "ctx"]), 1)
+            spawn.assert_not_called()
+            snapshot = next(Path(td).glob("app-*"))
+            meta = (
+                snapshot / "meta.txt"
+            ).read_text()  # nocheck: cache-read - tempdir fixture
+            journal = (
+                snapshot / "journal.txt"
+            ).read_bytes()  # nocheck: cache-read - tempdir fixture
+            self.assertIn("application_id: app", meta)
+            self.assertEqual(journal, mod._SKIPPED)
+
+
 class MainTests(unittest.TestCase):
     def test_main_requires_output_dir(self):
         mod = _load()