mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-09 12:17:13 +00:00
fix(ci): fit the rescue capture into the kill grace of a budget kill
distros.sh stops a distro at its budget with `timeout -k 60`: SIGTERM at the deadline, SIGKILL 60 s later. dedicated.sh's cleanup trap then ran the in-container capture under `timeout 1500` (266 s in Baserow#0 of run 40625679270) and the host capture under `timeout 900`, so the SIGKILL always landed mid-capture, the snapshot was never copied out and the rescue-diagnostics artifact was lost. The inner `timeout` also sat in its own process group and outlived the kill, holding the step open. - distros.sh keeps the grace in one place, KILL_GRACE_SECONDS, and exports INFINITO_CI_DISTRO_KILL_EPOCH next to the deadline epoch. - scripts/tests/deploy/utils/rescue_window.sh decides per trap step: past the deadline, the step gets half of the time left until the SIGKILL (at least 1 s) and an end epoch; before it, the nominal timeout and no end epoch. Inside the trap `$?` is 0 on a budget kill, so the deadline epoch is the only reliable signal. - dedicated.sh runs both captures through `timeout --foreground` with that window, keeping them in the trap's process group, and passes the end epoch as RESCUE_DEADLINE. - utils/diagnostics/container.py caps every command at RESCUE_DEADLINE, writes `[skipped: past RESCUE_DEADLINE]` for what no longer fits and hands the deadline to nested captures. Without the variable nothing changes, so the swarm and workspace callers behave as before. -k 60, the distro budget and every timeout-minutes are unchanged. Drill tests replay the trap with the real helper and container.py: a budget kill ends with rc 124 and the snapshot copied out, the old shape ends with rc 137 and nothing copied, an early failure keeps the full capture. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1d8d0c41e7
commit
2bf329cd3b
8 files changed
+362
-18
No files matched your search
@@ -10,6 +10,7 @@ import io
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
import unittest.mock as mock
|
||||
from pathlib import Path
|
||||
@@ -620,6 +621,86 @@ class RecurseTests(unittest.TestCase):
|
||||
self.assertTrue(any("rm" in c for c in calls))
|
||||
|
||||
|
||||
class DeadlineTests(unittest.TestCase):
|
||||
def _run_with(self, deadline: str, **kwargs):
|
||||
mod = _load()
|
||||
with (
|
||||
mock.patch.dict(os.environ, {mod._DEADLINE_ENV: deadline}),
|
||||
mock.patch.object(mod.subprocess, "run", return_value=_cp([])) as spawn,
|
||||
):
|
||||
result = mod.run(["probe"], **kwargs)
|
||||
return mod, result, spawn
|
||||
|
||||
def test_without_a_deadline_the_nominal_timeout_stands(self):
|
||||
_mod, _result, spawn = self._run_with("", timeout=600)
|
||||
self.assertEqual(spawn.call_args.kwargs["timeout"], 600)
|
||||
|
||||
def test_a_command_is_capped_at_the_deadline(self):
|
||||
_mod, _result, spawn = self._run_with(str(time.time() + 5), timeout=600)
|
||||
self.assertGreater(spawn.call_args.kwargs["timeout"], 0)
|
||||
self.assertLessEqual(spawn.call_args.kwargs["timeout"], 5)
|
||||
|
||||
def test_a_command_past_the_deadline_is_skipped_unrun(self):
|
||||
mod, result, spawn = self._run_with(str(time.time() - 1))
|
||||
spawn.assert_not_called()
|
||||
self.assertEqual(result.returncode, 124)
|
||||
self.assertEqual(result.stderr, mod._SKIPPED)
|
||||
|
||||
def test_a_running_command_is_cut_at_the_deadline(self):
|
||||
mod = _load()
|
||||
with mock.patch.dict(os.environ, {mod._DEADLINE_ENV: str(time.time() + 1)}):
|
||||
started = time.monotonic()
|
||||
result = mod.run(["sleep", "30"])
|
||||
self.assertEqual(result.returncode, 124)
|
||||
self.assertLess(time.monotonic() - started, 10)
|
||||
|
||||
def test_the_deadline_reaches_the_nested_capture(self):
|
||||
mod = _load()
|
||||
calls: list[list[str]] = []
|
||||
|
||||
def fake_run(cmd, **kw):
|
||||
calls.append(cmd)
|
||||
if cmd[-1] == "{{.Names}}":
|
||||
return _cp(cmd, stdout=b"node1\n")
|
||||
if "{{.Id}}" in cmd:
|
||||
return _cp(cmd, stdout=b"d00d1234\n")
|
||||
return _cp(cmd)
|
||||
|
||||
deadline = str(int(time.time()) + 3600)
|
||||
with (
|
||||
tempfile.TemporaryDirectory() as td,
|
||||
mock.patch.dict(os.environ, {mod._DEADLINE_ENV: deadline}),
|
||||
mock.patch.object(mod, "run", side_effect=fake_run),
|
||||
):
|
||||
mod.recurse(Path(td), "docker", "app", "ctx", 0, [], "S")
|
||||
nested_exec = next(c for c in calls if "python3" in c)
|
||||
self.assertIn(f"{mod._DEADLINE_ENV}={deadline}", nested_exec)
|
||||
|
||||
def test_a_capture_past_the_deadline_still_leaves_a_snapshot(self):
|
||||
mod = _load()
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
env = {
|
||||
"INFINITO_RESCUE_DIAGNOSTICS_DIR": td,
|
||||
mod._DEADLINE_ENV: str(time.time() - 1),
|
||||
}
|
||||
with (
|
||||
mock.patch.dict(os.environ, env),
|
||||
mock.patch.object(mod, "runtime_bin", return_value="docker"),
|
||||
mock.patch.object(mod.subprocess, "run") as spawn,
|
||||
):
|
||||
self.assertEqual(mod.main(["rescue.py", "app", "ctx"]), 1)
|
||||
spawn.assert_not_called()
|
||||
snapshot = next(Path(td).glob("app-*"))
|
||||
meta = (
|
||||
snapshot / "meta.txt"
|
||||
).read_text() # nocheck: cache-read - tempdir fixture
|
||||
journal = (
|
||||
snapshot / "journal.txt"
|
||||
).read_bytes() # nocheck: cache-read - tempdir fixture
|
||||
self.assertIn("application_id: app", meta)
|
||||
self.assertEqual(journal, mod._SKIPPED)
|
||||
|
||||
|
||||
class MainTests(unittest.TestCase):
|
||||
def test_main_requires_output_dir(self):
|
||||
mod = _load()
|
||||
|
||||
Reference in new issue
Block a user