mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-11 05:06:36 +00:00
kevinveenbirkenbachandClaude Opus 5.5
7b6800493b
test_run_streamed_outlives_a_deadline_while_it_keeps_talking patched RUN_SILENCE_SECONDS to 1.0 while the chatty child prints its first line only after the interpreter has started. run_streamed measures silence from process start, which is right for production (a command that never speaks must time out), so under xdist load on a full disk the cold start alone exceeded the window and the run returned 124 (reported from IAM's make quality). The window is now 2.0 s against a 0.1 s cadence over 40 lines (4 s, still twice the window, so the test keeps proving it outlives a deadline while talking), and the child runs with -I -S to skip the site import. With a simulated 1.5 s cold start the old parameters return 124 and the new ones 0. pull.py is unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
472 lines
16 KiB
Python
472 lines
16 KiB
Python
import importlib.util
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import ModuleType
|
|
from unittest.mock import patch
|
|
|
|
from utils.cache.yaml import load_yaml_any
|
|
|
|
from . import PROJECT_ROOT
|
|
|
|
|
|
def _load_module(rel_path: str, name: str) -> ModuleType:
|
|
path = PROJECT_ROOT / rel_path
|
|
spec = importlib.util.spec_from_file_location(name, str(path))
|
|
mod = importlib.util.module_from_spec(spec)
|
|
assert spec and spec.loader
|
|
spec.loader.exec_module(mod)
|
|
return mod
|
|
|
|
|
|
class TestComposePull(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.m = _load_module(
|
|
"roles/sys-svc-compose/files/python/pull.py", "compose_pull_mod"
|
|
)
|
|
|
|
def test_run_cmd_returns_rc_and_split_streams(self) -> None:
|
|
cwd = Path("/")
|
|
env = {}
|
|
|
|
class DummyProc:
|
|
def __init__(self, returncode: int, stdout: str, stderr: str) -> None:
|
|
self.returncode = returncode
|
|
self.stdout = stdout
|
|
self.stderr = stderr
|
|
|
|
def fake_run(*args, **kwargs):
|
|
return DummyProc(7, "hello\n", "warn: variable is not set\n")
|
|
|
|
with patch.object(self.m.subprocess, "run", side_effect=fake_run):
|
|
rc, out, err = self.m.run_cmd(["echo", "x"], cwd=cwd, env=env)
|
|
|
|
self.assertEqual(rc, 7)
|
|
self.assertEqual(out, "hello\n")
|
|
self.assertEqual(err, "warn: variable is not set\n")
|
|
|
|
def test_run_cmd_reports_a_timeout_instead_of_hanging(self) -> None:
|
|
def fake_run(*args, **kwargs):
|
|
self.assertEqual(kwargs["timeout"], self.m.RUN_TIMEOUT_SECONDS)
|
|
raise self.m.subprocess.TimeoutExpired(
|
|
cmd=["compose", "build"],
|
|
timeout=self.m.RUN_TIMEOUT_SECONDS,
|
|
output=b"step 3/4\n",
|
|
stderr=b"pulling\n",
|
|
)
|
|
|
|
with patch.object(self.m.subprocess, "run", side_effect=fake_run):
|
|
rc, out, err = self.m.run_cmd(["compose", "build"], cwd=Path("/"), env={})
|
|
|
|
self.assertEqual(rc, self.m.TIMEOUT_RC)
|
|
self.assertEqual(out, "step 3/4\n")
|
|
self.assertIn(f"timed out after {self.m.RUN_TIMEOUT_SECONDS}s", err)
|
|
self.assertIn("pulling", err)
|
|
|
|
def test_run_streamed_outlives_a_deadline_while_it_keeps_talking(self) -> None:
|
|
chatty = "import time\nfor step in range(40):\n print(step, flush=True)\n time.sleep(0.1)\n"
|
|
|
|
with patch.object(self.m, "RUN_SILENCE_SECONDS", 2.0):
|
|
rc, out, _err = self.m.run_streamed(
|
|
[sys.executable, "-I", "-S", "-c", chatty],
|
|
cwd=Path("/"),
|
|
env=dict(os.environ),
|
|
)
|
|
|
|
self.assertEqual(rc, 0)
|
|
self.assertEqual(out.split()[-1], "39")
|
|
|
|
def test_run_streamed_kills_a_command_that_falls_silent(self) -> None:
|
|
with patch.object(self.m, "RUN_SILENCE_SECONDS", 0.5):
|
|
rc, _out, err = self.m.run_streamed(
|
|
[sys.executable, "-c", "import time; time.sleep(60)"],
|
|
cwd=Path("/"),
|
|
env=dict(os.environ),
|
|
)
|
|
|
|
self.assertEqual(rc, self.m.TIMEOUT_RC)
|
|
self.assertIn("said nothing for 0.5s", err)
|
|
|
|
def test_base_compose_cmd_delegates_to_wrapper(self) -> None:
|
|
cmd = self.m.base_compose_cmd(project="p", cwd=Path("/x"))
|
|
self.assertEqual(cmd[0], "/usr/bin/compose")
|
|
self.assertIn("--chdir", cmd)
|
|
self.assertIn("/x", cmd)
|
|
self.assertIn("--project", cmd)
|
|
self.assertIn("p", cmd)
|
|
self.assertEqual(cmd[-1], "--")
|
|
|
|
def test_has_buildable_services_true(self) -> None:
|
|
config_out = """
|
|
services:
|
|
app:
|
|
build:
|
|
context: .
|
|
image: example/app
|
|
"""
|
|
base_cmd = ["docker", "compose", "-p", "p", "-f", "a.yml"]
|
|
with patch.object(self.m, "run_cmd", return_value=(0, config_out, "")):
|
|
self.assertTrue(
|
|
self.m.has_buildable_services(
|
|
base_cmd=base_cmd, cwd=Path("/tmp"), env={}
|
|
)
|
|
)
|
|
|
|
def test_has_buildable_services_false(self) -> None:
|
|
config_out = """
|
|
services:
|
|
app:
|
|
image: example/app
|
|
"""
|
|
base_cmd = ["docker", "compose", "-p", "p", "-f", "a.yml"]
|
|
with patch.object(self.m, "run_cmd", return_value=(0, config_out, "")):
|
|
self.assertFalse(
|
|
self.m.has_buildable_services(
|
|
base_cmd=base_cmd, cwd=Path("/tmp"), env={}
|
|
)
|
|
)
|
|
|
|
def test_run_or_fail_success_does_not_raise(self) -> None:
|
|
with patch.object(self.m, "run_cmd", return_value=(0, "ok\n", "")):
|
|
self.m.run_or_fail(
|
|
["docker", "compose", "ps"], cwd=Path("/"), env={}, label="x"
|
|
)
|
|
|
|
def test_run_or_fail_failure_raises(self) -> None:
|
|
with (
|
|
patch.object(self.m, "run_cmd", return_value=(9, "", "boom\n")),
|
|
self.assertRaises(RuntimeError) as ctx,
|
|
):
|
|
self.m.run_or_fail(
|
|
["docker", "compose", "pull"], cwd=Path("/"), env={}, label="pull"
|
|
)
|
|
self.assertIn("pull failed", str(ctx.exception))
|
|
|
|
def test_main_short_circuits_when_lock_exists(self) -> None:
|
|
with tempfile.TemporaryDirectory() as td:
|
|
lock_dir = Path(td) / "locks"
|
|
lock_dir.mkdir(parents=True, exist_ok=True)
|
|
lock_key = "abc123"
|
|
(lock_dir / f"{lock_key}.lock").write_text("ok\n", encoding="utf-8")
|
|
|
|
argv = [
|
|
"pull.py",
|
|
"--chdir",
|
|
td,
|
|
"--project",
|
|
"p",
|
|
"--compose-files",
|
|
"-f a.yml -f b.yml",
|
|
"--lock-dir",
|
|
str(lock_dir),
|
|
"--lock-key",
|
|
lock_key,
|
|
]
|
|
|
|
with (
|
|
patch.object(sys, "argv", argv),
|
|
patch.object(self.m, "run_or_fail") as rof_mock,
|
|
patch.object(self.m, "has_buildable_services") as hbs_mock,
|
|
):
|
|
rc = self.m.main()
|
|
|
|
self.assertEqual(rc, 0)
|
|
rof_mock.assert_not_called()
|
|
hbs_mock.assert_not_called()
|
|
|
|
def test_main_runs_build_and_pull_and_writes_lock(self) -> None:
|
|
with tempfile.TemporaryDirectory() as td:
|
|
chdir = Path(td) / "instance"
|
|
chdir.mkdir(parents=True, exist_ok=True)
|
|
|
|
lock_dir = Path(td) / "locks"
|
|
lock_key = "k1"
|
|
|
|
argv = [
|
|
"pull.py",
|
|
"--chdir",
|
|
str(chdir),
|
|
"--project",
|
|
"p",
|
|
"--compose-files",
|
|
"-f a.yml -f b.yml",
|
|
"--env-file",
|
|
"/x/.env",
|
|
"--lock-dir",
|
|
str(lock_dir),
|
|
"--lock-key",
|
|
lock_key,
|
|
"--ignore-buildable",
|
|
]
|
|
|
|
base_cmd = [
|
|
"docker",
|
|
"compose",
|
|
"-p",
|
|
"p",
|
|
"-f",
|
|
"a.yml",
|
|
"-f",
|
|
"b.yml",
|
|
"--env-file",
|
|
"/x/.env",
|
|
]
|
|
|
|
calls: list[list[str]] = []
|
|
|
|
def fake_run_cmd(
|
|
cmd: list[str], *, cwd: Path, env: dict[str, str]
|
|
) -> tuple[int, str, str]:
|
|
calls.append(cmd)
|
|
if cmd[-2:] == ["pull", "--help"]:
|
|
return 0, "Usage:\n --ignore-buildable\n", ""
|
|
return 0, "", ""
|
|
|
|
fallback_calls: list[list[str]] = []
|
|
|
|
def fake_run_or_fail(
|
|
cmd: list[str], *, cwd: Path, env: dict[str, str], label: str
|
|
) -> None:
|
|
fallback_calls.append(cmd)
|
|
|
|
with (
|
|
patch.object(sys, "argv", argv),
|
|
patch.object(self.m, "has_buildable_services", return_value=True),
|
|
patch.object(self.m, "run_cmd", side_effect=fake_run_cmd),
|
|
patch.object(self.m, "run_streamed", side_effect=fake_run_cmd),
|
|
patch.object(self.m, "base_compose_cmd", return_value=base_cmd),
|
|
patch.object(self.m, "run_or_fail", side_effect=fake_run_or_fail),
|
|
):
|
|
rc = self.m.main()
|
|
|
|
self.assertEqual(rc, 0)
|
|
self.assertTrue(
|
|
(lock_dir / f"{lock_key}.lock").exists(),
|
|
"lock file should be written on success",
|
|
)
|
|
|
|
self.assertEqual(calls[0], [*base_cmd, "build", "--pull"])
|
|
self.assertEqual(calls[1], [*base_cmd, "pull", "--help"])
|
|
self.assertEqual(calls[2], [*base_cmd, "config"])
|
|
self.assertEqual(calls[3], [*base_cmd, "pull", "--ignore-buildable"])
|
|
self.assertEqual(fallback_calls, [])
|
|
|
|
def test_main_falls_back_to_plain_build_when_build_pull_fails(self) -> None:
|
|
with tempfile.TemporaryDirectory() as td:
|
|
chdir = Path(td) / "instance"
|
|
chdir.mkdir(parents=True, exist_ok=True)
|
|
|
|
lock_dir = Path(td) / "locks"
|
|
lock_key = "kfb"
|
|
|
|
argv = [
|
|
"pull.py",
|
|
"--chdir",
|
|
str(chdir),
|
|
"--project",
|
|
"p",
|
|
"--compose-files",
|
|
"-f a.yml",
|
|
"--lock-dir",
|
|
str(lock_dir),
|
|
"--lock-key",
|
|
lock_key,
|
|
]
|
|
|
|
base_cmd = ["docker", "compose", "-p", "p", "-f", "a.yml"]
|
|
|
|
def fake_run_cmd(
|
|
cmd: list[str], *, cwd: Path, env: dict[str, str]
|
|
) -> tuple[int, str, str]:
|
|
if cmd[-2:] == ["build", "--pull"]:
|
|
return 1, "", "build --pull failed\n"
|
|
return 0, "", ""
|
|
|
|
fallback_calls: list[list[str]] = []
|
|
|
|
def fake_run_or_fail(
|
|
cmd: list[str], *, cwd: Path, env: dict[str, str], label: str
|
|
) -> None:
|
|
fallback_calls.append(cmd)
|
|
|
|
with (
|
|
patch.object(sys, "argv", argv),
|
|
patch.object(self.m, "has_buildable_services", return_value=True),
|
|
patch.object(self.m, "run_cmd", side_effect=fake_run_cmd),
|
|
patch.object(self.m, "run_streamed", side_effect=fake_run_cmd),
|
|
patch.object(self.m, "base_compose_cmd", return_value=base_cmd),
|
|
patch.object(self.m, "run_or_fail", side_effect=fake_run_or_fail),
|
|
):
|
|
rc = self.m.main()
|
|
|
|
self.assertEqual(rc, 0)
|
|
self.assertEqual(fallback_calls, [[*base_cmd, "build"]])
|
|
|
|
def test_main_pull_omits_ignore_buildable_when_not_supported(self) -> None:
|
|
with tempfile.TemporaryDirectory() as td:
|
|
chdir = Path(td) / "instance"
|
|
chdir.mkdir(parents=True, exist_ok=True)
|
|
|
|
lock_dir = Path(td) / "locks"
|
|
lock_key = "k2"
|
|
|
|
argv = [
|
|
"pull.py",
|
|
"--chdir",
|
|
str(chdir),
|
|
"--project",
|
|
"p",
|
|
"--compose-files",
|
|
"-f a.yml",
|
|
"--lock-dir",
|
|
str(lock_dir),
|
|
"--lock-key",
|
|
lock_key,
|
|
"--ignore-buildable",
|
|
]
|
|
|
|
base_cmd = ["docker", "compose", "-p", "p", "-f", "a.yml"]
|
|
|
|
calls: list[list[str]] = []
|
|
|
|
def fake_run_cmd(
|
|
cmd: list[str], *, cwd: Path, env: dict[str, str]
|
|
) -> tuple[int, str, str]:
|
|
calls.append(cmd)
|
|
if cmd[-2:] == ["pull", "--help"]:
|
|
return 0, "Usage:\n", ""
|
|
return 0, "", ""
|
|
|
|
with (
|
|
patch.object(sys, "argv", argv),
|
|
patch.object(self.m, "has_buildable_services", return_value=False),
|
|
patch.object(self.m, "run_cmd", side_effect=fake_run_cmd),
|
|
patch.object(self.m, "run_streamed", side_effect=fake_run_cmd),
|
|
patch.object(self.m, "base_compose_cmd", return_value=base_cmd),
|
|
):
|
|
rc = self.m.main()
|
|
|
|
self.assertEqual(rc, 0)
|
|
self.assertEqual(calls[0], [*base_cmd, "pull", "--help"])
|
|
self.assertEqual(calls[1], [*base_cmd, "config"])
|
|
self.assertEqual(calls[2], [*base_cmd, "pull"])
|
|
|
|
def test_main_tolerates_pull_failure_when_images_local(self) -> None:
|
|
with tempfile.TemporaryDirectory() as td:
|
|
chdir = Path(td) / "instance"
|
|
chdir.mkdir(parents=True, exist_ok=True)
|
|
|
|
lock_dir = Path(td) / "locks"
|
|
lock_key = "ktol"
|
|
|
|
argv = [
|
|
"pull.py",
|
|
"--chdir",
|
|
str(chdir),
|
|
"--project",
|
|
"p",
|
|
"--compose-files",
|
|
"-f a.yml",
|
|
"--lock-dir",
|
|
str(lock_dir),
|
|
"--lock-key",
|
|
lock_key,
|
|
]
|
|
|
|
base_cmd = ["docker", "compose", "-p", "p", "-f", "a.yml"]
|
|
|
|
def fake_run_cmd(
|
|
cmd: list[str], *, cwd: Path, env: dict[str, str]
|
|
) -> tuple[int, str, str]:
|
|
if cmd[-1] == "pull":
|
|
return 1, "", "pull failed\n"
|
|
if cmd[-2:] == ["config", "--images"]:
|
|
return 0, "image:1\nimage:2\n", ""
|
|
if cmd[:3] == ["docker", "image", "inspect"]:
|
|
return 0, "", ""
|
|
return 0, "", ""
|
|
|
|
with (
|
|
patch.object(sys, "argv", argv),
|
|
patch.object(self.m, "has_buildable_services", return_value=False),
|
|
patch.object(self.m, "run_cmd", side_effect=fake_run_cmd),
|
|
patch.object(self.m, "run_streamed", side_effect=fake_run_cmd),
|
|
patch.object(self.m, "base_compose_cmd", return_value=base_cmd),
|
|
):
|
|
rc = self.m.main()
|
|
|
|
self.assertEqual(rc, 0)
|
|
self.assertTrue(
|
|
(lock_dir / f"{lock_key}.lock").exists(),
|
|
"lock should still be written when pull fails but images are local",
|
|
)
|
|
|
|
def test_main_raises_when_pull_fails_and_images_missing(self) -> None:
|
|
with tempfile.TemporaryDirectory() as td:
|
|
chdir = Path(td) / "instance"
|
|
chdir.mkdir(parents=True, exist_ok=True)
|
|
|
|
lock_dir = Path(td) / "locks"
|
|
lock_key = "kmiss"
|
|
|
|
argv = [
|
|
"pull.py",
|
|
"--chdir",
|
|
str(chdir),
|
|
"--project",
|
|
"p",
|
|
"--compose-files",
|
|
"-f a.yml",
|
|
"--lock-dir",
|
|
str(lock_dir),
|
|
"--lock-key",
|
|
lock_key,
|
|
]
|
|
|
|
base_cmd = ["docker", "compose", "-p", "p", "-f", "a.yml"]
|
|
|
|
def fake_run_cmd(
|
|
cmd: list[str], *, cwd: Path, env: dict[str, str]
|
|
) -> tuple[int, str, str]:
|
|
if cmd[-1] == "pull":
|
|
return 1, "", "pull failed\n"
|
|
if cmd[-2:] == ["config", "--images"]:
|
|
return 0, "image:1\n", ""
|
|
if cmd[:3] == ["docker", "image", "inspect"]:
|
|
return 1, "", "no such image\n"
|
|
return 0, "", ""
|
|
|
|
with (
|
|
patch.object(sys, "argv", argv),
|
|
patch.object(self.m, "has_buildable_services", return_value=False),
|
|
patch.object(self.m, "run_cmd", side_effect=fake_run_cmd),
|
|
patch.object(self.m, "base_compose_cmd", return_value=base_cmd),
|
|
self.assertRaises(RuntimeError) as ctx,
|
|
):
|
|
self.m.main()
|
|
self.assertIn("missing locally", str(ctx.exception))
|
|
|
|
|
|
class TestBuildHandlersSharePullCache(unittest.TestCase):
|
|
def test_build_handlers_build_with_pull_and_fall_back_like_pull_py(self) -> None:
|
|
handlers = {
|
|
handler["name"]: handler.get("ansible.builtin.shell", "")
|
|
for handler in load_yaml_any(
|
|
str(PROJECT_ROOT / "roles/sys-svc-compose/handlers/compose.yml")
|
|
)
|
|
}
|
|
for name in (
|
|
"🔨 Build compose",
|
|
"🔨 Build the services queued for a lone recreate",
|
|
):
|
|
with self.subTest(handler=name):
|
|
self.assertRegex(
|
|
handlers[name], r"compose build --pull\b.*\|\| compose build\b"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|