mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 19:01:54 +00:00
fix(ci): put containerd, Docker and swap on the runner's empty spare disk
The small GitHub runner shape (AWS, 72G root) ran out of disk in swarm and compose rows (#1329: WordPress#2 of run 41261531965, Nextcloud#6 and #7 of run 41262286962) while an empty instance-store disk nvme1n1 sat unused. mount_spare_disk.sh formats and mounts it at /mnt, but only on a GitHub-hosted runner outside act, and only a whole unpartitioned disk without any signature, mount or swap use of at least 75G; every decline leaves a notice. relocate_docker_dataroot.sh then bind-mounts /var/lib/containerd from the spare disk and moves the Docker data-root to /mnt/docker-host (not /mnt/docker, which is the nested daemon's volume that dedicated.sh wipes). It refuses before stopping anything when images, volumes, build cache, foreign containerd namespaces or shims exist or daemon.json is unreadable, and rolls back the bind mount and daemon.json on any later failure or cancel. Swap goes on / once containerd lives on /mnt, dump_state.sh logs disk size, model and free space per storage root, and the swarm watchdog samples / and /var/lib/containerd every 10 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
kevinveenbirkenbachandClaude Opus 5.5 committed
1 parent
076eea636b
commit
2d7bf740b0
13 files changed
+1585
-89
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,135 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from utils.cache.files import PROJECT_ROOT
|
||||
|
||||
SCRIPT = PROJECT_ROOT / "scripts" / "github" / "runner" / "enlarge_swap.sh"
|
||||
SUDO_STUB = '#!/usr/bin/env bash\nprintf "%s\\n" "$*" >>"${STUB_LOG}"\n'
|
||||
DF_STUB = (
|
||||
"#!/usr/bin/env bash\n"
|
||||
'echo "Avail"\n'
|
||||
'if [ "${*: -1}" = /mnt ]; then echo "${STUB_MNT_FREE}"; else echo "${STUB_ROOT_FREE}"; fi\n'
|
||||
)
|
||||
MOUNTPOINT_STUB = '#!/usr/bin/env bash\n[ "${STUB_MNT_MOUNTED}" = 1 ]\n'
|
||||
STAT_STUB = (
|
||||
"#!/usr/bin/env bash\n"
|
||||
'case "${*: -1}" in\n'
|
||||
"/mnt) echo 2049 ;;\n"
|
||||
'/var/lib/containerd) if [ "${STUB_CONTAINERD_DEV}" = missing ]; then exit 1; fi\n'
|
||||
' echo "${STUB_CONTAINERD_DEV}" ;;\n'
|
||||
"*) exit 1 ;;\n"
|
||||
"esac\n"
|
||||
)
|
||||
DUMP_STUB = "#!/usr/bin/env bash\nexit 0\n"
|
||||
|
||||
|
||||
def enlarge(
|
||||
*, mounted: bool, mnt_free: int, root_free: int, containerd_dev: str
|
||||
) -> tuple[subprocess.CompletedProcess, list[str]]:
|
||||
"""Run enlarge_swap.sh against stubbed disks.
|
||||
|
||||
Args:
|
||||
mounted: whether /mnt is a separate mount.
|
||||
mnt_free: GB df reports free on /mnt.
|
||||
root_free: GB df reports free on /.
|
||||
containerd_dev: device id stat reports for /var/lib/containerd
|
||||
(2049 is the /mnt device).
|
||||
|
||||
Returns:
|
||||
The finished process and every sudo call in order.
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
runner_dir = root / "runner"
|
||||
runner_dir.mkdir()
|
||||
(runner_dir / "enlarge_swap.sh").symlink_to(SCRIPT)
|
||||
bin_dir = root / "bin"
|
||||
bin_dir.mkdir()
|
||||
for path, body in (
|
||||
(runner_dir / "dump_state.sh", DUMP_STUB),
|
||||
(bin_dir / "sudo", SUDO_STUB),
|
||||
(bin_dir / "df", DF_STUB),
|
||||
(bin_dir / "mountpoint", MOUNTPOINT_STUB),
|
||||
(bin_dir / "stat", STAT_STUB),
|
||||
):
|
||||
path.write_text(body)
|
||||
path.chmod(0o755)
|
||||
log = root / "sudo.log"
|
||||
log.touch()
|
||||
env = {
|
||||
k: v for k, v in os.environ.items() if k not in ("BASH_ENV", "SKIP_SWAP")
|
||||
}
|
||||
env.update(
|
||||
PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
|
||||
STUB_LOG=str(log),
|
||||
STUB_MNT_MOUNTED="1" if mounted else "0",
|
||||
STUB_MNT_FREE=str(mnt_free),
|
||||
STUB_ROOT_FREE=str(root_free),
|
||||
STUB_CONTAINERD_DEV=containerd_dev,
|
||||
)
|
||||
done = subprocess.run(
|
||||
["bash", str(runner_dir / "enlarge_swap.sh")],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return done, log.read_text().splitlines() # nocheck: cache-read stub call log
|
||||
|
||||
|
||||
def swapfile(calls: list[str]) -> list[str]:
|
||||
"""Return the fallocate calls among the recorded sudo calls.
|
||||
|
||||
Args:
|
||||
calls: every recorded sudo call.
|
||||
|
||||
Returns:
|
||||
The calls that create the swapfile.
|
||||
"""
|
||||
return [call for call in calls if call.startswith("fallocate ")]
|
||||
|
||||
|
||||
class TestEnlargeSwapPlacement(unittest.TestCase):
|
||||
def test_swap_goes_to_root_when_containerd_lives_on_the_spare_disk(self) -> None:
|
||||
done, calls = enlarge(
|
||||
mounted=True, mnt_free=200, root_free=54, containerd_dev="2049"
|
||||
)
|
||||
self.assertEqual(done.returncode, 0, done.stderr)
|
||||
self.assertEqual(swapfile(calls), ["fallocate -l 16G /swapfile"])
|
||||
|
||||
def test_swap_goes_to_mnt_when_containerd_stayed_on_root(self) -> None:
|
||||
done, calls = enlarge(
|
||||
mounted=True, mnt_free=60, root_free=54, containerd_dev="66305"
|
||||
)
|
||||
self.assertEqual(done.returncode, 0, done.stderr)
|
||||
self.assertEqual(swapfile(calls), ["fallocate -l 16G /mnt/swapfile"])
|
||||
|
||||
def test_large_shape_without_mnt_keeps_swap_on_root(self) -> None:
|
||||
done, calls = enlarge(
|
||||
mounted=False, mnt_free=0, root_free=123, containerd_dev="66305"
|
||||
)
|
||||
self.assertEqual(done.returncode, 0, done.stderr)
|
||||
self.assertEqual(swapfile(calls), ["fallocate -l 16G /swapfile"])
|
||||
|
||||
def test_missing_containerd_root_keeps_the_mnt_preference(self) -> None:
|
||||
done, calls = enlarge(
|
||||
mounted=True, mnt_free=60, root_free=54, containerd_dev="missing"
|
||||
)
|
||||
self.assertEqual(done.returncode, 0, done.stderr)
|
||||
self.assertEqual(swapfile(calls), ["fallocate -l 16G /mnt/swapfile"])
|
||||
|
||||
def test_full_root_still_uses_the_spare_disk(self) -> None:
|
||||
done, calls = enlarge(
|
||||
mounted=True, mnt_free=200, root_free=10, containerd_dev="2049"
|
||||
)
|
||||
self.assertEqual(done.returncode, 0, done.stderr)
|
||||
self.assertEqual(swapfile(calls), ["fallocate -l 16G /mnt/swapfile"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,457 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from utils.cache.files import PROJECT_ROOT
|
||||
from utils.cache.yaml import load_yaml
|
||||
|
||||
SCRIPT = PROJECT_ROOT / "scripts" / "github" / "runner" / "mount_spare_disk.sh"
|
||||
WORKFLOW = PROJECT_ROOT / ".github" / "workflows" / "call-test-deploy.yml"
|
||||
MIN_SIZE_GB = "75"
|
||||
GIB = 1024**3
|
||||
SPARE = "nvme9n9"
|
||||
SPARE_INDEX = 2
|
||||
DESTRUCTIVE = ("mkfs.ext4", "blkid", "wipefs", "mount", "mkswap")
|
||||
|
||||
SUDO_STUB = (
|
||||
"#!/usr/bin/env bash\n"
|
||||
'printf "%s\\n" "$*" >>"${STUB_LOG}"\n'
|
||||
'if [ -x "${STUB_BIN}/$1" ]; then exec "${STUB_BIN}/$1" "${@:2}"; fi\n'
|
||||
'echo "unstubbed sudo $1" >&2\n'
|
||||
"exit 97\n"
|
||||
)
|
||||
UNPRIVILEGED_STUB = (
|
||||
'#!/usr/bin/env bash\necho "$(basename "$0") ran without sudo" >&2\nexit 98\n'
|
||||
)
|
||||
LSBLK_STUB = '#!/usr/bin/env bash\ncat "${STUB_LSBLK_JSON}"\n'
|
||||
MOUNTPOINT_STUB = '#!/usr/bin/env bash\n[ "${STUB_MOUNTED}" = 1 ]\n'
|
||||
BLKID_STUB = (
|
||||
'#!/usr/bin/env bash\nprintf "%s" "${STUB_BLKID_OUT}"\nexit "${STUB_BLKID_RC}"\n'
|
||||
)
|
||||
WIPEFS_STUB = '#!/usr/bin/env bash\nprintf "%s" "${STUB_WIPEFS_OUT}"\n'
|
||||
MKFS_STUB = "#!/usr/bin/env bash\nexit 0\n"
|
||||
MOUNT_STUB = (
|
||||
'#!/usr/bin/env bash\nprintf "%s" "${STUB_MOUNT_AS:-$1}" >"${STUB_STATE}"\n'
|
||||
)
|
||||
FINDMNT_STUB = '#!/usr/bin/env bash\ncat "${STUB_STATE}"\n'
|
||||
DUMP_STUB = "#!/usr/bin/env bash\nexit 0\n"
|
||||
|
||||
|
||||
def part(name: str, size_gib: float, fstype: str | None, mount: str | None) -> dict:
|
||||
"""Build one lsblk partition entry.
|
||||
|
||||
Args:
|
||||
name: kernel name of the partition.
|
||||
size_gib: size in GiB.
|
||||
fstype: filesystem type, None when unformatted.
|
||||
mount: mount point, None when unmounted.
|
||||
|
||||
Returns:
|
||||
The entry as lsblk -J -b prints it.
|
||||
"""
|
||||
return {
|
||||
"name": name,
|
||||
"type": "part",
|
||||
"size": int(size_gib * GIB),
|
||||
"ro": False,
|
||||
"fstype": fstype,
|
||||
"pttype": "gpt",
|
||||
"mountpoints": [mount],
|
||||
}
|
||||
|
||||
|
||||
def snap_loops() -> list[dict]:
|
||||
"""Return the two snap squashfs loop devices every runner shape carries.
|
||||
|
||||
Returns:
|
||||
The loop entries as lsblk -J -b prints them.
|
||||
"""
|
||||
return [
|
||||
{
|
||||
"name": f"loop{index}",
|
||||
"type": "loop",
|
||||
"size": 77594624,
|
||||
"ro": True,
|
||||
"fstype": "squashfs",
|
||||
"pttype": None,
|
||||
"mountpoints": [mount],
|
||||
}
|
||||
for index, mount in enumerate(("/snap/core22/2955", "/snap/snapd/28254"))
|
||||
]
|
||||
|
||||
|
||||
def root_disk(name: str, prefix: str, size_gib: int) -> dict:
|
||||
"""Build the partitioned root disk of a GitHub runner image.
|
||||
|
||||
Args:
|
||||
name: kernel name of the disk.
|
||||
prefix: partition name prefix (``sda`` or ``nvme0n1p``).
|
||||
size_gib: size in GiB.
|
||||
|
||||
Returns:
|
||||
The disk entry with its four partitions.
|
||||
"""
|
||||
return {
|
||||
"name": name,
|
||||
"type": "disk",
|
||||
"size": size_gib * GIB,
|
||||
"ro": False,
|
||||
"fstype": None,
|
||||
"pttype": "gpt",
|
||||
"mountpoints": [None],
|
||||
"children": [
|
||||
part(f"{prefix}1", size_gib - 1.2, "ext4", "/"),
|
||||
part(f"{prefix}13", 1, "ext4", "/boot"),
|
||||
part(f"{prefix}14", 0.004, None, None),
|
||||
part(f"{prefix}15", 0.1, "vfat", "/boot/efi"),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def spare_disk(size_gib: int) -> dict:
|
||||
"""Build the empty second NVMe disk of the small runner shape.
|
||||
|
||||
Args:
|
||||
size_gib: size in GiB.
|
||||
|
||||
Returns:
|
||||
The disk entry without filesystem, partition table or mount.
|
||||
"""
|
||||
return {
|
||||
"name": SPARE,
|
||||
"type": "disk",
|
||||
"size": size_gib * GIB,
|
||||
"ro": False,
|
||||
"fstype": None,
|
||||
"pttype": None,
|
||||
"mountpoints": [None],
|
||||
}
|
||||
|
||||
|
||||
LARGE_SHAPE = {"blockdevices": [*snap_loops(), root_disk("sda", "sda", 145)]}
|
||||
SMALL_SHAPE = {
|
||||
"blockdevices": [
|
||||
*snap_loops(),
|
||||
spare_disk(128),
|
||||
root_disk("nvme0n1", "nvme0n1p", 72),
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def with_spare(**changes: object) -> dict:
|
||||
"""Return the small shape with fields of the spare disk replaced.
|
||||
|
||||
Args:
|
||||
**changes: lsblk fields to overwrite on the spare disk.
|
||||
|
||||
Returns:
|
||||
A modified deep copy of SMALL_SHAPE.
|
||||
"""
|
||||
tree = copy.deepcopy(SMALL_SHAPE)
|
||||
tree["blockdevices"][SPARE_INDEX].update(changes)
|
||||
return tree
|
||||
|
||||
|
||||
def run(
|
||||
tree: dict,
|
||||
*,
|
||||
runner_environment: str | None = "github-hosted",
|
||||
act: str | None = None,
|
||||
mounted: bool = False,
|
||||
blkid_rc: int = 2,
|
||||
blkid_out: str = "",
|
||||
wipefs_out: str = "",
|
||||
mount_as: str | None = None,
|
||||
occupied: bool = False,
|
||||
min_size_gb: str = MIN_SIZE_GB,
|
||||
) -> tuple[subprocess.CompletedProcess, list[str]]:
|
||||
"""Run mount_spare_disk.sh against a faked block-device layout.
|
||||
|
||||
Args:
|
||||
tree: what ``lsblk -J -b`` reports.
|
||||
runner_environment: RUNNER_ENVIRONMENT, None to leave it unset.
|
||||
act: ACT, None to leave it unset.
|
||||
mounted: whether the mount point already is a mount.
|
||||
blkid_rc: exit code of ``blkid -p`` (2 means no signature).
|
||||
blkid_out: what ``blkid -p`` prints.
|
||||
wipefs_out: what ``wipefs --no-act`` prints.
|
||||
mount_as: device findmnt reports after the mount, None for the mounted one.
|
||||
occupied: whether the mount point already holds a file.
|
||||
min_size_gb: the MIN_SIZE_GB argument.
|
||||
|
||||
Returns:
|
||||
The finished process and every privileged call in order.
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
runner_dir = root / "runner"
|
||||
runner_dir.mkdir()
|
||||
(runner_dir / "mount_spare_disk.sh").symlink_to(SCRIPT)
|
||||
bin_dir = root / "bin"
|
||||
bin_dir.mkdir()
|
||||
stub_bin = root / "privileged"
|
||||
stub_bin.mkdir()
|
||||
stubs = [
|
||||
(runner_dir / "dump_state.sh", DUMP_STUB),
|
||||
(bin_dir / "sudo", SUDO_STUB),
|
||||
(bin_dir / "lsblk", LSBLK_STUB),
|
||||
(bin_dir / "mountpoint", MOUNTPOINT_STUB),
|
||||
(bin_dir / "findmnt", FINDMNT_STUB),
|
||||
(stub_bin / "blkid", BLKID_STUB),
|
||||
(stub_bin / "wipefs", WIPEFS_STUB),
|
||||
(stub_bin / "mkfs.ext4", MKFS_STUB),
|
||||
(stub_bin / "mount", MOUNT_STUB),
|
||||
]
|
||||
stubs += [(bin_dir / name, UNPRIVILEGED_STUB) for name in DESTRUCTIVE]
|
||||
for path, body in stubs:
|
||||
path.write_text(body)
|
||||
path.chmod(0o755)
|
||||
mount_point = root / "mnt"
|
||||
mount_point.mkdir()
|
||||
if occupied:
|
||||
(mount_point / "keep.txt").write_text("data\n")
|
||||
lsblk_json = root / "lsblk.json"
|
||||
lsblk_json.write_text(json.dumps(tree))
|
||||
log = root / "sudo.log"
|
||||
log.touch()
|
||||
env = {
|
||||
k: v
|
||||
for k, v in os.environ.items()
|
||||
if k not in ("BASH_ENV", "ACT", "RUNNER_ENVIRONMENT")
|
||||
}
|
||||
env.update(
|
||||
PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
|
||||
STUB_LOG=str(log),
|
||||
STUB_BIN=str(stub_bin),
|
||||
STUB_LSBLK_JSON=str(lsblk_json),
|
||||
STUB_MOUNTED="1" if mounted else "0",
|
||||
STUB_BLKID_RC=str(blkid_rc),
|
||||
STUB_BLKID_OUT=blkid_out,
|
||||
STUB_WIPEFS_OUT=wipefs_out,
|
||||
STUB_STATE=str(root / "mounted-source"),
|
||||
)
|
||||
if mount_as is not None:
|
||||
env["STUB_MOUNT_AS"] = mount_as
|
||||
if runner_environment is not None:
|
||||
env["RUNNER_ENVIRONMENT"] = runner_environment
|
||||
if act is not None:
|
||||
env["ACT"] = act
|
||||
done = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(runner_dir / "mount_spare_disk.sh"),
|
||||
str(mount_point),
|
||||
min_size_gb,
|
||||
],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return done, log.read_text().splitlines() # nocheck: cache-read stub call log
|
||||
|
||||
|
||||
def formatted(calls: list[str]) -> bool:
|
||||
"""Tell whether any privileged call formatted or mounted a disk.
|
||||
|
||||
Args:
|
||||
calls: every recorded sudo call.
|
||||
|
||||
Returns:
|
||||
True when mkfs or mount ran.
|
||||
"""
|
||||
return any(call.startswith(("mkfs", "mount ")) for call in calls)
|
||||
|
||||
|
||||
class TestMountSpareDisk(unittest.TestCase):
|
||||
def assert_untouched(self, done: subprocess.CompletedProcess, reason: str) -> None:
|
||||
"""Assert a declined run: exit 0, no privileged call, the stated reason.
|
||||
|
||||
Args:
|
||||
done: the finished script.
|
||||
reason: text the script must print for this decline.
|
||||
"""
|
||||
self.assertEqual(done.returncode, 0, done.stderr)
|
||||
self.assertIn(reason, done.stdout + done.stderr)
|
||||
self.assertRegex(done.stdout, r"::(notice|warning)::")
|
||||
|
||||
def test_small_shape_formats_and_mounts_the_empty_nvme_disk(self) -> None:
|
||||
done, calls = run(SMALL_SHAPE)
|
||||
self.assertEqual(done.returncode, 0, done.stderr)
|
||||
self.assertEqual(
|
||||
calls[:3],
|
||||
[
|
||||
f"blkid -p /dev/{SPARE}",
|
||||
f"wipefs --no-act --noheadings /dev/{SPARE}",
|
||||
f"mkfs.ext4 -q -m 0 -L infinito-spare /dev/{SPARE}",
|
||||
],
|
||||
)
|
||||
self.assertTrue(calls[3].startswith(f"mount /dev/{SPARE} "), calls)
|
||||
self.assertEqual(len(calls), 4, calls)
|
||||
|
||||
def test_large_shape_without_a_spare_disk_stays_as_it_is(self) -> None:
|
||||
done, calls = run(LARGE_SHAPE)
|
||||
self.assert_untouched(done, '["sda"]')
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_self_hosted_runner_is_never_formatted(self) -> None:
|
||||
done, calls = run(SMALL_SHAPE, runner_environment="self-hosted")
|
||||
self.assert_untouched(done, "not a GitHub-hosted runner")
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_unknown_runner_environment_is_never_formatted(self) -> None:
|
||||
done, calls = run(SMALL_SHAPE, runner_environment=None)
|
||||
self.assert_untouched(done, "not a GitHub-hosted runner")
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_act_refuses_loudly_even_with_a_faked_github_runner(self) -> None:
|
||||
for act in ("true", "1", "TRUE"):
|
||||
with self.subTest(act=act):
|
||||
done, calls = run(SMALL_SHAPE, act=act)
|
||||
self.assertEqual(done.returncode, 1)
|
||||
self.assertEqual(calls, [])
|
||||
self.assertIn("refuses to run under act", done.stderr)
|
||||
|
||||
def test_disk_with_a_filesystem_is_never_formatted(self) -> None:
|
||||
done, calls = run(with_spare(fstype="ext4"))
|
||||
self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_disk_with_partitions_is_never_formatted(self) -> None:
|
||||
done, calls = run(with_spare(children=[part(f"{SPARE}p1", 127, None, None)]))
|
||||
self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_disk_held_by_lvm_is_never_formatted(self) -> None:
|
||||
holder = {
|
||||
"name": "vg-data",
|
||||
"type": "lvm",
|
||||
"size": 127 * GIB,
|
||||
"ro": False,
|
||||
"fstype": None,
|
||||
"pttype": None,
|
||||
"mountpoints": [None],
|
||||
}
|
||||
done, calls = run(with_spare(children=[holder]))
|
||||
self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_disk_with_a_bare_partition_table_is_never_formatted(self) -> None:
|
||||
done, calls = run(with_spare(pttype="dos"))
|
||||
self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_mounted_disk_is_never_formatted(self) -> None:
|
||||
done, calls = run(with_spare(mountpoints=["/data"]))
|
||||
self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_disk_used_as_swap_is_never_formatted(self) -> None:
|
||||
done, calls = run(with_spare(mountpoints=["[SWAP]"]))
|
||||
self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_read_only_disk_is_never_formatted(self) -> None:
|
||||
done, calls = run(with_spare(ro=True))
|
||||
self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_disk_below_the_size_floor_stays_as_it_is(self) -> None:
|
||||
done, calls = run(with_spare(size=32 * GIB))
|
||||
self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_signature_lsblk_missed_is_never_formatted(self) -> None:
|
||||
done, calls = run(SMALL_SHAPE, blkid_rc=0, blkid_out='TYPE="xfs"')
|
||||
self.assert_untouched(done, "carries a signature")
|
||||
self.assertFalse(formatted(calls), calls)
|
||||
|
||||
def test_signature_only_wipefs_sees_is_never_formatted(self) -> None:
|
||||
done, calls = run(SMALL_SHAPE, wipefs_out="0x218 LVM2_member")
|
||||
self.assert_untouched(done, "carries signatures blkid missed")
|
||||
self.assertFalse(formatted(calls), calls)
|
||||
|
||||
def test_inconclusive_probe_fails_loudly_without_formatting(self) -> None:
|
||||
done, calls = run(SMALL_SHAPE, blkid_rc=8, blkid_out="ambivalent result")
|
||||
self.assertEqual(done.returncode, 1)
|
||||
self.assertFalse(formatted(calls), calls)
|
||||
|
||||
def test_probe_that_found_nothing_but_complained_fails_loudly(self) -> None:
|
||||
done, calls = run(SMALL_SHAPE, blkid_rc=2, blkid_out="Input/output error")
|
||||
self.assertEqual(done.returncode, 1)
|
||||
self.assertFalse(formatted(calls), calls)
|
||||
|
||||
def test_lsblk_without_mountpoints_fails_loudly(self) -> None:
|
||||
tree = copy.deepcopy(SMALL_SHAPE)
|
||||
spare = tree["blockdevices"][SPARE_INDEX]
|
||||
spare["mountpoint"] = spare.pop("mountpoints")[0]
|
||||
done, calls = run(tree)
|
||||
self.assertEqual(done.returncode, 1)
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_layout_without_a_root_device_fails_loudly(self) -> None:
|
||||
done, calls = run({"blockdevices": [spare_disk(128)]})
|
||||
self.assertEqual(done.returncode, 1)
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_non_numeric_size_floor_fails_loudly(self) -> None:
|
||||
done, calls = run(SMALL_SHAPE, min_size_gb="RANDOM")
|
||||
self.assertEqual(done.returncode, 1)
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_existing_mount_stays_as_it_is(self) -> None:
|
||||
done, calls = run(SMALL_SHAPE, mounted=True)
|
||||
self.assert_untouched(done, "already a separate mount")
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_occupied_mount_point_stays_as_it_is(self) -> None:
|
||||
done, calls = run(SMALL_SHAPE, occupied=True)
|
||||
self.assert_untouched(done, "is not empty")
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_mount_landing_on_another_device_fails_loudly(self) -> None:
|
||||
done, _ = run(SMALL_SHAPE, mount_as="/dev/nvme0n1p1")
|
||||
self.assertEqual(done.returncode, 1)
|
||||
self.assertIn("backed by '/dev/nvme0n1p1'", done.stderr)
|
||||
|
||||
|
||||
class TestWorkflowWiring(unittest.TestCase):
|
||||
def test_spare_disk_step_is_act_gated_and_runs_before_swap_and_relocation(
|
||||
self,
|
||||
) -> None:
|
||||
workflow = load_yaml(str(WORKFLOW))
|
||||
steps = next(
|
||||
job["steps"]
|
||||
for job in workflow["jobs"].values()
|
||||
if any(
|
||||
"mount_spare_disk.sh" in str(step.get("run", ""))
|
||||
for step in job.get("steps", [])
|
||||
)
|
||||
)
|
||||
runs = [str(step.get("run", "")) for step in steps]
|
||||
index = next(i for i, cmd in enumerate(runs) if "mount_spare_disk.sh" in cmd)
|
||||
self.assertEqual(
|
||||
runs[index],
|
||||
f"./scripts/github/runner/mount_spare_disk.sh /mnt {MIN_SIZE_GB}",
|
||||
)
|
||||
self.assertEqual(steps[index].get("if"), "${{ !env.ACT }}")
|
||||
swap = next(i for i, cmd in enumerate(runs) if "enlarge_swap.sh" in cmd)
|
||||
relocate = next(
|
||||
i for i, cmd in enumerate(runs) if "relocate_docker_dataroot.sh" in cmd
|
||||
)
|
||||
self.assertLess(index, relocate)
|
||||
self.assertLess(relocate, swap)
|
||||
self.assertEqual(
|
||||
runs[relocate], "./scripts/github/runner/relocate_docker_dataroot.sh 75"
|
||||
)
|
||||
self.assertEqual(steps[relocate].get("if"), "${{ !env.ACT }}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,524 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from utils.cache.files import PROJECT_ROOT
|
||||
|
||||
SCRIPT = PROJECT_ROOT / "scripts" / "github" / "runner" / "relocate_docker_dataroot.sh"
|
||||
MIN_FREE_GB = "75"
|
||||
DUMP_V2 = "version = 3\nroot = '/var/lib/containerd'\nstate = '/run/containerd'\n"
|
||||
DUMP_V1 = 'version = 2\nroot = "/var/lib/containerd"\nstate = "/run/containerd"\n'
|
||||
STOP = "systemctl stop docker.socket docker.service containerd.service"
|
||||
BIND = "mount --bind /mnt/containerd /var/lib/containerd"
|
||||
UNBIND = "umount /var/lib/containerd"
|
||||
RESTART = "systemctl start containerd.service docker.socket docker.service"
|
||||
|
||||
SUDO_STUB = (
|
||||
"#!/usr/bin/env bash\n"
|
||||
'printf "%s\\n" "$*" >>"${STUB_LOG}"\n'
|
||||
'if [ -x "${STUB_BIN}/$1" ]; then exec "${STUB_BIN}/$1" "${@:2}"; fi\n'
|
||||
'echo "unstubbed sudo $1" >&2\n'
|
||||
"exit 97\n"
|
||||
)
|
||||
UNPRIVILEGED_STUB = (
|
||||
'#!/usr/bin/env bash\necho "$(basename "$0") ran without sudo" >&2\nexit 98\n'
|
||||
)
|
||||
TERM_HOOK = (
|
||||
'if [ "$(basename "$0")" = "${STUB_TERM_ON}" ]; then kill -TERM "${PPID}"; fi\n'
|
||||
)
|
||||
UNPRIVILEGED = {
|
||||
"mountpoint": (
|
||||
"#!/usr/bin/env bash\n"
|
||||
'case "${*: -1}" in\n'
|
||||
'/mnt) [ "${STUB_MNT_MOUNTED}" = 1 ] ;;\n'
|
||||
'/var/lib/containerd) [ -e "${STUB_STATE}/bound" ] ;;\n'
|
||||
"*) exit 1 ;;\n"
|
||||
"esac\n"
|
||||
),
|
||||
"df": (
|
||||
"#!/usr/bin/env bash\n"
|
||||
'echo "Avail"\n'
|
||||
'if [ "${*: -1}" = /mnt ]; then echo "${STUB_MNT_FREE}"; '
|
||||
'else echo "${STUB_ROOT_FREE}"; fi\n'
|
||||
),
|
||||
"systemctl": '#!/usr/bin/env bash\nexit "${STUB_CONTAINERD_UNIT_RC}"\n',
|
||||
"docker": (
|
||||
"#!/usr/bin/env bash\n"
|
||||
'case "$1" in\n'
|
||||
'ps) printf "%s" "${STUB_CONTAINERS}" ;;\n'
|
||||
'images) printf "%s" "${STUB_IMAGES}" ;;\n'
|
||||
'volume) printf "%s" "${STUB_VOLUMES}" ;;\n'
|
||||
'system) printf "Images=0\\nContainers=0\\nLocal Volumes=0\\n'
|
||||
'Build Cache=%s\\n" "${STUB_BUILD_CACHE}" ;;\n'
|
||||
'info) echo "${STUB_DOCKER_ROOT}" ;;\n'
|
||||
"*) exit 96 ;;\n"
|
||||
"esac\n"
|
||||
),
|
||||
"stat": (
|
||||
"#!/usr/bin/env bash\n"
|
||||
'case "${*: -1}" in\n'
|
||||
"/mnt) echo 2049 ;;\n"
|
||||
'/mnt/docker-host) echo "${STUB_DOCKER_ROOT_DEV}" ;;\n'
|
||||
"*) echo 66305 ;;\n"
|
||||
"esac\n"
|
||||
),
|
||||
"pgrep": '#!/usr/bin/env bash\nprintf "%s" "${STUB_SHIMS}"\nexit "${STUB_SHIM_RC}"\n',
|
||||
}
|
||||
PRIVILEGED = {
|
||||
"systemctl": (
|
||||
"#!/usr/bin/env bash\n"
|
||||
'if [ "$1" = status ]; then exit "${STUB_STATUS_RC}"; fi\n'
|
||||
'if [ "$*" = "${STUB_FAIL_SYSTEMCTL}" ]; then exit 1; fi\n'
|
||||
),
|
||||
"containerd": '#!/usr/bin/env bash\nprintf "%s" "${STUB_CONTAINERD_DUMP}"\n',
|
||||
"ctr": (
|
||||
"#!/usr/bin/env bash\n"
|
||||
'if [ "$1" = namespaces ]; then printf "%s" "${STUB_NAMESPACES}"; exit; fi\n'
|
||||
'if [ "$1" = -n ] && [ "$2" = "${STUB_FOREIGN_NS}" ]; then\n'
|
||||
' if [ "$3" = containers ]; then printf "%s" "${STUB_FOREIGN_CONTAINERS}"; fi\n'
|
||||
' if [ "$3" = tasks ]; then printf "%s" "${STUB_FOREIGN_TASKS}"; fi\n'
|
||||
"fi\n"
|
||||
),
|
||||
"du": '#!/usr/bin/env bash\nprintf "1.2G\\t%s\\n" "${*: -1}"\n',
|
||||
"true": '#!/usr/bin/env bash\nexit "${STUB_SUDO_RC}"\n',
|
||||
"test": (
|
||||
"#!/usr/bin/env bash\n"
|
||||
'if [ "$1" = -e ]; then [ "${STUB_DAEMON_EXISTS}" = 1 ]; exit; fi\n'
|
||||
'[ "${STUB_METADATA}" = 1 ]\n'
|
||||
),
|
||||
"cat": (
|
||||
"#!/usr/bin/env bash\n"
|
||||
'if [ "${STUB_DAEMON_CAT_RC}" != 0 ]; then exit "${STUB_DAEMON_CAT_RC}"; fi\n'
|
||||
'printf "%s" "${STUB_DAEMON_IN}"\n'
|
||||
),
|
||||
"mkdir": "#!/usr/bin/env bash\n" + TERM_HOOK,
|
||||
"mount": (
|
||||
"#!/usr/bin/env bash\n"
|
||||
'if [ "${STUB_MOUNT_RC}" != 0 ]; then exit "${STUB_MOUNT_RC}"; fi\n'
|
||||
'touch "${STUB_STATE}/bound"\n'
|
||||
),
|
||||
"umount": '#!/usr/bin/env bash\nrm -f "${STUB_STATE}/bound"\n',
|
||||
"install": '#!/usr/bin/env bash\ncp "${*: -2:1}" "${STUB_DAEMON_OUT}"\n',
|
||||
"rm": '#!/usr/bin/env bash\n/bin/rm -f "${STUB_DAEMON_OUT}"\n',
|
||||
"journalctl": "#!/usr/bin/env bash\nexit 0\n",
|
||||
}
|
||||
DESTRUCTIVE = ("mount", "umount", "install", "ctr", "containerd", "journalctl", "mkdir")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Runner:
|
||||
"""The faked runner state relocate_docker_dataroot.sh meets.
|
||||
|
||||
Args:
|
||||
runner_environment: RUNNER_ENVIRONMENT, None to leave it unset.
|
||||
act: ACT, None to leave it unset.
|
||||
skip: SKIP_DOCKER_RELOCATE, None to leave it unset.
|
||||
mounted: whether /mnt is a separate mount.
|
||||
mnt_free: GB df reports free on /mnt.
|
||||
root_free: GB df reports free on /.
|
||||
dump: what ``containerd config dump`` prints.
|
||||
containerd_unit_rc: exit code of ``systemctl cat containerd.service``.
|
||||
containers: what ``docker ps -aq`` prints.
|
||||
images: what ``docker images -q`` prints.
|
||||
volumes: what ``docker volume ls -q`` prints.
|
||||
build_cache: Build Cache record count in ``docker system df``.
|
||||
namespaces: what ``ctr namespaces ls -q`` prints.
|
||||
foreign_ns: namespace whose containers and tasks are faked.
|
||||
foreign_containers: containers in foreign_ns.
|
||||
foreign_tasks: tasks in foreign_ns.
|
||||
shims: what ``pgrep -a containerd-shim`` prints.
|
||||
shim_rc: exit code of pgrep (1 means no shim).
|
||||
sudo_rc: exit code of ``sudo true``.
|
||||
daemon_exists: whether daemon.json exists.
|
||||
daemon_in: content of the existing daemon.json.
|
||||
daemon_cat_rc: exit code of ``sudo cat`` on daemon.json.
|
||||
mount_rc: exit code of the bind mount.
|
||||
fail_systemctl: the one systemctl argument string that fails.
|
||||
status_rc: exit code of ``systemctl status``.
|
||||
term_on: privileged stub that sends SIGTERM to the script.
|
||||
metadata: whether containerd's meta.db shows up under /mnt/containerd.
|
||||
docker_root: what ``docker info`` reports as DockerRootDir.
|
||||
docker_root_dev: device id stat reports for /mnt/docker-host.
|
||||
min_free_gb: the MIN_FREE_GB argument.
|
||||
"""
|
||||
|
||||
runner_environment: str | None = "github-hosted"
|
||||
act: str | None = None
|
||||
skip: str | None = None
|
||||
mounted: bool = True
|
||||
mnt_free: int = 137
|
||||
root_free: int = 51
|
||||
dump: str = DUMP_V2
|
||||
containerd_unit_rc: int = 0
|
||||
containers: str = ""
|
||||
images: str = ""
|
||||
volumes: str = ""
|
||||
build_cache: str = "0"
|
||||
namespaces: str = "moby\n"
|
||||
foreign_ns: str = "k8s.io"
|
||||
foreign_containers: str = ""
|
||||
foreign_tasks: str = ""
|
||||
shims: str = ""
|
||||
shim_rc: int = 1
|
||||
sudo_rc: int = 0
|
||||
daemon_exists: bool = False
|
||||
daemon_in: str = ""
|
||||
daemon_cat_rc: int = 0
|
||||
mount_rc: int = 0
|
||||
fail_systemctl: str = ""
|
||||
status_rc: int = 0
|
||||
term_on: str = ""
|
||||
metadata: bool = True
|
||||
docker_root: str = "/mnt/docker-host"
|
||||
docker_root_dev: str = "2049"
|
||||
min_free_gb: str = MIN_FREE_GB
|
||||
extra: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Outcome:
|
||||
"""What a relocation run left behind.
|
||||
|
||||
Args:
|
||||
done: the finished script.
|
||||
calls: every privileged call in order.
|
||||
daemon: the daemon.json on disk afterwards, None when there is none
|
||||
or it is no valid JSON.
|
||||
daemon_raw: the daemon.json bytes on disk afterwards, None when there is none.
|
||||
bound: whether the containerd bind mount is still in place.
|
||||
"""
|
||||
|
||||
done: subprocess.CompletedProcess
|
||||
calls: list[str]
|
||||
daemon: dict | None
|
||||
daemon_raw: str | None
|
||||
bound: bool
|
||||
|
||||
|
||||
def relocate(runner: Runner) -> Outcome:
|
||||
"""Run relocate_docker_dataroot.sh against a faked runner.
|
||||
|
||||
Args:
|
||||
runner: the faked services, disks and files.
|
||||
|
||||
Returns:
|
||||
The outcome of the run.
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
runner_dir = root / "runner"
|
||||
bin_dir = root / "bin"
|
||||
stub_bin = root / "privileged"
|
||||
state = root / "state"
|
||||
for directory in (runner_dir, bin_dir, stub_bin, state):
|
||||
directory.mkdir()
|
||||
(runner_dir / "relocate_docker_dataroot.sh").symlink_to(SCRIPT)
|
||||
stubs = [(runner_dir / "dump_state.sh", "#!/usr/bin/env bash\nexit 0\n")]
|
||||
stubs += [(bin_dir / "sudo", SUDO_STUB)]
|
||||
stubs += [(bin_dir / name, UNPRIVILEGED_STUB) for name in DESTRUCTIVE]
|
||||
stubs += [(bin_dir / name, body) for name, body in UNPRIVILEGED.items()]
|
||||
stubs += [(stub_bin / name, body) for name, body in PRIVILEGED.items()]
|
||||
for path, body in stubs:
|
||||
path.write_text(body)
|
||||
path.chmod(0o755)
|
||||
log = root / "calls.log"
|
||||
log.touch()
|
||||
daemon_out = root / "daemon.json"
|
||||
if runner.daemon_exists:
|
||||
daemon_out.write_text(runner.daemon_in)
|
||||
env = {
|
||||
k: v
|
||||
for k, v in os.environ.items()
|
||||
if k
|
||||
not in ("BASH_ENV", "ACT", "RUNNER_ENVIRONMENT", "SKIP_DOCKER_RELOCATE")
|
||||
}
|
||||
env.update(
|
||||
PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
|
||||
STUB_LOG=str(log),
|
||||
STUB_BIN=str(stub_bin),
|
||||
STUB_STATE=str(state),
|
||||
STUB_MNT_MOUNTED="1" if runner.mounted else "0",
|
||||
STUB_MNT_FREE=str(runner.mnt_free),
|
||||
STUB_ROOT_FREE=str(runner.root_free),
|
||||
STUB_CONTAINERD_DUMP=runner.dump,
|
||||
STUB_CONTAINERD_UNIT_RC=str(runner.containerd_unit_rc),
|
||||
STUB_CONTAINERS=runner.containers,
|
||||
STUB_IMAGES=runner.images,
|
||||
STUB_VOLUMES=runner.volumes,
|
||||
STUB_BUILD_CACHE=runner.build_cache,
|
||||
STUB_NAMESPACES=runner.namespaces,
|
||||
STUB_FOREIGN_NS=runner.foreign_ns,
|
||||
STUB_FOREIGN_CONTAINERS=runner.foreign_containers,
|
||||
STUB_FOREIGN_TASKS=runner.foreign_tasks,
|
||||
STUB_SHIMS=runner.shims,
|
||||
STUB_SHIM_RC=str(runner.shim_rc),
|
||||
STUB_SUDO_RC=str(runner.sudo_rc),
|
||||
STUB_DAEMON_EXISTS="1" if runner.daemon_exists else "0",
|
||||
STUB_DAEMON_IN=runner.daemon_in,
|
||||
STUB_DAEMON_CAT_RC=str(runner.daemon_cat_rc),
|
||||
STUB_DAEMON_OUT=str(daemon_out),
|
||||
STUB_MOUNT_RC=str(runner.mount_rc),
|
||||
STUB_FAIL_SYSTEMCTL=runner.fail_systemctl,
|
||||
STUB_STATUS_RC=str(runner.status_rc),
|
||||
STUB_TERM_ON=runner.term_on,
|
||||
STUB_METADATA="1" if runner.metadata else "0",
|
||||
STUB_DOCKER_ROOT=runner.docker_root,
|
||||
STUB_DOCKER_ROOT_DEV=runner.docker_root_dev,
|
||||
**runner.extra,
|
||||
)
|
||||
optional = {
|
||||
"RUNNER_ENVIRONMENT": runner.runner_environment,
|
||||
"ACT": runner.act,
|
||||
"SKIP_DOCKER_RELOCATE": runner.skip,
|
||||
}
|
||||
env.update({k: v for k, v in optional.items() if v is not None})
|
||||
done = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(runner_dir / "relocate_docker_dataroot.sh"),
|
||||
runner.min_free_gb,
|
||||
],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
calls = log.read_text().splitlines() # nocheck: cache-read stub call log
|
||||
content = (
|
||||
daemon_out.read_text() # nocheck: cache-read stub output
|
||||
if daemon_out.exists()
|
||||
else None
|
||||
)
|
||||
try:
|
||||
daemon = json.loads(content) if content else None
|
||||
except json.JSONDecodeError:
|
||||
daemon = None
|
||||
return Outcome(
|
||||
done=done,
|
||||
calls=calls,
|
||||
daemon=daemon,
|
||||
daemon_raw=content,
|
||||
bound=(state / "bound").exists(),
|
||||
)
|
||||
|
||||
|
||||
class TestRelocation(unittest.TestCase):
|
||||
def test_small_shape_moves_containerd_and_docker_onto_the_spare_disk(self) -> None:
|
||||
out = relocate(Runner())
|
||||
self.assertEqual(out.done.returncode, 0, out.done.stderr)
|
||||
stop, bind = out.calls.index(STOP), out.calls.index(BIND)
|
||||
start = out.calls.index("systemctl start containerd.service")
|
||||
self.assertLess(stop, bind)
|
||||
self.assertLess(bind, start)
|
||||
self.assertIn("systemctl start docker.socket docker.service", out.calls)
|
||||
self.assertIn(
|
||||
"test -f /mnt/containerd/io.containerd.metadata.v1.bolt/meta.db", out.calls
|
||||
)
|
||||
self.assertEqual(out.daemon, {"data-root": "/mnt/docker-host"})
|
||||
self.assertTrue(out.bound)
|
||||
|
||||
def test_containerd_1x_dump_with_double_quotes_is_understood(self) -> None:
|
||||
out = relocate(Runner(dump=DUMP_V1))
|
||||
self.assertEqual(out.done.returncode, 0, out.done.stderr)
|
||||
self.assertIn(BIND, out.calls)
|
||||
|
||||
def test_existing_daemon_json_keeps_its_settings(self) -> None:
|
||||
out = relocate(Runner(daemon_exists=True, daemon_in='{"debug": true}'))
|
||||
self.assertEqual(out.done.returncode, 0, out.done.stderr)
|
||||
self.assertEqual(out.daemon, {"debug": True, "data-root": "/mnt/docker-host"})
|
||||
|
||||
|
||||
class TestDeclines(unittest.TestCase):
|
||||
def assert_declined(self, out: Outcome, reason: str) -> None:
|
||||
"""Assert a visible no-op: exit 0, a notice with the reason, no call.
|
||||
|
||||
Args:
|
||||
out: the outcome of the run.
|
||||
reason: text the notice must carry.
|
||||
"""
|
||||
self.assertEqual(out.done.returncode, 0, out.done.stderr)
|
||||
self.assertIn("::notice::", out.done.stdout)
|
||||
self.assertIn(reason, out.done.stdout)
|
||||
self.assertEqual(out.calls, [])
|
||||
|
||||
def test_large_shape_without_a_mnt_mount(self) -> None:
|
||||
self.assert_declined(
|
||||
relocate(Runner(mounted=False)), "/mnt is not a separate mount"
|
||||
)
|
||||
|
||||
def test_self_hosted_runner(self) -> None:
|
||||
self.assert_declined(
|
||||
relocate(Runner(runner_environment="self-hosted")),
|
||||
"not a GitHub-hosted runner",
|
||||
)
|
||||
|
||||
def test_unknown_runner_environment(self) -> None:
|
||||
self.assert_declined(
|
||||
relocate(Runner(runner_environment=None)), "not a GitHub-hosted runner"
|
||||
)
|
||||
|
||||
def test_mnt_below_the_floor(self) -> None:
|
||||
self.assert_declined(relocate(Runner(mnt_free=60, root_free=20)), "below 75G")
|
||||
|
||||
def test_mnt_not_roomier_than_root(self) -> None:
|
||||
self.assert_declined(
|
||||
relocate(Runner(mnt_free=80, root_free=90)), "not more than"
|
||||
)
|
||||
|
||||
def test_skip_toggle(self) -> None:
|
||||
self.assert_declined(relocate(Runner(skip="1")), "SKIP_DOCKER_RELOCATE=1")
|
||||
|
||||
|
||||
class TestRefusalsBeforeTheStop(unittest.TestCase):
|
||||
def assert_refused(self, out: Outcome, reason: str) -> None:
|
||||
"""Assert a loud refusal before anything was stopped or written.
|
||||
|
||||
Args:
|
||||
out: the outcome of the run.
|
||||
reason: text the error must carry.
|
||||
"""
|
||||
self.assertEqual(out.done.returncode, 1)
|
||||
self.assertIn(reason, out.done.stderr)
|
||||
self.assertNotIn(STOP, out.calls)
|
||||
self.assertFalse([c for c in out.calls if c.startswith(("mount ", "install "))])
|
||||
self.assertFalse(out.bound)
|
||||
|
||||
def test_act_even_with_a_faked_github_runner(self) -> None:
|
||||
for act in ("true", "1"):
|
||||
with self.subTest(act=act):
|
||||
self.assert_refused(
|
||||
relocate(Runner(act=act)), "refuses to run under act"
|
||||
)
|
||||
|
||||
def test_non_numeric_floor(self) -> None:
|
||||
self.assert_refused(relocate(Runner(min_free_gb="forty")), "positive integer")
|
||||
|
||||
def test_missing_containerd_unit(self) -> None:
|
||||
self.assert_refused(
|
||||
relocate(Runner(containerd_unit_rc=1)), "containerd.service is missing"
|
||||
)
|
||||
|
||||
def test_containerd_root_elsewhere(self) -> None:
|
||||
self.assert_refused(
|
||||
relocate(Runner(dump="version = 3\nroot = '/data/containerd'\n")),
|
||||
"/data/containerd",
|
||||
)
|
||||
|
||||
def test_existing_containers(self) -> None:
|
||||
self.assert_refused(relocate(Runner(containers="abc123\n")), "abc123")
|
||||
|
||||
def test_existing_images(self) -> None:
|
||||
self.assert_refused(relocate(Runner(images="sha256feed\n")), "sha256feed")
|
||||
|
||||
def test_existing_volumes(self) -> None:
|
||||
self.assert_refused(
|
||||
relocate(Runner(volumes="core_infinito\n")), "core_infinito"
|
||||
)
|
||||
|
||||
def test_existing_build_cache(self) -> None:
|
||||
self.assert_refused(relocate(Runner(build_cache="3")), "build cache")
|
||||
|
||||
def test_containers_in_another_namespace(self) -> None:
|
||||
out = relocate(
|
||||
Runner(namespaces="moby\nk8s.io\n", foreign_containers="pause1\n")
|
||||
)
|
||||
self.assert_refused(out, "k8s.io holds containers: pause1")
|
||||
|
||||
def test_tasks_in_another_namespace(self) -> None:
|
||||
out = relocate(Runner(namespaces="moby\nk8s.io\n", foreign_tasks="task7\n"))
|
||||
self.assert_refused(out, "k8s.io runs tasks: task7")
|
||||
|
||||
def test_running_shims(self) -> None:
|
||||
out = relocate(Runner(shims="4242 containerd-shim-runc-v2 -id x\n", shim_rc=0))
|
||||
self.assert_refused(out, "containerd shims are running")
|
||||
|
||||
def test_shim_listing_error(self) -> None:
|
||||
self.assert_refused(
|
||||
relocate(Runner(shim_rc=2)), "could not list containerd shims"
|
||||
)
|
||||
|
||||
def test_broken_sudo(self) -> None:
|
||||
out = relocate(Runner(sudo_rc=1))
|
||||
self.assertEqual(out.done.returncode, 1)
|
||||
self.assertNotIn(STOP, out.calls)
|
||||
|
||||
def test_unreadable_daemon_json_is_never_overwritten(self) -> None:
|
||||
out = relocate(Runner(daemon_exists=True, daemon_in="{}", daemon_cat_rc=1))
|
||||
self.assertEqual(out.done.returncode, 1)
|
||||
self.assertNotIn(STOP, out.calls)
|
||||
self.assertFalse([c for c in out.calls if c.startswith("install ")])
|
||||
self.assertEqual(out.daemon, {})
|
||||
|
||||
def test_invalid_daemon_json_is_never_overwritten(self) -> None:
|
||||
out = relocate(Runner(daemon_exists=True, daemon_in="{not json"))
|
||||
self.assertEqual(out.done.returncode, 1)
|
||||
self.assertIn("is not valid JSON", out.done.stderr)
|
||||
self.assertNotIn(STOP, out.calls)
|
||||
self.assertFalse([c for c in out.calls if c.startswith("install ")])
|
||||
self.assertEqual(out.daemon_raw, "{not json")
|
||||
|
||||
|
||||
class TestRollback(unittest.TestCase):
|
||||
def assert_rolled_back(self, out: Outcome, reason: str) -> None:
|
||||
"""Assert the rollback: no bind left, daemons restarted last, exit 1.
|
||||
|
||||
Args:
|
||||
out: the outcome of the run.
|
||||
reason: text the rollback error must carry.
|
||||
"""
|
||||
self.assertEqual(out.done.returncode, 1, out.done.stderr)
|
||||
self.assertIn("rolling Docker and containerd back onto /", out.done.stderr)
|
||||
self.assertIn(reason, out.done.stderr)
|
||||
self.assertFalse(out.bound)
|
||||
self.assertIn(RESTART, out.calls)
|
||||
self.assertTrue([c for c in out.calls if c.startswith("journalctl ")])
|
||||
|
||||
def test_failed_bind_mount_removes_the_daemon_json_it_never_had(self) -> None:
|
||||
out = relocate(Runner(mount_rc=32))
|
||||
self.assert_rolled_back(out, "mount --bind")
|
||||
self.assertIsNone(out.daemon)
|
||||
|
||||
def test_failed_containerd_start_unbinds_before_the_restart(self) -> None:
|
||||
out = relocate(
|
||||
Runner(
|
||||
daemon_exists=True,
|
||||
daemon_in='{"debug": true}',
|
||||
fail_systemctl="start containerd.service",
|
||||
)
|
||||
)
|
||||
self.assert_rolled_back(out, "systemctl start containerd.service")
|
||||
self.assertLess(out.calls.index(UNBIND), len(out.calls) - 1)
|
||||
self.assertLess(out.calls.index(UNBIND), out.calls.index(RESTART))
|
||||
self.assertEqual(out.daemon, {"debug": True})
|
||||
|
||||
def test_cancel_during_the_move_rolls_back(self) -> None:
|
||||
out = relocate(Runner(term_on="mkdir"))
|
||||
self.assert_rolled_back(out, "interrupted")
|
||||
|
||||
def test_down_units_still_yield_the_journal(self) -> None:
|
||||
out = relocate(Runner(mount_rc=32, status_rc=3))
|
||||
self.assert_rolled_back(out, "mount --bind")
|
||||
|
||||
def test_containerd_without_metadata_on_the_spare_disk(self) -> None:
|
||||
out = relocate(Runner(metadata=False))
|
||||
self.assert_rolled_back(out, "without its metadata")
|
||||
self.assertIn(UNBIND, out.calls)
|
||||
self.assertIsNone(out.daemon)
|
||||
|
||||
def test_docker_root_elsewhere(self) -> None:
|
||||
out = relocate(Runner(docker_root="/var/lib/docker"))
|
||||
self.assert_rolled_back(out, "expected '/mnt/docker-host'")
|
||||
|
||||
def test_docker_root_off_the_spare_disk(self) -> None:
|
||||
out = relocate(Runner(docker_root_dev="66305"))
|
||||
self.assert_rolled_back(out, "not on the /mnt filesystem")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user