Merge remote-tracking branch 'origin/main' into fix/sys-ctl-hlth-runtime/vulnerability-engines/1324

# Conflicts:
#	.github/workflows/call-test-deploy.yml

tests/unit/python/scripts/github/runner/test_enlarge_swap_placement.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+from utils.cache.files import PROJECT_ROOT
+
+SCRIPT = PROJECT_ROOT / "scripts" / "github" / "runner" / "enlarge_swap.sh"
+SUDO_STUB = '#!/usr/bin/env bash\nprintf "%s\\n" "$*" >>"${STUB_LOG}"\n'
+DF_STUB = (
+    "#!/usr/bin/env bash\n"
+    'echo "Avail"\n'
+    'if [ "${*: -1}" = /mnt ]; then echo "${STUB_MNT_FREE}"; else echo "${STUB_ROOT_FREE}"; fi\n'
+)
+MOUNTPOINT_STUB = '#!/usr/bin/env bash\n[ "${STUB_MNT_MOUNTED}" = 1 ]\n'
+STAT_STUB = (
+    "#!/usr/bin/env bash\n"
+    'case "${*: -1}" in\n'
+    "/mnt) echo 2049 ;;\n"
+    '/var/lib/containerd) if [ "${STUB_CONTAINERD_DEV}" = missing ]; then exit 1; fi\n'
+    '  echo "${STUB_CONTAINERD_DEV}" ;;\n'
+    "*) exit 1 ;;\n"
+    "esac\n"
+)
+DUMP_STUB = "#!/usr/bin/env bash\nexit 0\n"
+
+
+def enlarge(
+    *, mounted: bool, mnt_free: int, root_free: int, containerd_dev: str
+) -> tuple[subprocess.CompletedProcess, list[str]]:
+    """Run enlarge_swap.sh against stubbed disks.
+
+    Args:
+        mounted: whether /mnt is a separate mount.
+        mnt_free: GB df reports free on /mnt.
+        root_free: GB df reports free on /.
+        containerd_dev: device id stat reports for /var/lib/containerd
+            (2049 is the /mnt device).
+
+    Returns:
+        The finished process and every sudo call in order.
+    """
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        runner_dir = root / "runner"
+        runner_dir.mkdir()
+        (runner_dir / "enlarge_swap.sh").symlink_to(SCRIPT)
+        bin_dir = root / "bin"
+        bin_dir.mkdir()
+        for path, body in (
+            (runner_dir / "dump_state.sh", DUMP_STUB),
+            (bin_dir / "sudo", SUDO_STUB),
+            (bin_dir / "df", DF_STUB),
+            (bin_dir / "mountpoint", MOUNTPOINT_STUB),
+            (bin_dir / "stat", STAT_STUB),
+        ):
+            path.write_text(body)
+            path.chmod(0o755)
+        log = root / "sudo.log"
+        log.touch()
+        env = {
+            k: v for k, v in os.environ.items() if k not in ("BASH_ENV", "SKIP_SWAP")
+        }
+        env.update(
+            PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
+            STUB_LOG=str(log),
+            STUB_MNT_MOUNTED="1" if mounted else "0",
+            STUB_MNT_FREE=str(mnt_free),
+            STUB_ROOT_FREE=str(root_free),
+            STUB_CONTAINERD_DEV=containerd_dev,
+        )
+        done = subprocess.run(
+            ["bash", str(runner_dir / "enlarge_swap.sh")],
+            env=env,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        return done, log.read_text().splitlines()  # nocheck: cache-read  stub call log
+
+
+def swapfile(calls: list[str]) -> list[str]:
+    """Return the fallocate calls among the recorded sudo calls.
+
+    Args:
+        calls: every recorded sudo call.
+
+    Returns:
+        The calls that create the swapfile.
+    """
+    return [call for call in calls if call.startswith("fallocate ")]
+
+
+class TestEnlargeSwapPlacement(unittest.TestCase):
+    def test_swap_goes_to_root_when_containerd_lives_on_the_spare_disk(self) -> None:
+        done, calls = enlarge(
+            mounted=True, mnt_free=200, root_free=54, containerd_dev="2049"
+        )
+        self.assertEqual(done.returncode, 0, done.stderr)
+        self.assertEqual(swapfile(calls), ["fallocate -l 16G /swapfile"])
+
+    def test_swap_goes_to_mnt_when_containerd_stayed_on_root(self) -> None:
+        done, calls = enlarge(
+            mounted=True, mnt_free=60, root_free=54, containerd_dev="66305"
+        )
+        self.assertEqual(done.returncode, 0, done.stderr)
+        self.assertEqual(swapfile(calls), ["fallocate -l 16G /mnt/swapfile"])
+
+    def test_large_shape_without_mnt_keeps_swap_on_root(self) -> None:
+        done, calls = enlarge(
+            mounted=False, mnt_free=0, root_free=123, containerd_dev="66305"
+        )
+        self.assertEqual(done.returncode, 0, done.stderr)
+        self.assertEqual(swapfile(calls), ["fallocate -l 16G /swapfile"])
+
+    def test_missing_containerd_root_keeps_the_mnt_preference(self) -> None:
+        done, calls = enlarge(
+            mounted=True, mnt_free=60, root_free=54, containerd_dev="missing"
+        )
+        self.assertEqual(done.returncode, 0, done.stderr)
+        self.assertEqual(swapfile(calls), ["fallocate -l 16G /mnt/swapfile"])
+
+    def test_full_root_still_uses_the_spare_disk(self) -> None:
+        done, calls = enlarge(
+            mounted=True, mnt_free=200, root_free=10, containerd_dev="2049"
+        )
+        self.assertEqual(done.returncode, 0, done.stderr)
+        self.assertEqual(swapfile(calls), ["fallocate -l 16G /mnt/swapfile"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/python/scripts/github/runner/test_mount_spare_disk.py

@@ -0,0 +1,457 @@
+from __future__ import annotations
+
+import copy
+import json
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+from utils.cache.files import PROJECT_ROOT
+from utils.cache.yaml import load_yaml
+
+SCRIPT = PROJECT_ROOT / "scripts" / "github" / "runner" / "mount_spare_disk.sh"
+WORKFLOW = PROJECT_ROOT / ".github" / "workflows" / "call-test-deploy.yml"
+MIN_SIZE_GB = "75"
+GIB = 1024**3
+SPARE = "nvme9n9"
+SPARE_INDEX = 2
+DESTRUCTIVE = ("mkfs.ext4", "blkid", "wipefs", "mount", "mkswap")
+
+SUDO_STUB = (
+    "#!/usr/bin/env bash\n"
+    'printf "%s\\n" "$*" >>"${STUB_LOG}"\n'
+    'if [ -x "${STUB_BIN}/$1" ]; then exec "${STUB_BIN}/$1" "${@:2}"; fi\n'
+    'echo "unstubbed sudo $1" >&2\n'
+    "exit 97\n"
+)
+UNPRIVILEGED_STUB = (
+    '#!/usr/bin/env bash\necho "$(basename "$0") ran without sudo" >&2\nexit 98\n'
+)
+LSBLK_STUB = '#!/usr/bin/env bash\ncat "${STUB_LSBLK_JSON}"\n'
+MOUNTPOINT_STUB = '#!/usr/bin/env bash\n[ "${STUB_MOUNTED}" = 1 ]\n'
+BLKID_STUB = (
+    '#!/usr/bin/env bash\nprintf "%s" "${STUB_BLKID_OUT}"\nexit "${STUB_BLKID_RC}"\n'
+)
+WIPEFS_STUB = '#!/usr/bin/env bash\nprintf "%s" "${STUB_WIPEFS_OUT}"\n'
+MKFS_STUB = "#!/usr/bin/env bash\nexit 0\n"
+MOUNT_STUB = (
+    '#!/usr/bin/env bash\nprintf "%s" "${STUB_MOUNT_AS:-$1}" >"${STUB_STATE}"\n'
+)
+FINDMNT_STUB = '#!/usr/bin/env bash\ncat "${STUB_STATE}"\n'
+DUMP_STUB = "#!/usr/bin/env bash\nexit 0\n"
+
+
+def part(name: str, size_gib: float, fstype: str | None, mount: str | None) -> dict:
+    """Build one lsblk partition entry.
+
+    Args:
+        name: kernel name of the partition.
+        size_gib: size in GiB.
+        fstype: filesystem type, None when unformatted.
+        mount: mount point, None when unmounted.
+
+    Returns:
+        The entry as lsblk -J -b prints it.
+    """
+    return {
+        "name": name,
+        "type": "part",
+        "size": int(size_gib * GIB),
+        "ro": False,
+        "fstype": fstype,
+        "pttype": "gpt",
+        "mountpoints": [mount],
+    }
+
+
+def snap_loops() -> list[dict]:
+    """Return the two snap squashfs loop devices every runner shape carries.
+
+    Returns:
+        The loop entries as lsblk -J -b prints them.
+    """
+    return [
+        {
+            "name": f"loop{index}",
+            "type": "loop",
+            "size": 77594624,
+            "ro": True,
+            "fstype": "squashfs",
+            "pttype": None,
+            "mountpoints": [mount],
+        }
+        for index, mount in enumerate(("/snap/core22/2955", "/snap/snapd/28254"))
+    ]
+
+
+def root_disk(name: str, prefix: str, size_gib: int) -> dict:
+    """Build the partitioned root disk of a GitHub runner image.
+
+    Args:
+        name: kernel name of the disk.
+        prefix: partition name prefix (``sda`` or ``nvme0n1p``).
+        size_gib: size in GiB.
+
+    Returns:
+        The disk entry with its four partitions.
+    """
+    return {
+        "name": name,
+        "type": "disk",
+        "size": size_gib * GIB,
+        "ro": False,
+        "fstype": None,
+        "pttype": "gpt",
+        "mountpoints": [None],
+        "children": [
+            part(f"{prefix}1", size_gib - 1.2, "ext4", "/"),
+            part(f"{prefix}13", 1, "ext4", "/boot"),
+            part(f"{prefix}14", 0.004, None, None),
+            part(f"{prefix}15", 0.1, "vfat", "/boot/efi"),
+        ],
+    }
+
+
+def spare_disk(size_gib: int) -> dict:
+    """Build the empty second NVMe disk of the small runner shape.
+
+    Args:
+        size_gib: size in GiB.
+
+    Returns:
+        The disk entry without filesystem, partition table or mount.
+    """
+    return {
+        "name": SPARE,
+        "type": "disk",
+        "size": size_gib * GIB,
+        "ro": False,
+        "fstype": None,
+        "pttype": None,
+        "mountpoints": [None],
+    }
+
+
+LARGE_SHAPE = {"blockdevices": [*snap_loops(), root_disk("sda", "sda", 145)]}
+SMALL_SHAPE = {
+    "blockdevices": [
+        *snap_loops(),
+        spare_disk(128),
+        root_disk("nvme0n1", "nvme0n1p", 72),
+    ]
+}
+
+
+def with_spare(**changes: object) -> dict:
+    """Return the small shape with fields of the spare disk replaced.
+
+    Args:
+        **changes: lsblk fields to overwrite on the spare disk.
+
+    Returns:
+        A modified deep copy of SMALL_SHAPE.
+    """
+    tree = copy.deepcopy(SMALL_SHAPE)
+    tree["blockdevices"][SPARE_INDEX].update(changes)
+    return tree
+
+
+def run(
+    tree: dict,
+    *,
+    runner_environment: str | None = "github-hosted",
+    act: str | None = None,
+    mounted: bool = False,
+    blkid_rc: int = 2,
+    blkid_out: str = "",
+    wipefs_out: str = "",
+    mount_as: str | None = None,
+    occupied: bool = False,
+    min_size_gb: str = MIN_SIZE_GB,
+) -> tuple[subprocess.CompletedProcess, list[str]]:
+    """Run mount_spare_disk.sh against a faked block-device layout.
+
+    Args:
+        tree: what ``lsblk -J -b`` reports.
+        runner_environment: RUNNER_ENVIRONMENT, None to leave it unset.
+        act: ACT, None to leave it unset.
+        mounted: whether the mount point already is a mount.
+        blkid_rc: exit code of ``blkid -p`` (2 means no signature).
+        blkid_out: what ``blkid -p`` prints.
+        wipefs_out: what ``wipefs --no-act`` prints.
+        mount_as: device findmnt reports after the mount, None for the mounted one.
+        occupied: whether the mount point already holds a file.
+        min_size_gb: the MIN_SIZE_GB argument.
+
+    Returns:
+        The finished process and every privileged call in order.
+    """
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        runner_dir = root / "runner"
+        runner_dir.mkdir()
+        (runner_dir / "mount_spare_disk.sh").symlink_to(SCRIPT)
+        bin_dir = root / "bin"
+        bin_dir.mkdir()
+        stub_bin = root / "privileged"
+        stub_bin.mkdir()
+        stubs = [
+            (runner_dir / "dump_state.sh", DUMP_STUB),
+            (bin_dir / "sudo", SUDO_STUB),
+            (bin_dir / "lsblk", LSBLK_STUB),
+            (bin_dir / "mountpoint", MOUNTPOINT_STUB),
+            (bin_dir / "findmnt", FINDMNT_STUB),
+            (stub_bin / "blkid", BLKID_STUB),
+            (stub_bin / "wipefs", WIPEFS_STUB),
+            (stub_bin / "mkfs.ext4", MKFS_STUB),
+            (stub_bin / "mount", MOUNT_STUB),
+        ]
+        stubs += [(bin_dir / name, UNPRIVILEGED_STUB) for name in DESTRUCTIVE]
+        for path, body in stubs:
+            path.write_text(body)
+            path.chmod(0o755)
+        mount_point = root / "mnt"
+        mount_point.mkdir()
+        if occupied:
+            (mount_point / "keep.txt").write_text("data\n")
+        lsblk_json = root / "lsblk.json"
+        lsblk_json.write_text(json.dumps(tree))
+        log = root / "sudo.log"
+        log.touch()
+        env = {
+            k: v
+            for k, v in os.environ.items()
+            if k not in ("BASH_ENV", "ACT", "RUNNER_ENVIRONMENT")
+        }
+        env.update(
+            PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
+            STUB_LOG=str(log),
+            STUB_BIN=str(stub_bin),
+            STUB_LSBLK_JSON=str(lsblk_json),
+            STUB_MOUNTED="1" if mounted else "0",
+            STUB_BLKID_RC=str(blkid_rc),
+            STUB_BLKID_OUT=blkid_out,
+            STUB_WIPEFS_OUT=wipefs_out,
+            STUB_STATE=str(root / "mounted-source"),
+        )
+        if mount_as is not None:
+            env["STUB_MOUNT_AS"] = mount_as
+        if runner_environment is not None:
+            env["RUNNER_ENVIRONMENT"] = runner_environment
+        if act is not None:
+            env["ACT"] = act
+        done = subprocess.run(
+            [
+                "bash",
+                str(runner_dir / "mount_spare_disk.sh"),
+                str(mount_point),
+                min_size_gb,
+            ],
+            env=env,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        return done, log.read_text().splitlines()  # nocheck: cache-read  stub call log
+
+
+def formatted(calls: list[str]) -> bool:
+    """Tell whether any privileged call formatted or mounted a disk.
+
+    Args:
+        calls: every recorded sudo call.
+
+    Returns:
+        True when mkfs or mount ran.
+    """
+    return any(call.startswith(("mkfs", "mount ")) for call in calls)
+
+
+class TestMountSpareDisk(unittest.TestCase):
+    def assert_untouched(self, done: subprocess.CompletedProcess, reason: str) -> None:
+        """Assert a declined run: exit 0, no privileged call, the stated reason.
+
+        Args:
+            done: the finished script.
+            reason: text the script must print for this decline.
+        """
+        self.assertEqual(done.returncode, 0, done.stderr)
+        self.assertIn(reason, done.stdout + done.stderr)
+        self.assertRegex(done.stdout, r"::(notice|warning)::")
+
+    def test_small_shape_formats_and_mounts_the_empty_nvme_disk(self) -> None:
+        done, calls = run(SMALL_SHAPE)
+        self.assertEqual(done.returncode, 0, done.stderr)
+        self.assertEqual(
+            calls[:3],
+            [
+                f"blkid -p /dev/{SPARE}",
+                f"wipefs --no-act --noheadings /dev/{SPARE}",
+                f"mkfs.ext4 -q -m 0 -L infinito-spare /dev/{SPARE}",
+            ],
+        )
+        self.assertTrue(calls[3].startswith(f"mount /dev/{SPARE} "), calls)
+        self.assertEqual(len(calls), 4, calls)
+
+    def test_large_shape_without_a_spare_disk_stays_as_it_is(self) -> None:
+        done, calls = run(LARGE_SHAPE)
+        self.assert_untouched(done, '["sda"]')
+        self.assertEqual(calls, [])
+
+    def test_self_hosted_runner_is_never_formatted(self) -> None:
+        done, calls = run(SMALL_SHAPE, runner_environment="self-hosted")
+        self.assert_untouched(done, "not a GitHub-hosted runner")
+        self.assertEqual(calls, [])
+
+    def test_unknown_runner_environment_is_never_formatted(self) -> None:
+        done, calls = run(SMALL_SHAPE, runner_environment=None)
+        self.assert_untouched(done, "not a GitHub-hosted runner")
+        self.assertEqual(calls, [])
+
+    def test_act_refuses_loudly_even_with_a_faked_github_runner(self) -> None:
+        for act in ("true", "1", "TRUE"):
+            with self.subTest(act=act):
+                done, calls = run(SMALL_SHAPE, act=act)
+                self.assertEqual(done.returncode, 1)
+                self.assertEqual(calls, [])
+                self.assertIn("refuses to run under act", done.stderr)
+
+    def test_disk_with_a_filesystem_is_never_formatted(self) -> None:
+        done, calls = run(with_spare(fstype="ext4"))
+        self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
+        self.assertEqual(calls, [])
+
+    def test_disk_with_partitions_is_never_formatted(self) -> None:
+        done, calls = run(with_spare(children=[part(f"{SPARE}p1", 127, None, None)]))
+        self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
+        self.assertEqual(calls, [])
+
+    def test_disk_held_by_lvm_is_never_formatted(self) -> None:
+        holder = {
+            "name": "vg-data",
+            "type": "lvm",
+            "size": 127 * GIB,
+            "ro": False,
+            "fstype": None,
+            "pttype": None,
+            "mountpoints": [None],
+        }
+        done, calls = run(with_spare(children=[holder]))
+        self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
+        self.assertEqual(calls, [])
+
+    def test_disk_with_a_bare_partition_table_is_never_formatted(self) -> None:
+        done, calls = run(with_spare(pttype="dos"))
+        self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
+        self.assertEqual(calls, [])
+
+    def test_mounted_disk_is_never_formatted(self) -> None:
+        done, calls = run(with_spare(mountpoints=["/data"]))
+        self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
+        self.assertEqual(calls, [])
+
+    def test_disk_used_as_swap_is_never_formatted(self) -> None:
+        done, calls = run(with_spare(mountpoints=["[SWAP]"]))
+        self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
+        self.assertEqual(calls, [])
+
+    def test_read_only_disk_is_never_formatted(self) -> None:
+        done, calls = run(with_spare(ro=True))
+        self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
+        self.assertEqual(calls, [])
+
+    def test_disk_below_the_size_floor_stays_as_it_is(self) -> None:
+        done, calls = run(with_spare(size=32 * GIB))
+        self.assert_untouched(done, "No empty, unpartitioned, unmounted disk")
+        self.assertEqual(calls, [])
+
+    def test_signature_lsblk_missed_is_never_formatted(self) -> None:
+        done, calls = run(SMALL_SHAPE, blkid_rc=0, blkid_out='TYPE="xfs"')
+        self.assert_untouched(done, "carries a signature")
+        self.assertFalse(formatted(calls), calls)
+
+    def test_signature_only_wipefs_sees_is_never_formatted(self) -> None:
+        done, calls = run(SMALL_SHAPE, wipefs_out="0x218 LVM2_member")
+        self.assert_untouched(done, "carries signatures blkid missed")
+        self.assertFalse(formatted(calls), calls)
+
+    def test_inconclusive_probe_fails_loudly_without_formatting(self) -> None:
+        done, calls = run(SMALL_SHAPE, blkid_rc=8, blkid_out="ambivalent result")
+        self.assertEqual(done.returncode, 1)
+        self.assertFalse(formatted(calls), calls)
+
+    def test_probe_that_found_nothing_but_complained_fails_loudly(self) -> None:
+        done, calls = run(SMALL_SHAPE, blkid_rc=2, blkid_out="Input/output error")
+        self.assertEqual(done.returncode, 1)
+        self.assertFalse(formatted(calls), calls)
+
+    def test_lsblk_without_mountpoints_fails_loudly(self) -> None:
+        tree = copy.deepcopy(SMALL_SHAPE)
+        spare = tree["blockdevices"][SPARE_INDEX]
+        spare["mountpoint"] = spare.pop("mountpoints")[0]
+        done, calls = run(tree)
+        self.assertEqual(done.returncode, 1)
+        self.assertEqual(calls, [])
+
+    def test_layout_without_a_root_device_fails_loudly(self) -> None:
+        done, calls = run({"blockdevices": [spare_disk(128)]})
+        self.assertEqual(done.returncode, 1)
+        self.assertEqual(calls, [])
+
+    def test_non_numeric_size_floor_fails_loudly(self) -> None:
+        done, calls = run(SMALL_SHAPE, min_size_gb="RANDOM")
+        self.assertEqual(done.returncode, 1)
+        self.assertEqual(calls, [])
+
+    def test_existing_mount_stays_as_it_is(self) -> None:
+        done, calls = run(SMALL_SHAPE, mounted=True)
+        self.assert_untouched(done, "already a separate mount")
+        self.assertEqual(calls, [])
+
+    def test_occupied_mount_point_stays_as_it_is(self) -> None:
+        done, calls = run(SMALL_SHAPE, occupied=True)
+        self.assert_untouched(done, "is not empty")
+        self.assertEqual(calls, [])
+
+    def test_mount_landing_on_another_device_fails_loudly(self) -> None:
+        done, _ = run(SMALL_SHAPE, mount_as="/dev/nvme0n1p1")
+        self.assertEqual(done.returncode, 1)
+        self.assertIn("backed by '/dev/nvme0n1p1'", done.stderr)
+
+
+class TestWorkflowWiring(unittest.TestCase):
+    def test_spare_disk_step_is_act_gated_and_runs_before_swap_and_relocation(
+        self,
+    ) -> None:
+        workflow = load_yaml(str(WORKFLOW))
+        steps = next(
+            job["steps"]
+            for job in workflow["jobs"].values()
+            if any(
+                "mount_spare_disk.sh" in str(step.get("run", ""))
+                for step in job.get("steps", [])
+            )
+        )
+        runs = [str(step.get("run", "")) for step in steps]
+        index = next(i for i, cmd in enumerate(runs) if "mount_spare_disk.sh" in cmd)
+        self.assertEqual(
+            runs[index],
+            f"./scripts/github/runner/mount_spare_disk.sh /mnt {MIN_SIZE_GB}",
+        )
+        self.assertEqual(steps[index].get("if"), "${{ !env.ACT }}")
+        swap = next(i for i, cmd in enumerate(runs) if "enlarge_swap.sh" in cmd)
+        relocate = next(
+            i for i, cmd in enumerate(runs) if "relocate_docker_dataroot.sh" in cmd
+        )
+        self.assertLess(index, relocate)
+        self.assertLess(relocate, swap)
+        self.assertEqual(
+            runs[relocate], "./scripts/github/runner/relocate_docker_dataroot.sh 75"
+        )
+        self.assertEqual(steps[relocate].get("if"), "${{ !env.ACT }}")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/python/scripts/github/runner/test_relocate_docker_dataroot.py

@@ -0,0 +1,524 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import tempfile
+import unittest
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from utils.cache.files import PROJECT_ROOT
+
+SCRIPT = PROJECT_ROOT / "scripts" / "github" / "runner" / "relocate_docker_dataroot.sh"
+MIN_FREE_GB = "75"
+DUMP_V2 = "version = 3\nroot = '/var/lib/containerd'\nstate = '/run/containerd'\n"
+DUMP_V1 = 'version = 2\nroot = "/var/lib/containerd"\nstate = "/run/containerd"\n'
+STOP = "systemctl stop docker.socket docker.service containerd.service"
+BIND = "mount --bind /mnt/containerd /var/lib/containerd"
+UNBIND = "umount /var/lib/containerd"
+RESTART = "systemctl start containerd.service docker.socket docker.service"
+
+SUDO_STUB = (
+    "#!/usr/bin/env bash\n"
+    'printf "%s\\n" "$*" >>"${STUB_LOG}"\n'
+    'if [ -x "${STUB_BIN}/$1" ]; then exec "${STUB_BIN}/$1" "${@:2}"; fi\n'
+    'echo "unstubbed sudo $1" >&2\n'
+    "exit 97\n"
+)
+UNPRIVILEGED_STUB = (
+    '#!/usr/bin/env bash\necho "$(basename "$0") ran without sudo" >&2\nexit 98\n'
+)
+TERM_HOOK = (
+    'if [ "$(basename "$0")" = "${STUB_TERM_ON}" ]; then kill -TERM "${PPID}"; fi\n'
+)
+UNPRIVILEGED = {
+    "mountpoint": (
+        "#!/usr/bin/env bash\n"
+        'case "${*: -1}" in\n'
+        '/mnt) [ "${STUB_MNT_MOUNTED}" = 1 ] ;;\n'
+        '/var/lib/containerd) [ -e "${STUB_STATE}/bound" ] ;;\n'
+        "*) exit 1 ;;\n"
+        "esac\n"
+    ),
+    "df": (
+        "#!/usr/bin/env bash\n"
+        'echo "Avail"\n'
+        'if [ "${*: -1}" = /mnt ]; then echo "${STUB_MNT_FREE}"; '
+        'else echo "${STUB_ROOT_FREE}"; fi\n'
+    ),
+    "systemctl": '#!/usr/bin/env bash\nexit "${STUB_CONTAINERD_UNIT_RC}"\n',
+    "docker": (
+        "#!/usr/bin/env bash\n"
+        'case "$1" in\n'
+        'ps) printf "%s" "${STUB_CONTAINERS}" ;;\n'
+        'images) printf "%s" "${STUB_IMAGES}" ;;\n'
+        'volume) printf "%s" "${STUB_VOLUMES}" ;;\n'
+        'system) printf "Images=0\\nContainers=0\\nLocal Volumes=0\\n'
+        'Build Cache=%s\\n" "${STUB_BUILD_CACHE}" ;;\n'
+        'info) echo "${STUB_DOCKER_ROOT}" ;;\n'
+        "*) exit 96 ;;\n"
+        "esac\n"
+    ),
+    "stat": (
+        "#!/usr/bin/env bash\n"
+        'case "${*: -1}" in\n'
+        "/mnt) echo 2049 ;;\n"
+        '/mnt/docker-host) echo "${STUB_DOCKER_ROOT_DEV}" ;;\n'
+        "*) echo 66305 ;;\n"
+        "esac\n"
+    ),
+    "pgrep": '#!/usr/bin/env bash\nprintf "%s" "${STUB_SHIMS}"\nexit "${STUB_SHIM_RC}"\n',
+}
+PRIVILEGED = {
+    "systemctl": (
+        "#!/usr/bin/env bash\n"
+        'if [ "$1" = status ]; then exit "${STUB_STATUS_RC}"; fi\n'
+        'if [ "$*" = "${STUB_FAIL_SYSTEMCTL}" ]; then exit 1; fi\n'
+    ),
+    "containerd": '#!/usr/bin/env bash\nprintf "%s" "${STUB_CONTAINERD_DUMP}"\n',
+    "ctr": (
+        "#!/usr/bin/env bash\n"
+        'if [ "$1" = namespaces ]; then printf "%s" "${STUB_NAMESPACES}"; exit; fi\n'
+        'if [ "$1" = -n ] && [ "$2" = "${STUB_FOREIGN_NS}" ]; then\n'
+        '  if [ "$3" = containers ]; then printf "%s" "${STUB_FOREIGN_CONTAINERS}"; fi\n'
+        '  if [ "$3" = tasks ]; then printf "%s" "${STUB_FOREIGN_TASKS}"; fi\n'
+        "fi\n"
+    ),
+    "du": '#!/usr/bin/env bash\nprintf "1.2G\\t%s\\n" "${*: -1}"\n',
+    "true": '#!/usr/bin/env bash\nexit "${STUB_SUDO_RC}"\n',
+    "test": (
+        "#!/usr/bin/env bash\n"
+        'if [ "$1" = -e ]; then [ "${STUB_DAEMON_EXISTS}" = 1 ]; exit; fi\n'
+        '[ "${STUB_METADATA}" = 1 ]\n'
+    ),
+    "cat": (
+        "#!/usr/bin/env bash\n"
+        'if [ "${STUB_DAEMON_CAT_RC}" != 0 ]; then exit "${STUB_DAEMON_CAT_RC}"; fi\n'
+        'printf "%s" "${STUB_DAEMON_IN}"\n'
+    ),
+    "mkdir": "#!/usr/bin/env bash\n" + TERM_HOOK,
+    "mount": (
+        "#!/usr/bin/env bash\n"
+        'if [ "${STUB_MOUNT_RC}" != 0 ]; then exit "${STUB_MOUNT_RC}"; fi\n'
+        'touch "${STUB_STATE}/bound"\n'
+    ),
+    "umount": '#!/usr/bin/env bash\nrm -f "${STUB_STATE}/bound"\n',
+    "install": '#!/usr/bin/env bash\ncp "${*: -2:1}" "${STUB_DAEMON_OUT}"\n',
+    "rm": '#!/usr/bin/env bash\n/bin/rm -f "${STUB_DAEMON_OUT}"\n',
+    "journalctl": "#!/usr/bin/env bash\nexit 0\n",
+}
+DESTRUCTIVE = ("mount", "umount", "install", "ctr", "containerd", "journalctl", "mkdir")
+
+
+@dataclass
+class Runner:
+    """The faked runner state relocate_docker_dataroot.sh meets.
+
+    Args:
+        runner_environment: RUNNER_ENVIRONMENT, None to leave it unset.
+        act: ACT, None to leave it unset.
+        skip: SKIP_DOCKER_RELOCATE, None to leave it unset.
+        mounted: whether /mnt is a separate mount.
+        mnt_free: GB df reports free on /mnt.
+        root_free: GB df reports free on /.
+        dump: what ``containerd config dump`` prints.
+        containerd_unit_rc: exit code of ``systemctl cat containerd.service``.
+        containers: what ``docker ps -aq`` prints.
+        images: what ``docker images -q`` prints.
+        volumes: what ``docker volume ls -q`` prints.
+        build_cache: Build Cache record count in ``docker system df``.
+        namespaces: what ``ctr namespaces ls -q`` prints.
+        foreign_ns: namespace whose containers and tasks are faked.
+        foreign_containers: containers in foreign_ns.
+        foreign_tasks: tasks in foreign_ns.
+        shims: what ``pgrep -a containerd-shim`` prints.
+        shim_rc: exit code of pgrep (1 means no shim).
+        sudo_rc: exit code of ``sudo true``.
+        daemon_exists: whether daemon.json exists.
+        daemon_in: content of the existing daemon.json.
+        daemon_cat_rc: exit code of ``sudo cat`` on daemon.json.
+        mount_rc: exit code of the bind mount.
+        fail_systemctl: the one systemctl argument string that fails.
+        status_rc: exit code of ``systemctl status``.
+        term_on: privileged stub that sends SIGTERM to the script.
+        metadata: whether containerd's meta.db shows up under /mnt/containerd.
+        docker_root: what ``docker info`` reports as DockerRootDir.
+        docker_root_dev: device id stat reports for /mnt/docker-host.
+        min_free_gb: the MIN_FREE_GB argument.
+    """
+
+    runner_environment: str | None = "github-hosted"
+    act: str | None = None
+    skip: str | None = None
+    mounted: bool = True
+    mnt_free: int = 137
+    root_free: int = 51
+    dump: str = DUMP_V2
+    containerd_unit_rc: int = 0
+    containers: str = ""
+    images: str = ""
+    volumes: str = ""
+    build_cache: str = "0"
+    namespaces: str = "moby\n"
+    foreign_ns: str = "k8s.io"
+    foreign_containers: str = ""
+    foreign_tasks: str = ""
+    shims: str = ""
+    shim_rc: int = 1
+    sudo_rc: int = 0
+    daemon_exists: bool = False
+    daemon_in: str = ""
+    daemon_cat_rc: int = 0
+    mount_rc: int = 0
+    fail_systemctl: str = ""
+    status_rc: int = 0
+    term_on: str = ""
+    metadata: bool = True
+    docker_root: str = "/mnt/docker-host"
+    docker_root_dev: str = "2049"
+    min_free_gb: str = MIN_FREE_GB
+    extra: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class Outcome:
+    """What a relocation run left behind.
+
+    Args:
+        done: the finished script.
+        calls: every privileged call in order.
+        daemon: the daemon.json on disk afterwards, None when there is none
+            or it is no valid JSON.
+        daemon_raw: the daemon.json bytes on disk afterwards, None when there is none.
+        bound: whether the containerd bind mount is still in place.
+    """
+
+    done: subprocess.CompletedProcess
+    calls: list[str]
+    daemon: dict | None
+    daemon_raw: str | None
+    bound: bool
+
+
+def relocate(runner: Runner) -> Outcome:
+    """Run relocate_docker_dataroot.sh against a faked runner.
+
+    Args:
+        runner: the faked services, disks and files.
+
+    Returns:
+        The outcome of the run.
+    """
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        runner_dir = root / "runner"
+        bin_dir = root / "bin"
+        stub_bin = root / "privileged"
+        state = root / "state"
+        for directory in (runner_dir, bin_dir, stub_bin, state):
+            directory.mkdir()
+        (runner_dir / "relocate_docker_dataroot.sh").symlink_to(SCRIPT)
+        stubs = [(runner_dir / "dump_state.sh", "#!/usr/bin/env bash\nexit 0\n")]
+        stubs += [(bin_dir / "sudo", SUDO_STUB)]
+        stubs += [(bin_dir / name, UNPRIVILEGED_STUB) for name in DESTRUCTIVE]
+        stubs += [(bin_dir / name, body) for name, body in UNPRIVILEGED.items()]
+        stubs += [(stub_bin / name, body) for name, body in PRIVILEGED.items()]
+        for path, body in stubs:
+            path.write_text(body)
+            path.chmod(0o755)
+        log = root / "calls.log"
+        log.touch()
+        daemon_out = root / "daemon.json"
+        if runner.daemon_exists:
+            daemon_out.write_text(runner.daemon_in)
+        env = {
+            k: v
+            for k, v in os.environ.items()
+            if k
+            not in ("BASH_ENV", "ACT", "RUNNER_ENVIRONMENT", "SKIP_DOCKER_RELOCATE")
+        }
+        env.update(
+            PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
+            STUB_LOG=str(log),
+            STUB_BIN=str(stub_bin),
+            STUB_STATE=str(state),
+            STUB_MNT_MOUNTED="1" if runner.mounted else "0",
+            STUB_MNT_FREE=str(runner.mnt_free),
+            STUB_ROOT_FREE=str(runner.root_free),
+            STUB_CONTAINERD_DUMP=runner.dump,
+            STUB_CONTAINERD_UNIT_RC=str(runner.containerd_unit_rc),
+            STUB_CONTAINERS=runner.containers,
+            STUB_IMAGES=runner.images,
+            STUB_VOLUMES=runner.volumes,
+            STUB_BUILD_CACHE=runner.build_cache,
+            STUB_NAMESPACES=runner.namespaces,
+            STUB_FOREIGN_NS=runner.foreign_ns,
+            STUB_FOREIGN_CONTAINERS=runner.foreign_containers,
+            STUB_FOREIGN_TASKS=runner.foreign_tasks,
+            STUB_SHIMS=runner.shims,
+            STUB_SHIM_RC=str(runner.shim_rc),
+            STUB_SUDO_RC=str(runner.sudo_rc),
+            STUB_DAEMON_EXISTS="1" if runner.daemon_exists else "0",
+            STUB_DAEMON_IN=runner.daemon_in,
+            STUB_DAEMON_CAT_RC=str(runner.daemon_cat_rc),
+            STUB_DAEMON_OUT=str(daemon_out),
+            STUB_MOUNT_RC=str(runner.mount_rc),
+            STUB_FAIL_SYSTEMCTL=runner.fail_systemctl,
+            STUB_STATUS_RC=str(runner.status_rc),
+            STUB_TERM_ON=runner.term_on,
+            STUB_METADATA="1" if runner.metadata else "0",
+            STUB_DOCKER_ROOT=runner.docker_root,
+            STUB_DOCKER_ROOT_DEV=runner.docker_root_dev,
+            **runner.extra,
+        )
+        optional = {
+            "RUNNER_ENVIRONMENT": runner.runner_environment,
+            "ACT": runner.act,
+            "SKIP_DOCKER_RELOCATE": runner.skip,
+        }
+        env.update({k: v for k, v in optional.items() if v is not None})
+        done = subprocess.run(
+            [
+                "bash",
+                str(runner_dir / "relocate_docker_dataroot.sh"),
+                runner.min_free_gb,
+            ],
+            env=env,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        calls = log.read_text().splitlines()  # nocheck: cache-read  stub call log
+        content = (
+            daemon_out.read_text()  # nocheck: cache-read  stub output
+            if daemon_out.exists()
+            else None
+        )
+        try:
+            daemon = json.loads(content) if content else None
+        except json.JSONDecodeError:
+            daemon = None
+        return Outcome(
+            done=done,
+            calls=calls,
+            daemon=daemon,
+            daemon_raw=content,
+            bound=(state / "bound").exists(),
+        )
+
+
+class TestRelocation(unittest.TestCase):
+    def test_small_shape_moves_containerd_and_docker_onto_the_spare_disk(self) -> None:
+        out = relocate(Runner())
+        self.assertEqual(out.done.returncode, 0, out.done.stderr)
+        stop, bind = out.calls.index(STOP), out.calls.index(BIND)
+        start = out.calls.index("systemctl start containerd.service")
+        self.assertLess(stop, bind)
+        self.assertLess(bind, start)
+        self.assertIn("systemctl start docker.socket docker.service", out.calls)
+        self.assertIn(
+            "test -f /mnt/containerd/io.containerd.metadata.v1.bolt/meta.db", out.calls
+        )
+        self.assertEqual(out.daemon, {"data-root": "/mnt/docker-host"})
+        self.assertTrue(out.bound)
+
+    def test_containerd_1x_dump_with_double_quotes_is_understood(self) -> None:
+        out = relocate(Runner(dump=DUMP_V1))
+        self.assertEqual(out.done.returncode, 0, out.done.stderr)
+        self.assertIn(BIND, out.calls)
+
+    def test_existing_daemon_json_keeps_its_settings(self) -> None:
+        out = relocate(Runner(daemon_exists=True, daemon_in='{"debug": true}'))
+        self.assertEqual(out.done.returncode, 0, out.done.stderr)
+        self.assertEqual(out.daemon, {"debug": True, "data-root": "/mnt/docker-host"})
+
+
+class TestDeclines(unittest.TestCase):
+    def assert_declined(self, out: Outcome, reason: str) -> None:
+        """Assert a visible no-op: exit 0, a notice with the reason, no call.
+
+        Args:
+            out: the outcome of the run.
+            reason: text the notice must carry.
+        """
+        self.assertEqual(out.done.returncode, 0, out.done.stderr)
+        self.assertIn("::notice::", out.done.stdout)
+        self.assertIn(reason, out.done.stdout)
+        self.assertEqual(out.calls, [])
+
+    def test_large_shape_without_a_mnt_mount(self) -> None:
+        self.assert_declined(
+            relocate(Runner(mounted=False)), "/mnt is not a separate mount"
+        )
+
+    def test_self_hosted_runner(self) -> None:
+        self.assert_declined(
+            relocate(Runner(runner_environment="self-hosted")),
+            "not a GitHub-hosted runner",
+        )
+
+    def test_unknown_runner_environment(self) -> None:
+        self.assert_declined(
+            relocate(Runner(runner_environment=None)), "not a GitHub-hosted runner"
+        )
+
+    def test_mnt_below_the_floor(self) -> None:
+        self.assert_declined(relocate(Runner(mnt_free=60, root_free=20)), "below 75G")
+
+    def test_mnt_not_roomier_than_root(self) -> None:
+        self.assert_declined(
+            relocate(Runner(mnt_free=80, root_free=90)), "not more than"
+        )
+
+    def test_skip_toggle(self) -> None:
+        self.assert_declined(relocate(Runner(skip="1")), "SKIP_DOCKER_RELOCATE=1")
+
+
+class TestRefusalsBeforeTheStop(unittest.TestCase):
+    def assert_refused(self, out: Outcome, reason: str) -> None:
+        """Assert a loud refusal before anything was stopped or written.
+
+        Args:
+            out: the outcome of the run.
+            reason: text the error must carry.
+        """
+        self.assertEqual(out.done.returncode, 1)
+        self.assertIn(reason, out.done.stderr)
+        self.assertNotIn(STOP, out.calls)
+        self.assertFalse([c for c in out.calls if c.startswith(("mount ", "install "))])
+        self.assertFalse(out.bound)
+
+    def test_act_even_with_a_faked_github_runner(self) -> None:
+        for act in ("true", "1"):
+            with self.subTest(act=act):
+                self.assert_refused(
+                    relocate(Runner(act=act)), "refuses to run under act"
+                )
+
+    def test_non_numeric_floor(self) -> None:
+        self.assert_refused(relocate(Runner(min_free_gb="forty")), "positive integer")
+
+    def test_missing_containerd_unit(self) -> None:
+        self.assert_refused(
+            relocate(Runner(containerd_unit_rc=1)), "containerd.service is missing"
+        )
+
+    def test_containerd_root_elsewhere(self) -> None:
+        self.assert_refused(
+            relocate(Runner(dump="version = 3\nroot = '/data/containerd'\n")),
+            "/data/containerd",
+        )
+
+    def test_existing_containers(self) -> None:
+        self.assert_refused(relocate(Runner(containers="abc123\n")), "abc123")
+
+    def test_existing_images(self) -> None:
+        self.assert_refused(relocate(Runner(images="sha256feed\n")), "sha256feed")
+
+    def test_existing_volumes(self) -> None:
+        self.assert_refused(
+            relocate(Runner(volumes="core_infinito\n")), "core_infinito"
+        )
+
+    def test_existing_build_cache(self) -> None:
+        self.assert_refused(relocate(Runner(build_cache="3")), "build cache")
+
+    def test_containers_in_another_namespace(self) -> None:
+        out = relocate(
+            Runner(namespaces="moby\nk8s.io\n", foreign_containers="pause1\n")
+        )
+        self.assert_refused(out, "k8s.io holds containers: pause1")
+
+    def test_tasks_in_another_namespace(self) -> None:
+        out = relocate(Runner(namespaces="moby\nk8s.io\n", foreign_tasks="task7\n"))
+        self.assert_refused(out, "k8s.io runs tasks: task7")
+
+    def test_running_shims(self) -> None:
+        out = relocate(Runner(shims="4242 containerd-shim-runc-v2 -id x\n", shim_rc=0))
+        self.assert_refused(out, "containerd shims are running")
+
+    def test_shim_listing_error(self) -> None:
+        self.assert_refused(
+            relocate(Runner(shim_rc=2)), "could not list containerd shims"
+        )
+
+    def test_broken_sudo(self) -> None:
+        out = relocate(Runner(sudo_rc=1))
+        self.assertEqual(out.done.returncode, 1)
+        self.assertNotIn(STOP, out.calls)
+
+    def test_unreadable_daemon_json_is_never_overwritten(self) -> None:
+        out = relocate(Runner(daemon_exists=True, daemon_in="{}", daemon_cat_rc=1))
+        self.assertEqual(out.done.returncode, 1)
+        self.assertNotIn(STOP, out.calls)
+        self.assertFalse([c for c in out.calls if c.startswith("install ")])
+        self.assertEqual(out.daemon, {})
+
+    def test_invalid_daemon_json_is_never_overwritten(self) -> None:
+        out = relocate(Runner(daemon_exists=True, daemon_in="{not json"))
+        self.assertEqual(out.done.returncode, 1)
+        self.assertIn("is not valid JSON", out.done.stderr)
+        self.assertNotIn(STOP, out.calls)
+        self.assertFalse([c for c in out.calls if c.startswith("install ")])
+        self.assertEqual(out.daemon_raw, "{not json")
+
+
+class TestRollback(unittest.TestCase):
+    def assert_rolled_back(self, out: Outcome, reason: str) -> None:
+        """Assert the rollback: no bind left, daemons restarted last, exit 1.
+
+        Args:
+            out: the outcome of the run.
+            reason: text the rollback error must carry.
+        """
+        self.assertEqual(out.done.returncode, 1, out.done.stderr)
+        self.assertIn("rolling Docker and containerd back onto /", out.done.stderr)
+        self.assertIn(reason, out.done.stderr)
+        self.assertFalse(out.bound)
+        self.assertIn(RESTART, out.calls)
+        self.assertTrue([c for c in out.calls if c.startswith("journalctl ")])
+
+    def test_failed_bind_mount_removes_the_daemon_json_it_never_had(self) -> None:
+        out = relocate(Runner(mount_rc=32))
+        self.assert_rolled_back(out, "mount --bind")
+        self.assertIsNone(out.daemon)
+
+    def test_failed_containerd_start_unbinds_before_the_restart(self) -> None:
+        out = relocate(
+            Runner(
+                daemon_exists=True,
+                daemon_in='{"debug": true}',
+                fail_systemctl="start containerd.service",
+            )
+        )
+        self.assert_rolled_back(out, "systemctl start containerd.service")
+        self.assertLess(out.calls.index(UNBIND), len(out.calls) - 1)
+        self.assertLess(out.calls.index(UNBIND), out.calls.index(RESTART))
+        self.assertEqual(out.daemon, {"debug": True})
+
+    def test_cancel_during_the_move_rolls_back(self) -> None:
+        out = relocate(Runner(term_on="mkdir"))
+        self.assert_rolled_back(out, "interrupted")
+
+    def test_down_units_still_yield_the_journal(self) -> None:
+        out = relocate(Runner(mount_rc=32, status_rc=3))
+        self.assert_rolled_back(out, "mount --bind")
+
+    def test_containerd_without_metadata_on_the_spare_disk(self) -> None:
+        out = relocate(Runner(metadata=False))
+        self.assert_rolled_back(out, "without its metadata")
+        self.assertIn(UNBIND, out.calls)
+        self.assertIsNone(out.daemon)
+
+    def test_docker_root_elsewhere(self) -> None:
+        out = relocate(Runner(docker_root="/var/lib/docker"))
+        self.assert_rolled_back(out, "expected '/mnt/docker-host'")
+
+    def test_docker_root_off_the_spare_disk(self) -> None:
+        out = relocate(Runner(docker_root_dev="66305"))
+        self.assert_rolled_back(out, "not on the /mnt filesystem")
+
+
+if __name__ == "__main__":
+    unittest.main()