From 2d7bf740b0c21e9882a1a3ec23e54bcb0c18dba9 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sun, 1 Nov 2026 12:28:12 +0100 Subject: [PATCH] fix(ci): put containerd, Docker and swap on the runner's empty spare disk The small GitHub runner shape (AWS, 72G root) ran out of disk in swarm and compose rows (#1329: WordPress#2 of run 41261531965, Nextcloud#6 and #7 of run 41262286962) while an empty instance-store disk nvme1n1 sat unused. mount_spare_disk.sh formats and mounts it at /mnt, but only on a GitHub-hosted runner outside act, and only a whole unpartitioned disk without any signature, mount or swap use of at least 75G; every decline leaves a notice. relocate_docker_dataroot.sh then bind-mounts /var/lib/containerd from the spare disk and moves the Docker data-root to /mnt/docker-host (not /mnt/docker, which is the nested daemon's volume that dedicated.sh wipes). It refuses before stopping anything when images, volumes, build cache, foreign containerd namespaces or shims exist or daemon.json is unreadable, and rolls back the bind mount and daemon.json on any later failure or cancel. Swap goes on / once containerd lives on /mnt, dump_state.sh logs disk size, model and free space per storage root, and the swarm watchdog samples / and /var/lib/containerd every 10 s. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/call-test-deploy.yml | 15 +- .../artefact/files/github/workflow.md | 30 +- scripts/github/runner/dump_state.sh | 18 +- scripts/github/runner/enlarge_swap.sh | 14 +- scripts/github/runner/mount_spare_disk.sh | 122 ++++ .../github/runner/relocate_docker_dataroot.sh | 217 +++++--- .../utils/filesystem/docker_dataroot.sh | 6 +- .../python/scripts/github/runner/__init__.py | 0 .../runner/test_enlarge_swap_placement.py | 135 +++++ .../github/runner/test_mount_spare_disk.py | 457 +++++++++++++++ .../runner/test_relocate_docker_dataroot.py | 524 ++++++++++++++++++ .../tests/swarm/test_run_storage_roots.py | 110 ++++ utils/tests/swarm/run.py | 26 +- 13 files changed, 1585 insertions(+), 89 deletions(-) create mode 100755 scripts/github/runner/mount_spare_disk.sh create mode 100644 tests/unit/python/scripts/github/runner/__init__.py create mode 100644 tests/unit/python/scripts/github/runner/test_enlarge_swap_placement.py create mode 100644 tests/unit/python/scripts/github/runner/test_mount_spare_disk.py create mode 100644 tests/unit/python/scripts/github/runner/test_relocate_docker_dataroot.py create mode 100644 tests/unit/python/utils/tests/swarm/test_run_storage_roots.py diff --git a/.github/workflows/call-test-deploy.yml b/.github/workflows/call-test-deploy.yml index dcdf313fdc..d20d798d3c 100644 --- a/.github/workflows/call-test-deploy.yml +++ b/.github/workflows/call-test-deploy.yml @@ -298,16 +298,21 @@ jobs: shell: bash run: grep '^INFINITO_RESCUE_DIAGNOSTICS_BASE=' default.env >> "$GITHUB_ENV" + - name: 💽 Mount the empty spare disk at /mnt + if: ${{ !env.ACT }} + shell: bash + run: ./scripts/github/runner/mount_spare_disk.sh /mnt 75 + + - name: 📦 Relocate containerd and the Docker data-root onto /mnt when roomier + if: ${{ !env.ACT }} + shell: bash + run: ./scripts/github/runner/relocate_docker_dataroot.sh 75 + - name: 💾 Enlarge swap space if: ${{ !env.ACT }} shell: bash run: ./scripts/github/runner/enlarge_swap.sh - - name: 📦 Relocate Docker data-root onto /mnt when roomier - if: ${{ !env.ACT }} - shell: bash - run: ./scripts/github/runner/relocate_docker_dataroot.sh - - name: 🛡️ Tear down host AppArmor profiles (dev-host parity) if: ${{ !env.ACT }} shell: bash diff --git a/docs/contributing/artefact/files/github/workflow.md b/docs/contributing/artefact/files/github/workflow.md index 383e8e7961..a186b9c65c 100644 --- a/docs/contributing/artefact/files/github/workflow.md +++ b/docs/contributing/artefact/files/github/workflow.md @@ -78,11 +78,39 @@ Deploy test workflows use the `jlumbroso/free-disk-space` action to reclaim runn Set an option back to `false` only when a new host-side step in the same workflow genuinely needs the removed payload. +## Spare disk 💽 + +Some GitHub-hosted runners have a 72 GB root disk plus a second, unformatted NVMe disk. [mount_spare_disk.sh](../../../../../scripts/github/runner/mount_spare_disk.sh) formats that disk as ext4 and mounts it at `/mnt`. The nested Docker volume (`/mnt/docker`) and the loop image of [docker_dataroot.sh](../../../../../scripts/tests/deploy/utils/filesystem/docker_dataroot.sh) (`/mnt/docker-fs.img`) then land on it. + +When `/mnt` has at least `MIN_FREE_GB` free and more than `/`, [relocate_docker_dataroot.sh](../../../../../scripts/github/runner/relocate_docker_dataroot.sh) stops Docker and containerd, bind-mounts `/mnt/containerd` over `/var/lib/containerd` (images, container layers, the swarm node containers), sets the Docker data-root to `/mnt/docker-host` (named volumes), restarts both and verifies that containerd's metadata and the Docker root sit on `/mnt`. Before the stop it fails if Docker already holds containers, images, volumes or build cache, if another containerd namespace holds containers or tasks, if containerd shims run, or if `daemon.json` cannot be read as JSON. Any failure after the stop, and a cancel, unmounts the bind, restores `daemon.json`, restarts both daemons and fails the step. It refuses under act and changes nothing on a runner that is not GitHub-hosted. Every decline prints a `::notice::` annotation. [run.py](../../../../../utils/tests/swarm/run.py) then watches the free space of both `/` and `/var/lib/containerd`. + +Step order: spare disk, relocation, swap. With containerd on `/mnt`, the swapfile goes to `/`. + +The script formats a disk only when all of these hold; otherwise the runner stays as it is: + +- `ACT` is unset (the script fails under act) and `RUNNER_ENVIRONMENT` is `github-hosted`. +- `/mnt` is an existing, empty directory and no mount. +- The disk is a whole disk other than the root disk, with no partitions or holders, no filesystem or partition-table signature (`lsblk`, `blkid -p` and `wipefs`), no mount or swap use, not read-only, and at least `MIN_SIZE_GB` large. + +Workflow step invocations, in this order and gated alike: + +```yaml +- name: Mount the empty spare disk at /mnt + if: ${{ !env.ACT }} + shell: bash + run: ./scripts/github/runner/mount_spare_disk.sh /mnt 75 + +- name: Relocate containerd and the Docker data-root onto /mnt when roomier + if: ${{ !env.ACT }} + shell: bash + run: ./scripts/github/runner/relocate_docker_dataroot.sh 75 +``` + ## Swap 💾 Deploy test workflows enlarge host swap via [enlarge_swap.sh](../../../../../scripts/github/runner/enlarge_swap.sh) to absorb transient memory spikes (e.g. PeerTube plugin install [#162](https://github.com/infinito-nexus/core/issues/162)) that would otherwise trip the host OOM-killer on the 16 GB GitHub-hosted runner. -The script **prefers `/mnt`** when it is a separate partition (classic `ubuntu-latest` layout with `/dev/sdb` mounted at `/mnt`) and falls back to `/` otherwise. Current public runners no longer expose a separate `/mnt`, so the swapfile lands on `/` and must not starve the root filesystem. +The script puts the swapfile on `/` when containerd already lives on `/mnt`, **prefers `/mnt`** otherwise when it is a separate partition (classic `ubuntu-latest` layout with `/dev/sdb` mounted at `/mnt`), and falls back to `/`. Current public runners no longer expose a separate `/mnt`, so the swapfile lands on `/` and must not starve the root filesystem. Swap size is a **fixed constant**, not a "free space minus buffer" calculation. An earlier version took ~99 GB on a 145 GB root partition, leaving so little headroom that later Docker layer writes (Playwright test image pull) failed with `no space left on device`. A fixed cap is large enough to absorb the peertube spike and small enough that Docker layer writes keep succeeding throughout the 30+ min deploy. diff --git a/scripts/github/runner/dump_state.sh b/scripts/github/runner/dump_state.sh index 93f41a74ac..b8c9ce617f 100755 --- a/scripts/github/runner/dump_state.sh +++ b/scripts/github/runner/dump_state.sh @@ -18,14 +18,24 @@ date -u +'%Y-%m-%dT%H:%M:%SZ' echo "--- disk (bytes) ---" df -h / /mnt 2>/dev/null || df -h / +echo "--- disk per storage root (/, /mnt, containerd root, Docker data root) ---" +docker_root="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null)" || docker_root="" +df -h --output=source,size,used,avail,target / /mnt /var/lib/containerd ${docker_root:+"${docker_root}"} || + echo "df could not read every storage root" + +containerd --version 2>/dev/null || echo "containerd not on PATH" + echo "--- disk (inodes) ---" df -hi / /mnt 2>/dev/null || df -hi / -echo "--- block devices ---" -lsblk -f || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error +echo "--- disks (bytes, model: Amazon EC2 NVMe Instance Storage versus Amazon Elastic Block Store) ---" +lsblk -d -b -o NAME,SIZE,MODEL,ROTA,TYPE || echo "lsblk could not list the disks" -echo "--- mounts (/, /mnt, /var/lib/docker, /mnt/docker) ---" -mount | grep -E ' / | /mnt | /var/lib/docker | /mnt/docker ' || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error +echo "--- block devices ---" +lsblk -o NAME,TYPE,SIZE,FSTYPE,FSVER,LABEL,UUID,FSAVAIL,FSUSE%,MOUNTPOINTS || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error + +echo "--- mounts (/, /mnt, /var/lib/docker, /var/lib/containerd, /mnt/docker) ---" +mount | grep -E ' / | /mnt | /var/lib/docker | /var/lib/containerd | /mnt/docker ' || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error echo "--- docker ---" docker info 2>/dev/null | diff --git a/scripts/github/runner/enlarge_swap.sh b/scripts/github/runner/enlarge_swap.sh index 48b56e1473..f61a347c36 100755 --- a/scripts/github/runner/enlarge_swap.sh +++ b/scripts/github/runner/enlarge_swap.sh @@ -4,7 +4,8 @@ # spikes (e.g. peertube plugin install, issue #162) don't trip the host # OOM-killer, WITHOUT starving the rest of the job of disk space. # -# Prefers /mnt when it is a separate partition (classic ubuntu-latest +# Uses / when containerd already lives on /mnt (relocate_docker_dataroot.sh +# runs first), otherwise prefers /mnt when it is a separate partition (classic ubuntu-latest # layout with /dev/sdb mounted at /mnt). On current public runners # there is no separate /mnt partition — everything lives on /, so we # place the swapfile there. The swap size is a hard constant, never a @@ -46,7 +47,16 @@ root_free=$(df --output=avail -B1G / | tail -n1 | tr -d ' ') echo "Target swap: ${SWAP_GB}G. Free: / ${root_free}G, /mnt ${mnt_free}G (separate mount: $(mountpoint -q /mnt 2>/dev/null && echo yes || echo no))" -if [ "$mnt_free" -ge "$MIN_FREE_GB" ]; then +storage_on_mnt=no +if mountpoint -q /mnt 2>/dev/null && containerd_dev="$(stat -c %d /var/lib/containerd 2>/dev/null)" && + [ "$containerd_dev" = "$(stat -c %d /mnt)" ]; then + storage_on_mnt=yes +fi + +if [ "$storage_on_mnt" = yes ] && [ "$root_free" -ge "$MIN_FREE_GB" ]; then + target=/swapfile + chosen="/ (containerd and Docker live on /mnt)" +elif [ "$mnt_free" -ge "$MIN_FREE_GB" ]; then target=/mnt/swapfile chosen="/mnt (separate partition)" elif [ "$root_free" -ge "$MIN_FREE_GB" ]; then diff --git a/scripts/github/runner/mount_spare_disk.sh b/scripts/github/runner/mount_spare_disk.sh new file mode 100755 index 0000000000..0d52e941ba --- /dev/null +++ b/scripts/github/runner/mount_spare_disk.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Usage: mount_spare_disk.sh MOUNT_POINT MIN_SIZE_GB +# MOUNT_POINT where the spare disk lands; enlarge_swap.sh and +# relocate_docker_dataroot.sh pick it up when it is /mnt +# MIN_SIZE_GB smallest spare disk worth mounting; smaller disks leave the runner as it is +set -euo pipefail + +MOUNT_POINT="${1:?MOUNT_POINT is required - state where the spare disk is mounted at the call site}" +MIN_SIZE_GB="${2:?MIN_SIZE_GB is required - state the smallest spare disk in GB worth mounting at the call site}" + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DUMP="${HERE}/dump_state.sh" + +if [ -n "${ACT:-}" ]; then + echo "::error::mount_spare_disk.sh refuses to run under act (ACT=${ACT}): it formats block devices and must never touch the operator's machine" >&2 + exit 1 +fi + +if ! [[ "${MIN_SIZE_GB}" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::MIN_SIZE_GB must be a positive integer, got '${MIN_SIZE_GB}'" >&2 + exit 1 +fi + +if [ "${RUNNER_ENVIRONMENT:-}" != github-hosted ]; then + echo "::notice::mount_spare_disk.sh mounted no spare disk: not a GitHub-hosted runner (RUNNER_ENVIRONMENT='${RUNNER_ENVIRONMENT:-}'), every block device stays untouched" + exit 0 +fi + +if mountpoint -q "${MOUNT_POINT}"; then + echo "::notice::mount_spare_disk.sh mounted no spare disk: ${MOUNT_POINT} is already a separate mount" + exit 0 +fi + +if [ ! -d "${MOUNT_POINT}" ]; then + echo "::error::mount point ${MOUNT_POINT} does not exist" >&2 + exit 1 +fi + +occupant="$(find "${MOUNT_POINT}" -mindepth 1 -maxdepth 1 -print -quit)" +if [ -n "${occupant}" ]; then + echo "::warning::${MOUNT_POINT} is not empty; mounting over it would hide its content, so no spare disk is mounted" + exit 0 +fi + +min_bytes=$((MIN_SIZE_GB * 1024 * 1024 * 1024)) +tree="$(lsblk -J -b -o NAME,TYPE,SIZE,RO,FSTYPE,PTTYPE,MOUNTPOINTS)" + +if ! jq -e ' + [.. | objects | select(has("name"))] + | all( + ((.mountpoints | type) == "array") + and ((.ro | type) == "boolean" or .ro == "0" or .ro == "1") + )' <<<"${tree}" >/dev/null; then + echo "::error::lsblk did not report mountpoints and ro for every device; refusing to pick a spare disk" >&2 + exit 1 +fi + +root_disks="$(jq -c '[ + .blockdevices[] + | select([.. | objects | .mountpoints[] | select(. == "/")] | length > 0) + | .name +]' <<<"${tree}")" +if [ "${root_disks}" = "[]" ]; then + echo "::error::no block device carries / according to lsblk; refusing to pick a spare disk" >&2 + exit 1 +fi + +disk="$(jq -r --argjson roots "${root_disks}" --argjson min "${min_bytes}" ' + [ + .blockdevices[] + | select(.type == "disk") + | select(.name | IN($roots[]) | not) + | select(.name | test("^(nvme[0-9]+n[0-9]+|sd[a-z]+|vd[a-z]+|xvd[a-z]+)$")) + | select(.ro == false or .ro == "0") + | select((.children // []) | length == 0) + | select((.fstype // "") == "" and (.pttype // "") == "") + | select([.mountpoints[] | select(. != null)] | length == 0) + | select((.size | tonumber) >= $min) + ] + | max_by(.size | tonumber) + | .name // empty' <<<"${tree}")" + +if [ -z "${disk}" ]; then + echo "::notice::mount_spare_disk.sh mounted no spare disk: No empty, unpartitioned, unmounted disk of at least ${MIN_SIZE_GB}G besides the root disk ${root_disks}" + exit 0 +fi + +probe_rc=0 +probe="$(sudo blkid -p "/dev/${disk}" 2>&1)" || probe_rc=$? +if [ "${probe_rc}" = 2 ] && [ -n "${probe}" ]; then + echo "::error::blkid -p /dev/${disk} found nothing but complained: ${probe}" >&2 + exit 1 +fi +case "${probe_rc}" in +2) ;; +0) + echo "::notice::mount_spare_disk.sh mounted no spare disk: /dev/${disk} carries a signature (${probe}), leaving it untouched" + exit 0 + ;; +*) + echo "::error::blkid -p /dev/${disk} could not tell whether the disk is empty (rc=${probe_rc}): ${probe}" >&2 + exit 1 + ;; +esac + +signatures="$(sudo wipefs --no-act --noheadings "/dev/${disk}")" +if [ -n "${signatures}" ]; then + echo "::notice::mount_spare_disk.sh mounted no spare disk: /dev/${disk} carries signatures blkid missed (${signatures}), leaving it untouched" + exit 0 +fi + +echo "Formatting the empty spare disk /dev/${disk} and mounting it at ${MOUNT_POINT}" +sudo mkfs.ext4 -q -m 0 -L infinito-spare "/dev/${disk}" +sudo mount "/dev/${disk}" "${MOUNT_POINT}" + +source_dev="$(findmnt -n -o SOURCE "${MOUNT_POINT}")" +if [ "${source_dev}" != "/dev/${disk}" ]; then + echo "::error::${MOUNT_POINT} is backed by '${source_dev}' after mounting /dev/${disk}" >&2 + exit 1 +fi + +"$DUMP" "after spare-disk mount -> /dev/${disk} at ${MOUNT_POINT}" diff --git a/scripts/github/runner/relocate_docker_dataroot.sh b/scripts/github/runner/relocate_docker_dataroot.sh index d83a7638c9..1296fcc5e7 100755 --- a/scripts/github/runner/relocate_docker_dataroot.sh +++ b/scripts/github/runner/relocate_docker_dataroot.sh @@ -1,98 +1,173 @@ #!/usr/bin/env bash -# Move the host Docker data-root onto /mnt when it is a separate, roomier -# partition. Image/volume storage (swarm's nested DinD-node daemons, the -# DR-drill LUKS loop image, per-node registry pulls) otherwise piles onto -# the ~50 GB root fs and trips the <6 GB abort guard in -# utils/tests/swarm/matrix.py. On public runners without a separate /mnt -# this is a no-op and Docker stays on /. -# -# Best-effort: relocation is an optimization, never a requirement. Any -# failure rolls Docker back onto / and the step still exits 0, so a hiccup -# here can never turn an otherwise-green deploy red. Runs BEFORE any Docker -# usage; the free-disk-space step already pruned the preinstalled images, so -# /var/lib/docker is ~empty and nothing is copied. -# -# Usage: relocate_docker_dataroot.sh [min-free-gb] -# Default floor: 40 GB (swarm peak footprint; below it /mnt buys nothing). -# -# Debug toggle: SKIP_DOCKER_RELOCATE=1 leaves the data-root on /. +# Usage: relocate_docker_dataroot.sh MIN_FREE_GB +# MIN_FREE_GB smallest free space on /mnt that makes the move worth it +# SKIP_DOCKER_RELOCATE=1 leave Docker and containerd on / set -euo pipefail +MIN_FREE_GB="${1:?MIN_FREE_GB is required - state the smallest free GB on /mnt worth relocating to at the call site}" + HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DUMP="${HERE}/dump_state.sh" +DATA_ROOT=/mnt/docker-host +CONTAINERD_ROOT=/var/lib/containerd +CONTAINERD_TARGET=/mnt/containerd +DAEMON_JSON=/etc/docker/daemon.json + +# Param: $1 why Docker and containerd stay on / +decline() { + echo "::notice::relocate_docker_dataroot.sh left Docker and containerd on /: $1" + "$DUMP" "docker-relocate skipped ($1)" + exit 0 +} + +# Param: $1 what was found that the relocation would lose or orphan +refuse() { + echo "::error::relocate_docker_dataroot.sh refuses to move Docker and containerd: $1" >&2 + exit 1 +} + if [ "${SKIP_DOCKER_RELOCATE:-0}" = "1" ]; then - echo "SKIP_DOCKER_RELOCATE=1 — leaving Docker data-root on /." - "$DUMP" "docker-relocate skipped (SKIP_DOCKER_RELOCATE=1)" + decline "SKIP_DOCKER_RELOCATE=1" +fi + +if [ -n "${ACT:-}" ]; then + echo "::error::relocate_docker_dataroot.sh refuses to run under act (ACT=${ACT}): it restarts Docker and containerd and must never touch the operator's machine" >&2 + exit 1 +fi + +if ! [[ "${MIN_FREE_GB}" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::MIN_FREE_GB must be a positive integer, got '${MIN_FREE_GB}'" >&2 + exit 1 +fi + +if [ "${RUNNER_ENVIRONMENT:-}" != github-hosted ]; then + echo "::notice::relocate_docker_dataroot.sh left Docker and containerd where they are: not a GitHub-hosted runner (RUNNER_ENVIRONMENT='${RUNNER_ENVIRONMENT:-}')" exit 0 fi -TARGET_DIR=/mnt/docker -MIN_FREE_GB="${1:-40}" - -if ! mountpoint -q /mnt 2>/dev/null; then - echo "/mnt is not a separate mount — leaving Docker data-root on /." - "$DUMP" "docker-relocate skipped (no separate /mnt)" - exit 0 +if ! mountpoint -q /mnt; then + decline "/mnt is not a separate mount" fi mnt_free=$(df --output=avail -B1G /mnt | tail -n1 | tr -d ' ') root_free=$(df --output=avail -B1G / | tail -n1 | tr -d ' ') echo "Free: / ${root_free}G, /mnt ${mnt_free}G; need >= ${MIN_FREE_GB}G on /mnt and roomier than /" -if [ "$mnt_free" -lt "$MIN_FREE_GB" ] || [ "$mnt_free" -le "$root_free" ]; then - echo "/mnt below floor or not roomier than / — leaving Docker data-root on /." - "$DUMP" "docker-relocate skipped (/mnt not roomier)" - exit 0 +if [ "$mnt_free" -lt "$MIN_FREE_GB" ]; then + decline "/mnt has ${mnt_free}G free, below ${MIN_FREE_GB}G" +fi +if [ "$mnt_free" -le "$root_free" ]; then + decline "/mnt has ${mnt_free}G free, not more than the ${root_free}G on /" fi -backup="" -if [ -s /etc/docker/daemon.json ]; then - backup="$(mktemp)" - cp /etc/docker/daemon.json "$backup" +if ! systemctl cat containerd.service >/dev/null; then + refuse "containerd.service is missing" fi -restore_and_continue() { - echo "WARN: Docker data-root relocation failed — keeping Docker on /." >&2 - if [ -n "$backup" ]; then - sudo install -m 0644 "$backup" /etc/docker/daemon.json || true # nocheck: shell-or-true -- rollback must reach the docker restart even if the restore fails - else - sudo rm -f /etc/docker/daemon.json || true # nocheck: shell-or-true -- rollback must reach the docker restart even if the cleanup fails - fi - sudo systemctl start docker.service || sudo systemctl restart docker.service || true # nocheck: shell-or-true -- last-ditch daemon start; rollback still exits 0 by design - "$DUMP" "docker-relocate rolled back (kept on /)" - exit 0 -} - -echo "Relocating Docker data-root to ${TARGET_DIR}" -sudo systemctl stop docker.service docker.socket || true # nocheck: shell-or-true -- inactive/missing units may exit non-zero; a wedged daemon is caught by the post-start root-dir check -sudo mkdir -p "$TARGET_DIR" /etc/docker || restore_and_continue - -tmp="$(mktemp)" -if [ -n "$backup" ]; then - jq --arg dr "$TARGET_DIR" '. + {"data-root": $dr}' "$backup" >"$tmp" || restore_and_continue -else - printf '{"data-root": "%s"}\n' "$TARGET_DIR" >"$tmp" +configured_root="$(sudo containerd config dump | sed -nE "s/^root = ['\"](.*)['\"]\$/\1/p")" +if [ "${configured_root}" != "${CONTAINERD_ROOT}" ]; then + refuse "containerd root is '${configured_root}', expected ${CONTAINERD_ROOT}" fi -sudo install -m 0644 "$tmp" /etc/docker/daemon.json || restore_and_continue -rm -f "$tmp" -sudo systemctl start docker.service || restore_and_continue +containers="$(docker ps -aq)" +[ -z "${containers}" ] || refuse "Docker containers exist: ${containers//$'\n'/ }" +images="$(docker images -q)" +[ -z "${images}" ] || refuse "Docker images exist: ${images//$'\n'/ }" +volumes="$(docker volume ls -q)" +[ -z "${volumes}" ] || refuse "Docker volumes exist: ${volumes//$'\n'/ }" +build_cache="$(docker system df --format '{{.Type}}={{.TotalCount}}' | sed -n 's/^Build Cache=//p')" +[ "${build_cache}" = 0 ] || refuse "the build cache holds '${build_cache}' records" -root_dir="" -attempt=0 -while [ "$attempt" -lt 30 ]; do - root_dir="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)" - [ -n "$root_dir" ] && break - attempt=$((attempt + 1)) - sleep 1 +namespaces="$(sudo ctr namespaces ls -q)" +for namespace in ${namespaces}; do + [ "${namespace}" != moby ] || continue + foreign="$(sudo ctr -n "${namespace}" containers ls -q)" + [ -z "${foreign}" ] || refuse "containerd namespace ${namespace} holds containers: ${foreign//$'\n'/ }" + tasks="$(sudo ctr -n "${namespace}" tasks ls -q)" + [ -z "${tasks}" ] || refuse "containerd namespace ${namespace} runs tasks: ${tasks//$'\n'/ }" done -if [ "$root_dir" != "$TARGET_DIR" ]; then - echo "Docker came up with root '${root_dir}', expected '${TARGET_DIR}'." >&2 - restore_and_continue +shim_rc=0 +shims="$(pgrep -a containerd-shim)" || shim_rc=$? +case "${shim_rc}" in +0) refuse "containerd shims are running: ${shims//$'\n'/; }" ;; +1) ;; +*) refuse "pgrep could not list containerd shims (rc=${shim_rc})" ;; +esac + +sudo true +daemon_backup="$(mktemp)" +daemon_new="$(mktemp)" +if sudo test -e "${DAEMON_JSON}"; then + daemon_existed=yes + # shellcheck disable=SC2024 + sudo cat "${DAEMON_JSON}" >"${daemon_backup}" + if [ -s "${daemon_backup}" ] && ! jq empty "${daemon_backup}"; then + refuse "${DAEMON_JSON} is not valid JSON" + fi +else + daemon_existed=no +fi +if [ -s "${daemon_backup}" ]; then + jq --arg dr "${DATA_ROOT}" '. + {"data-root": $dr}' "${daemon_backup}" >"${daemon_new}" +else + jq -n --arg dr "${DATA_ROOT}" '{"data-root": $dr}' >"${daemon_new}" fi -echo "Docker Root Dir: ${root_dir}" -[ -n "$backup" ] && rm -f "$backup" -"$DUMP" "after docker-relocate -> ${TARGET_DIR}" +left_behind="$(sudo du -sxh "${CONTAINERD_ROOT}")" +echo "Left behind on /: ${left_behind%%[[:space:]]*} in ${CONTAINERD_ROOT}" +echo "Relocating the Docker data-root to ${DATA_ROOT} and the containerd root to ${CONTAINERD_TARGET}" + +# Param: $1 what failed +rollback() { + trap - ERR INT TERM + set +e + echo "::error::relocation failed ($1); rolling Docker and containerd back onto /" >&2 + if ! sudo systemctl stop docker.socket docker.service containerd.service; then + echo "::error::could not stop Docker and containerd for the rollback" >&2 + fi + if mountpoint -q "${CONTAINERD_ROOT}" && ! sudo umount "${CONTAINERD_ROOT}"; then + echo "::error::could not unmount the bind over ${CONTAINERD_ROOT}" >&2 + fi + if [ "${daemon_existed}" = yes ]; then + if ! sudo install -m 0644 "${daemon_backup}" "${DAEMON_JSON}"; then + echo "::error::could not restore ${DAEMON_JSON}" >&2 + fi + elif ! sudo rm -f "${DAEMON_JSON}"; then + echo "::error::could not remove the ${DAEMON_JSON} this script wrote" >&2 + fi + if ! sudo systemctl start containerd.service docker.socket docker.service; then + echo "::error::Docker or containerd did not come back up" >&2 + fi + sudo systemctl status --no-pager containerd.service docker.service >&2 + sudo journalctl --no-pager -n 80 -u containerd.service -u docker.service >&2 + exit 1 +} + +trap 'rollback "command failed: ${BASH_COMMAND}"' ERR +trap 'rollback "interrupted"' INT TERM +sudo systemctl stop docker.socket docker.service containerd.service +sudo mkdir -p "${DATA_ROOT}" "${CONTAINERD_TARGET}" "$(dirname "${DAEMON_JSON}")" +sudo mount --bind "${CONTAINERD_TARGET}" "${CONTAINERD_ROOT}" +sudo install -m 0644 "${daemon_new}" "${DAEMON_JSON}" +sudo systemctl start containerd.service +sudo systemctl start docker.socket docker.service +sudo ctr version >/dev/null + +if ! sudo test -f "${CONTAINERD_TARGET}/io.containerd.metadata.v1.bolt/meta.db"; then + rollback "containerd came up without its metadata in ${CONTAINERD_TARGET}" +fi +root_dir="$(docker info --format '{{.DockerRootDir}}')" +if [ "${root_dir}" != "${DATA_ROOT}" ]; then + rollback "Docker came up with root '${root_dir}', expected '${DATA_ROOT}'" +fi +mnt_dev="$(stat -c %d /mnt)" +if [ "$(stat -c %d "${root_dir}")" != "${mnt_dev}" ]; then + rollback "Docker root ${root_dir} is not on the /mnt filesystem" +fi + +trap - ERR INT TERM +rm -f "${daemon_backup}" "${daemon_new}" +echo "Docker Root Dir: ${root_dir}; containerd root ${CONTAINERD_ROOT} bound from ${CONTAINERD_TARGET}" +"$DUMP" "after docker-relocate -> ${DATA_ROOT}, containerd -> ${CONTAINERD_TARGET}" diff --git a/scripts/tests/deploy/utils/filesystem/docker_dataroot.sh b/scripts/tests/deploy/utils/filesystem/docker_dataroot.sh index 7f810384fc..204f9d0413 100755 --- a/scripts/tests/deploy/utils/filesystem/docker_dataroot.sh +++ b/scripts/tests/deploy/utils/filesystem/docker_dataroot.sh @@ -25,7 +25,11 @@ SIZE="${3:?SIZE is required - state the loop image size (e.g. 30G) at the call s IDENTITY="${4:?IDENTITY is required - state what this machine is called at the call site}" MOUNT=/mnt/docker-fs -IMAGE=/var/tmp/docker-fs.img +if mountpoint -q /mnt; then + IMAGE=/mnt/docker-fs.img +else + IMAGE=/var/tmp/docker-fs.img +fi POOL="infinito_ci_$(printf '%s' "${IDENTITY}" | tr -dc 'A-Za-z0-9_.:-')" DAEMON=/etc/docker/daemon.json MISC_MAJOR=10 diff --git a/tests/unit/python/scripts/github/runner/__init__.py b/tests/unit/python/scripts/github/runner/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/scripts/github/runner/test_enlarge_swap_placement.py b/tests/unit/python/scripts/github/runner/test_enlarge_swap_placement.py new file mode 100644 index 0000000000..e3ddc8a5a4 --- /dev/null +++ b/tests/unit/python/scripts/github/runner/test_enlarge_swap_placement.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils.cache.files import PROJECT_ROOT + +SCRIPT = PROJECT_ROOT / "scripts" / "github" / "runner" / "enlarge_swap.sh" +SUDO_STUB = '#!/usr/bin/env bash\nprintf "%s\\n" "$*" >>"${STUB_LOG}"\n' +DF_STUB = ( + "#!/usr/bin/env bash\n" + 'echo "Avail"\n' + 'if [ "${*: -1}" = /mnt ]; then echo "${STUB_MNT_FREE}"; else echo "${STUB_ROOT_FREE}"; fi\n' +) +MOUNTPOINT_STUB = '#!/usr/bin/env bash\n[ "${STUB_MNT_MOUNTED}" = 1 ]\n' +STAT_STUB = ( + "#!/usr/bin/env bash\n" + 'case "${*: -1}" in\n' + "/mnt) echo 2049 ;;\n" + '/var/lib/containerd) if [ "${STUB_CONTAINERD_DEV}" = missing ]; then exit 1; fi\n' + ' echo "${STUB_CONTAINERD_DEV}" ;;\n' + "*) exit 1 ;;\n" + "esac\n" +) +DUMP_STUB = "#!/usr/bin/env bash\nexit 0\n" + + +def enlarge( + *, mounted: bool, mnt_free: int, root_free: int, containerd_dev: str +) -> tuple[subprocess.CompletedProcess, list[str]]: + """Run enlarge_swap.sh against stubbed disks. + + Args: + mounted: whether /mnt is a separate mount. + mnt_free: GB df reports free on /mnt. + root_free: GB df reports free on /. + containerd_dev: device id stat reports for /var/lib/containerd + (2049 is the /mnt device). + + Returns: + The finished process and every sudo call in order. + """ + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + runner_dir = root / "runner" + runner_dir.mkdir() + (runner_dir / "enlarge_swap.sh").symlink_to(SCRIPT) + bin_dir = root / "bin" + bin_dir.mkdir() + for path, body in ( + (runner_dir / "dump_state.sh", DUMP_STUB), + (bin_dir / "sudo", SUDO_STUB), + (bin_dir / "df", DF_STUB), + (bin_dir / "mountpoint", MOUNTPOINT_STUB), + (bin_dir / "stat", STAT_STUB), + ): + path.write_text(body) + path.chmod(0o755) + log = root / "sudo.log" + log.touch() + env = { + k: v for k, v in os.environ.items() if k not in ("BASH_ENV", "SKIP_SWAP") + } + env.update( + PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + STUB_LOG=str(log), + STUB_MNT_MOUNTED="1" if mounted else "0", + STUB_MNT_FREE=str(mnt_free), + STUB_ROOT_FREE=str(root_free), + STUB_CONTAINERD_DEV=containerd_dev, + ) + done = subprocess.run( + ["bash", str(runner_dir / "enlarge_swap.sh")], + env=env, + capture_output=True, + text=True, + check=False, + ) + return done, log.read_text().splitlines() # nocheck: cache-read stub call log + + +def swapfile(calls: list[str]) -> list[str]: + """Return the fallocate calls among the recorded sudo calls. + + Args: + calls: every recorded sudo call. + + Returns: + The calls that create the swapfile. + """ + return [call for call in calls if call.startswith("fallocate ")] + + +class TestEnlargeSwapPlacement(unittest.TestCase): + def test_swap_goes_to_root_when_containerd_lives_on_the_spare_disk(self) -> None: + done, calls = enlarge( + mounted=True, mnt_free=200, root_free=54, containerd_dev="2049" + ) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(swapfile(calls), ["fallocate -l 16G /swapfile"]) + + def test_swap_goes_to_mnt_when_containerd_stayed_on_root(self) -> None: + done, calls = enlarge( + mounted=True, mnt_free=60, root_free=54, containerd_dev="66305" + ) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(swapfile(calls), ["fallocate -l 16G /mnt/swapfile"]) + + def test_large_shape_without_mnt_keeps_swap_on_root(self) -> None: + done, calls = enlarge( + mounted=False, mnt_free=0, root_free=123, containerd_dev="66305" + ) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(swapfile(calls), ["fallocate -l 16G /swapfile"]) + + def test_missing_containerd_root_keeps_the_mnt_preference(self) -> None: + done, calls = enlarge( + mounted=True, mnt_free=60, root_free=54, containerd_dev="missing" + ) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(swapfile(calls), ["fallocate -l 16G /mnt/swapfile"]) + + def test_full_root_still_uses_the_spare_disk(self) -> None: + done, calls = enlarge( + mounted=True, mnt_free=200, root_free=10, containerd_dev="2049" + ) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(swapfile(calls), ["fallocate -l 16G /mnt/swapfile"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/scripts/github/runner/test_mount_spare_disk.py b/tests/unit/python/scripts/github/runner/test_mount_spare_disk.py new file mode 100644 index 0000000000..a22fb980ba --- /dev/null +++ b/tests/unit/python/scripts/github/runner/test_mount_spare_disk.py @@ -0,0 +1,457 @@ +from __future__ import annotations + +import copy +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils.cache.files import PROJECT_ROOT +from utils.cache.yaml import load_yaml + +SCRIPT = PROJECT_ROOT / "scripts" / "github" / "runner" / "mount_spare_disk.sh" +WORKFLOW = PROJECT_ROOT / ".github" / "workflows" / "call-test-deploy.yml" +MIN_SIZE_GB = "75" +GIB = 1024**3 +SPARE = "nvme9n9" +SPARE_INDEX = 2 +DESTRUCTIVE = ("mkfs.ext4", "blkid", "wipefs", "mount", "mkswap") + +SUDO_STUB = ( + "#!/usr/bin/env bash\n" + 'printf "%s\\n" "$*" >>"${STUB_LOG}"\n' + 'if [ -x "${STUB_BIN}/$1" ]; then exec "${STUB_BIN}/$1" "${@:2}"; fi\n' + 'echo "unstubbed sudo $1" >&2\n' + "exit 97\n" +) +UNPRIVILEGED_STUB = ( + '#!/usr/bin/env bash\necho "$(basename "$0") ran without sudo" >&2\nexit 98\n' +) +LSBLK_STUB = '#!/usr/bin/env bash\ncat "${STUB_LSBLK_JSON}"\n' +MOUNTPOINT_STUB = '#!/usr/bin/env bash\n[ "${STUB_MOUNTED}" = 1 ]\n' +BLKID_STUB = ( + '#!/usr/bin/env bash\nprintf "%s" "${STUB_BLKID_OUT}"\nexit "${STUB_BLKID_RC}"\n' +) +WIPEFS_STUB = '#!/usr/bin/env bash\nprintf "%s" "${STUB_WIPEFS_OUT}"\n' +MKFS_STUB = "#!/usr/bin/env bash\nexit 0\n" +MOUNT_STUB = ( + '#!/usr/bin/env bash\nprintf "%s" "${STUB_MOUNT_AS:-$1}" >"${STUB_STATE}"\n' +) +FINDMNT_STUB = '#!/usr/bin/env bash\ncat "${STUB_STATE}"\n' +DUMP_STUB = "#!/usr/bin/env bash\nexit 0\n" + + +def part(name: str, size_gib: float, fstype: str | None, mount: str | None) -> dict: + """Build one lsblk partition entry. + + Args: + name: kernel name of the partition. + size_gib: size in GiB. + fstype: filesystem type, None when unformatted. + mount: mount point, None when unmounted. + + Returns: + The entry as lsblk -J -b prints it. + """ + return { + "name": name, + "type": "part", + "size": int(size_gib * GIB), + "ro": False, + "fstype": fstype, + "pttype": "gpt", + "mountpoints": [mount], + } + + +def snap_loops() -> list[dict]: + """Return the two snap squashfs loop devices every runner shape carries. + + Returns: + The loop entries as lsblk -J -b prints them. + """ + return [ + { + "name": f"loop{index}", + "type": "loop", + "size": 77594624, + "ro": True, + "fstype": "squashfs", + "pttype": None, + "mountpoints": [mount], + } + for index, mount in enumerate(("/snap/core22/2955", "/snap/snapd/28254")) + ] + + +def root_disk(name: str, prefix: str, size_gib: int) -> dict: + """Build the partitioned root disk of a GitHub runner image. + + Args: + name: kernel name of the disk. + prefix: partition name prefix (``sda`` or ``nvme0n1p``). + size_gib: size in GiB. + + Returns: + The disk entry with its four partitions. + """ + return { + "name": name, + "type": "disk", + "size": size_gib * GIB, + "ro": False, + "fstype": None, + "pttype": "gpt", + "mountpoints": [None], + "children": [ + part(f"{prefix}1", size_gib - 1.2, "ext4", "/"), + part(f"{prefix}13", 1, "ext4", "/boot"), + part(f"{prefix}14", 0.004, None, None), + part(f"{prefix}15", 0.1, "vfat", "/boot/efi"), + ], + } + + +def spare_disk(size_gib: int) -> dict: + """Build the empty second NVMe disk of the small runner shape. + + Args: + size_gib: size in GiB. + + Returns: + The disk entry without filesystem, partition table or mount. + """ + return { + "name": SPARE, + "type": "disk", + "size": size_gib * GIB, + "ro": False, + "fstype": None, + "pttype": None, + "mountpoints": [None], + } + + +LARGE_SHAPE = {"blockdevices": [*snap_loops(), root_disk("sda", "sda", 145)]} +SMALL_SHAPE = { + "blockdevices": [ + *snap_loops(), + spare_disk(128), + root_disk("nvme0n1", "nvme0n1p", 72), + ] +} + + +def with_spare(**changes: object) -> dict: + """Return the small shape with fields of the spare disk replaced. + + Args: + **changes: lsblk fields to overwrite on the spare disk. + + Returns: + A modified deep copy of SMALL_SHAPE. + """ + tree = copy.deepcopy(SMALL_SHAPE) + tree["blockdevices"][SPARE_INDEX].update(changes) + return tree + + +def run( + tree: dict, + *, + runner_environment: str | None = "github-hosted", + act: str | None = None, + mounted: bool = False, + blkid_rc: int = 2, + blkid_out: str = "", + wipefs_out: str = "", + mount_as: str | None = None, + occupied: bool = False, + min_size_gb: str = MIN_SIZE_GB, +) -> tuple[subprocess.CompletedProcess, list[str]]: + """Run mount_spare_disk.sh against a faked block-device layout. + + Args: + tree: what ``lsblk -J -b`` reports. + runner_environment: RUNNER_ENVIRONMENT, None to leave it unset. + act: ACT, None to leave it unset. + mounted: whether the mount point already is a mount. + blkid_rc: exit code of ``blkid -p`` (2 means no signature). + blkid_out: what ``blkid -p`` prints. + wipefs_out: what ``wipefs --no-act`` prints. + mount_as: device findmnt reports after the mount, None for the mounted one. + occupied: whether the mount point already holds a file. + min_size_gb: the MIN_SIZE_GB argument. + + Returns: + The finished process and every privileged call in order. + """ + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + runner_dir = root / "runner" + runner_dir.mkdir() + (runner_dir / "mount_spare_disk.sh").symlink_to(SCRIPT) + bin_dir = root / "bin" + bin_dir.mkdir() + stub_bin = root / "privileged" + stub_bin.mkdir() + stubs = [ + (runner_dir / "dump_state.sh", DUMP_STUB), + (bin_dir / "sudo", SUDO_STUB), + (bin_dir / "lsblk", LSBLK_STUB), + (bin_dir / "mountpoint", MOUNTPOINT_STUB), + (bin_dir / "findmnt", FINDMNT_STUB), + (stub_bin / "blkid", BLKID_STUB), + (stub_bin / "wipefs", WIPEFS_STUB), + (stub_bin / "mkfs.ext4", MKFS_STUB), + (stub_bin / "mount", MOUNT_STUB), + ] + stubs += [(bin_dir / name, UNPRIVILEGED_STUB) for name in DESTRUCTIVE] + for path, body in stubs: + path.write_text(body) + path.chmod(0o755) + mount_point = root / "mnt" + mount_point.mkdir() + if occupied: + (mount_point / "keep.txt").write_text("data\n") + lsblk_json = root / "lsblk.json" + lsblk_json.write_text(json.dumps(tree)) + log = root / "sudo.log" + log.touch() + env = { + k: v + for k, v in os.environ.items() + if k not in ("BASH_ENV", "ACT", "RUNNER_ENVIRONMENT") + } + env.update( + PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + STUB_LOG=str(log), + STUB_BIN=str(stub_bin), + STUB_LSBLK_JSON=str(lsblk_json), + STUB_MOUNTED="1" if mounted else "0", + STUB_BLKID_RC=str(blkid_rc), + STUB_BLKID_OUT=blkid_out, + STUB_WIPEFS_OUT=wipefs_out, + STUB_STATE=str(root / "mounted-source"), + ) + if mount_as is not None: + env["STUB_MOUNT_AS"] = mount_as + if runner_environment is not None: + env["RUNNER_ENVIRONMENT"] = runner_environment + if act is not None: + env["ACT"] = act + done = subprocess.run( + [ + "bash", + str(runner_dir / "mount_spare_disk.sh"), + str(mount_point), + min_size_gb, + ], + env=env, + capture_output=True, + text=True, + check=False, + ) + return done, log.read_text().splitlines() # nocheck: cache-read stub call log + + +def formatted(calls: list[str]) -> bool: + """Tell whether any privileged call formatted or mounted a disk. + + Args: + calls: every recorded sudo call. + + Returns: + True when mkfs or mount ran. + """ + return any(call.startswith(("mkfs", "mount ")) for call in calls) + + +class TestMountSpareDisk(unittest.TestCase): + def assert_untouched(self, done: subprocess.CompletedProcess, reason: str) -> None: + """Assert a declined run: exit 0, no privileged call, the stated reason. + + Args: + done: the finished script. + reason: text the script must print for this decline. + """ + self.assertEqual(done.returncode, 0, done.stderr) + self.assertIn(reason, done.stdout + done.stderr) + self.assertRegex(done.stdout, r"::(notice|warning)::") + + def test_small_shape_formats_and_mounts_the_empty_nvme_disk(self) -> None: + done, calls = run(SMALL_SHAPE) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual( + calls[:3], + [ + f"blkid -p /dev/{SPARE}", + f"wipefs --no-act --noheadings /dev/{SPARE}", + f"mkfs.ext4 -q -m 0 -L infinito-spare /dev/{SPARE}", + ], + ) + self.assertTrue(calls[3].startswith(f"mount /dev/{SPARE} "), calls) + self.assertEqual(len(calls), 4, calls) + + def test_large_shape_without_a_spare_disk_stays_as_it_is(self) -> None: + done, calls = run(LARGE_SHAPE) + self.assert_untouched(done, '["sda"]') + self.assertEqual(calls, []) + + def test_self_hosted_runner_is_never_formatted(self) -> None: + done, calls = run(SMALL_SHAPE, runner_environment="self-hosted") + self.assert_untouched(done, "not a GitHub-hosted runner") + self.assertEqual(calls, []) + + def test_unknown_runner_environment_is_never_formatted(self) -> None: + done, calls = run(SMALL_SHAPE, runner_environment=None) + self.assert_untouched(done, "not a GitHub-hosted runner") + self.assertEqual(calls, []) + + def test_act_refuses_loudly_even_with_a_faked_github_runner(self) -> None: + for act in ("true", "1", "TRUE"): + with self.subTest(act=act): + done, calls = run(SMALL_SHAPE, act=act) + self.assertEqual(done.returncode, 1) + self.assertEqual(calls, []) + self.assertIn("refuses to run under act", done.stderr) + + def test_disk_with_a_filesystem_is_never_formatted(self) -> None: + done, calls = run(with_spare(fstype="ext4")) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_disk_with_partitions_is_never_formatted(self) -> None: + done, calls = run(with_spare(children=[part(f"{SPARE}p1", 127, None, None)])) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_disk_held_by_lvm_is_never_formatted(self) -> None: + holder = { + "name": "vg-data", + "type": "lvm", + "size": 127 * GIB, + "ro": False, + "fstype": None, + "pttype": None, + "mountpoints": [None], + } + done, calls = run(with_spare(children=[holder])) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_disk_with_a_bare_partition_table_is_never_formatted(self) -> None: + done, calls = run(with_spare(pttype="dos")) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_mounted_disk_is_never_formatted(self) -> None: + done, calls = run(with_spare(mountpoints=["/data"])) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_disk_used_as_swap_is_never_formatted(self) -> None: + done, calls = run(with_spare(mountpoints=["[SWAP]"])) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_read_only_disk_is_never_formatted(self) -> None: + done, calls = run(with_spare(ro=True)) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_disk_below_the_size_floor_stays_as_it_is(self) -> None: + done, calls = run(with_spare(size=32 * GIB)) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_signature_lsblk_missed_is_never_formatted(self) -> None: + done, calls = run(SMALL_SHAPE, blkid_rc=0, blkid_out='TYPE="xfs"') + self.assert_untouched(done, "carries a signature") + self.assertFalse(formatted(calls), calls) + + def test_signature_only_wipefs_sees_is_never_formatted(self) -> None: + done, calls = run(SMALL_SHAPE, wipefs_out="0x218 LVM2_member") + self.assert_untouched(done, "carries signatures blkid missed") + self.assertFalse(formatted(calls), calls) + + def test_inconclusive_probe_fails_loudly_without_formatting(self) -> None: + done, calls = run(SMALL_SHAPE, blkid_rc=8, blkid_out="ambivalent result") + self.assertEqual(done.returncode, 1) + self.assertFalse(formatted(calls), calls) + + def test_probe_that_found_nothing_but_complained_fails_loudly(self) -> None: + done, calls = run(SMALL_SHAPE, blkid_rc=2, blkid_out="Input/output error") + self.assertEqual(done.returncode, 1) + self.assertFalse(formatted(calls), calls) + + def test_lsblk_without_mountpoints_fails_loudly(self) -> None: + tree = copy.deepcopy(SMALL_SHAPE) + spare = tree["blockdevices"][SPARE_INDEX] + spare["mountpoint"] = spare.pop("mountpoints")[0] + done, calls = run(tree) + self.assertEqual(done.returncode, 1) + self.assertEqual(calls, []) + + def test_layout_without_a_root_device_fails_loudly(self) -> None: + done, calls = run({"blockdevices": [spare_disk(128)]}) + self.assertEqual(done.returncode, 1) + self.assertEqual(calls, []) + + def test_non_numeric_size_floor_fails_loudly(self) -> None: + done, calls = run(SMALL_SHAPE, min_size_gb="RANDOM") + self.assertEqual(done.returncode, 1) + self.assertEqual(calls, []) + + def test_existing_mount_stays_as_it_is(self) -> None: + done, calls = run(SMALL_SHAPE, mounted=True) + self.assert_untouched(done, "already a separate mount") + self.assertEqual(calls, []) + + def test_occupied_mount_point_stays_as_it_is(self) -> None: + done, calls = run(SMALL_SHAPE, occupied=True) + self.assert_untouched(done, "is not empty") + self.assertEqual(calls, []) + + def test_mount_landing_on_another_device_fails_loudly(self) -> None: + done, _ = run(SMALL_SHAPE, mount_as="/dev/nvme0n1p1") + self.assertEqual(done.returncode, 1) + self.assertIn("backed by '/dev/nvme0n1p1'", done.stderr) + + +class TestWorkflowWiring(unittest.TestCase): + def test_spare_disk_step_is_act_gated_and_runs_before_swap_and_relocation( + self, + ) -> None: + workflow = load_yaml(str(WORKFLOW)) + steps = next( + job["steps"] + for job in workflow["jobs"].values() + if any( + "mount_spare_disk.sh" in str(step.get("run", "")) + for step in job.get("steps", []) + ) + ) + runs = [str(step.get("run", "")) for step in steps] + index = next(i for i, cmd in enumerate(runs) if "mount_spare_disk.sh" in cmd) + self.assertEqual( + runs[index], + f"./scripts/github/runner/mount_spare_disk.sh /mnt {MIN_SIZE_GB}", + ) + self.assertEqual(steps[index].get("if"), "${{ !env.ACT }}") + swap = next(i for i, cmd in enumerate(runs) if "enlarge_swap.sh" in cmd) + relocate = next( + i for i, cmd in enumerate(runs) if "relocate_docker_dataroot.sh" in cmd + ) + self.assertLess(index, relocate) + self.assertLess(relocate, swap) + self.assertEqual( + runs[relocate], "./scripts/github/runner/relocate_docker_dataroot.sh 75" + ) + self.assertEqual(steps[relocate].get("if"), "${{ !env.ACT }}") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/scripts/github/runner/test_relocate_docker_dataroot.py b/tests/unit/python/scripts/github/runner/test_relocate_docker_dataroot.py new file mode 100644 index 0000000000..f60d9ab24a --- /dev/null +++ b/tests/unit/python/scripts/github/runner/test_relocate_docker_dataroot.py @@ -0,0 +1,524 @@ +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import unittest +from dataclasses import dataclass, field +from pathlib import Path + +from utils.cache.files import PROJECT_ROOT + +SCRIPT = PROJECT_ROOT / "scripts" / "github" / "runner" / "relocate_docker_dataroot.sh" +MIN_FREE_GB = "75" +DUMP_V2 = "version = 3\nroot = '/var/lib/containerd'\nstate = '/run/containerd'\n" +DUMP_V1 = 'version = 2\nroot = "/var/lib/containerd"\nstate = "/run/containerd"\n' +STOP = "systemctl stop docker.socket docker.service containerd.service" +BIND = "mount --bind /mnt/containerd /var/lib/containerd" +UNBIND = "umount /var/lib/containerd" +RESTART = "systemctl start containerd.service docker.socket docker.service" + +SUDO_STUB = ( + "#!/usr/bin/env bash\n" + 'printf "%s\\n" "$*" >>"${STUB_LOG}"\n' + 'if [ -x "${STUB_BIN}/$1" ]; then exec "${STUB_BIN}/$1" "${@:2}"; fi\n' + 'echo "unstubbed sudo $1" >&2\n' + "exit 97\n" +) +UNPRIVILEGED_STUB = ( + '#!/usr/bin/env bash\necho "$(basename "$0") ran without sudo" >&2\nexit 98\n' +) +TERM_HOOK = ( + 'if [ "$(basename "$0")" = "${STUB_TERM_ON}" ]; then kill -TERM "${PPID}"; fi\n' +) +UNPRIVILEGED = { + "mountpoint": ( + "#!/usr/bin/env bash\n" + 'case "${*: -1}" in\n' + '/mnt) [ "${STUB_MNT_MOUNTED}" = 1 ] ;;\n' + '/var/lib/containerd) [ -e "${STUB_STATE}/bound" ] ;;\n' + "*) exit 1 ;;\n" + "esac\n" + ), + "df": ( + "#!/usr/bin/env bash\n" + 'echo "Avail"\n' + 'if [ "${*: -1}" = /mnt ]; then echo "${STUB_MNT_FREE}"; ' + 'else echo "${STUB_ROOT_FREE}"; fi\n' + ), + "systemctl": '#!/usr/bin/env bash\nexit "${STUB_CONTAINERD_UNIT_RC}"\n', + "docker": ( + "#!/usr/bin/env bash\n" + 'case "$1" in\n' + 'ps) printf "%s" "${STUB_CONTAINERS}" ;;\n' + 'images) printf "%s" "${STUB_IMAGES}" ;;\n' + 'volume) printf "%s" "${STUB_VOLUMES}" ;;\n' + 'system) printf "Images=0\\nContainers=0\\nLocal Volumes=0\\n' + 'Build Cache=%s\\n" "${STUB_BUILD_CACHE}" ;;\n' + 'info) echo "${STUB_DOCKER_ROOT}" ;;\n' + "*) exit 96 ;;\n" + "esac\n" + ), + "stat": ( + "#!/usr/bin/env bash\n" + 'case "${*: -1}" in\n' + "/mnt) echo 2049 ;;\n" + '/mnt/docker-host) echo "${STUB_DOCKER_ROOT_DEV}" ;;\n' + "*) echo 66305 ;;\n" + "esac\n" + ), + "pgrep": '#!/usr/bin/env bash\nprintf "%s" "${STUB_SHIMS}"\nexit "${STUB_SHIM_RC}"\n', +} +PRIVILEGED = { + "systemctl": ( + "#!/usr/bin/env bash\n" + 'if [ "$1" = status ]; then exit "${STUB_STATUS_RC}"; fi\n' + 'if [ "$*" = "${STUB_FAIL_SYSTEMCTL}" ]; then exit 1; fi\n' + ), + "containerd": '#!/usr/bin/env bash\nprintf "%s" "${STUB_CONTAINERD_DUMP}"\n', + "ctr": ( + "#!/usr/bin/env bash\n" + 'if [ "$1" = namespaces ]; then printf "%s" "${STUB_NAMESPACES}"; exit; fi\n' + 'if [ "$1" = -n ] && [ "$2" = "${STUB_FOREIGN_NS}" ]; then\n' + ' if [ "$3" = containers ]; then printf "%s" "${STUB_FOREIGN_CONTAINERS}"; fi\n' + ' if [ "$3" = tasks ]; then printf "%s" "${STUB_FOREIGN_TASKS}"; fi\n' + "fi\n" + ), + "du": '#!/usr/bin/env bash\nprintf "1.2G\\t%s\\n" "${*: -1}"\n', + "true": '#!/usr/bin/env bash\nexit "${STUB_SUDO_RC}"\n', + "test": ( + "#!/usr/bin/env bash\n" + 'if [ "$1" = -e ]; then [ "${STUB_DAEMON_EXISTS}" = 1 ]; exit; fi\n' + '[ "${STUB_METADATA}" = 1 ]\n' + ), + "cat": ( + "#!/usr/bin/env bash\n" + 'if [ "${STUB_DAEMON_CAT_RC}" != 0 ]; then exit "${STUB_DAEMON_CAT_RC}"; fi\n' + 'printf "%s" "${STUB_DAEMON_IN}"\n' + ), + "mkdir": "#!/usr/bin/env bash\n" + TERM_HOOK, + "mount": ( + "#!/usr/bin/env bash\n" + 'if [ "${STUB_MOUNT_RC}" != 0 ]; then exit "${STUB_MOUNT_RC}"; fi\n' + 'touch "${STUB_STATE}/bound"\n' + ), + "umount": '#!/usr/bin/env bash\nrm -f "${STUB_STATE}/bound"\n', + "install": '#!/usr/bin/env bash\ncp "${*: -2:1}" "${STUB_DAEMON_OUT}"\n', + "rm": '#!/usr/bin/env bash\n/bin/rm -f "${STUB_DAEMON_OUT}"\n', + "journalctl": "#!/usr/bin/env bash\nexit 0\n", +} +DESTRUCTIVE = ("mount", "umount", "install", "ctr", "containerd", "journalctl", "mkdir") + + +@dataclass +class Runner: + """The faked runner state relocate_docker_dataroot.sh meets. + + Args: + runner_environment: RUNNER_ENVIRONMENT, None to leave it unset. + act: ACT, None to leave it unset. + skip: SKIP_DOCKER_RELOCATE, None to leave it unset. + mounted: whether /mnt is a separate mount. + mnt_free: GB df reports free on /mnt. + root_free: GB df reports free on /. + dump: what ``containerd config dump`` prints. + containerd_unit_rc: exit code of ``systemctl cat containerd.service``. + containers: what ``docker ps -aq`` prints. + images: what ``docker images -q`` prints. + volumes: what ``docker volume ls -q`` prints. + build_cache: Build Cache record count in ``docker system df``. + namespaces: what ``ctr namespaces ls -q`` prints. + foreign_ns: namespace whose containers and tasks are faked. + foreign_containers: containers in foreign_ns. + foreign_tasks: tasks in foreign_ns. + shims: what ``pgrep -a containerd-shim`` prints. + shim_rc: exit code of pgrep (1 means no shim). + sudo_rc: exit code of ``sudo true``. + daemon_exists: whether daemon.json exists. + daemon_in: content of the existing daemon.json. + daemon_cat_rc: exit code of ``sudo cat`` on daemon.json. + mount_rc: exit code of the bind mount. + fail_systemctl: the one systemctl argument string that fails. + status_rc: exit code of ``systemctl status``. + term_on: privileged stub that sends SIGTERM to the script. + metadata: whether containerd's meta.db shows up under /mnt/containerd. + docker_root: what ``docker info`` reports as DockerRootDir. + docker_root_dev: device id stat reports for /mnt/docker-host. + min_free_gb: the MIN_FREE_GB argument. + """ + + runner_environment: str | None = "github-hosted" + act: str | None = None + skip: str | None = None + mounted: bool = True + mnt_free: int = 137 + root_free: int = 51 + dump: str = DUMP_V2 + containerd_unit_rc: int = 0 + containers: str = "" + images: str = "" + volumes: str = "" + build_cache: str = "0" + namespaces: str = "moby\n" + foreign_ns: str = "k8s.io" + foreign_containers: str = "" + foreign_tasks: str = "" + shims: str = "" + shim_rc: int = 1 + sudo_rc: int = 0 + daemon_exists: bool = False + daemon_in: str = "" + daemon_cat_rc: int = 0 + mount_rc: int = 0 + fail_systemctl: str = "" + status_rc: int = 0 + term_on: str = "" + metadata: bool = True + docker_root: str = "/mnt/docker-host" + docker_root_dev: str = "2049" + min_free_gb: str = MIN_FREE_GB + extra: dict[str, str] = field(default_factory=dict) + + +@dataclass +class Outcome: + """What a relocation run left behind. + + Args: + done: the finished script. + calls: every privileged call in order. + daemon: the daemon.json on disk afterwards, None when there is none + or it is no valid JSON. + daemon_raw: the daemon.json bytes on disk afterwards, None when there is none. + bound: whether the containerd bind mount is still in place. + """ + + done: subprocess.CompletedProcess + calls: list[str] + daemon: dict | None + daemon_raw: str | None + bound: bool + + +def relocate(runner: Runner) -> Outcome: + """Run relocate_docker_dataroot.sh against a faked runner. + + Args: + runner: the faked services, disks and files. + + Returns: + The outcome of the run. + """ + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + runner_dir = root / "runner" + bin_dir = root / "bin" + stub_bin = root / "privileged" + state = root / "state" + for directory in (runner_dir, bin_dir, stub_bin, state): + directory.mkdir() + (runner_dir / "relocate_docker_dataroot.sh").symlink_to(SCRIPT) + stubs = [(runner_dir / "dump_state.sh", "#!/usr/bin/env bash\nexit 0\n")] + stubs += [(bin_dir / "sudo", SUDO_STUB)] + stubs += [(bin_dir / name, UNPRIVILEGED_STUB) for name in DESTRUCTIVE] + stubs += [(bin_dir / name, body) for name, body in UNPRIVILEGED.items()] + stubs += [(stub_bin / name, body) for name, body in PRIVILEGED.items()] + for path, body in stubs: + path.write_text(body) + path.chmod(0o755) + log = root / "calls.log" + log.touch() + daemon_out = root / "daemon.json" + if runner.daemon_exists: + daemon_out.write_text(runner.daemon_in) + env = { + k: v + for k, v in os.environ.items() + if k + not in ("BASH_ENV", "ACT", "RUNNER_ENVIRONMENT", "SKIP_DOCKER_RELOCATE") + } + env.update( + PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + STUB_LOG=str(log), + STUB_BIN=str(stub_bin), + STUB_STATE=str(state), + STUB_MNT_MOUNTED="1" if runner.mounted else "0", + STUB_MNT_FREE=str(runner.mnt_free), + STUB_ROOT_FREE=str(runner.root_free), + STUB_CONTAINERD_DUMP=runner.dump, + STUB_CONTAINERD_UNIT_RC=str(runner.containerd_unit_rc), + STUB_CONTAINERS=runner.containers, + STUB_IMAGES=runner.images, + STUB_VOLUMES=runner.volumes, + STUB_BUILD_CACHE=runner.build_cache, + STUB_NAMESPACES=runner.namespaces, + STUB_FOREIGN_NS=runner.foreign_ns, + STUB_FOREIGN_CONTAINERS=runner.foreign_containers, + STUB_FOREIGN_TASKS=runner.foreign_tasks, + STUB_SHIMS=runner.shims, + STUB_SHIM_RC=str(runner.shim_rc), + STUB_SUDO_RC=str(runner.sudo_rc), + STUB_DAEMON_EXISTS="1" if runner.daemon_exists else "0", + STUB_DAEMON_IN=runner.daemon_in, + STUB_DAEMON_CAT_RC=str(runner.daemon_cat_rc), + STUB_DAEMON_OUT=str(daemon_out), + STUB_MOUNT_RC=str(runner.mount_rc), + STUB_FAIL_SYSTEMCTL=runner.fail_systemctl, + STUB_STATUS_RC=str(runner.status_rc), + STUB_TERM_ON=runner.term_on, + STUB_METADATA="1" if runner.metadata else "0", + STUB_DOCKER_ROOT=runner.docker_root, + STUB_DOCKER_ROOT_DEV=runner.docker_root_dev, + **runner.extra, + ) + optional = { + "RUNNER_ENVIRONMENT": runner.runner_environment, + "ACT": runner.act, + "SKIP_DOCKER_RELOCATE": runner.skip, + } + env.update({k: v for k, v in optional.items() if v is not None}) + done = subprocess.run( + [ + "bash", + str(runner_dir / "relocate_docker_dataroot.sh"), + runner.min_free_gb, + ], + env=env, + capture_output=True, + text=True, + check=False, + ) + calls = log.read_text().splitlines() # nocheck: cache-read stub call log + content = ( + daemon_out.read_text() # nocheck: cache-read stub output + if daemon_out.exists() + else None + ) + try: + daemon = json.loads(content) if content else None + except json.JSONDecodeError: + daemon = None + return Outcome( + done=done, + calls=calls, + daemon=daemon, + daemon_raw=content, + bound=(state / "bound").exists(), + ) + + +class TestRelocation(unittest.TestCase): + def test_small_shape_moves_containerd_and_docker_onto_the_spare_disk(self) -> None: + out = relocate(Runner()) + self.assertEqual(out.done.returncode, 0, out.done.stderr) + stop, bind = out.calls.index(STOP), out.calls.index(BIND) + start = out.calls.index("systemctl start containerd.service") + self.assertLess(stop, bind) + self.assertLess(bind, start) + self.assertIn("systemctl start docker.socket docker.service", out.calls) + self.assertIn( + "test -f /mnt/containerd/io.containerd.metadata.v1.bolt/meta.db", out.calls + ) + self.assertEqual(out.daemon, {"data-root": "/mnt/docker-host"}) + self.assertTrue(out.bound) + + def test_containerd_1x_dump_with_double_quotes_is_understood(self) -> None: + out = relocate(Runner(dump=DUMP_V1)) + self.assertEqual(out.done.returncode, 0, out.done.stderr) + self.assertIn(BIND, out.calls) + + def test_existing_daemon_json_keeps_its_settings(self) -> None: + out = relocate(Runner(daemon_exists=True, daemon_in='{"debug": true}')) + self.assertEqual(out.done.returncode, 0, out.done.stderr) + self.assertEqual(out.daemon, {"debug": True, "data-root": "/mnt/docker-host"}) + + +class TestDeclines(unittest.TestCase): + def assert_declined(self, out: Outcome, reason: str) -> None: + """Assert a visible no-op: exit 0, a notice with the reason, no call. + + Args: + out: the outcome of the run. + reason: text the notice must carry. + """ + self.assertEqual(out.done.returncode, 0, out.done.stderr) + self.assertIn("::notice::", out.done.stdout) + self.assertIn(reason, out.done.stdout) + self.assertEqual(out.calls, []) + + def test_large_shape_without_a_mnt_mount(self) -> None: + self.assert_declined( + relocate(Runner(mounted=False)), "/mnt is not a separate mount" + ) + + def test_self_hosted_runner(self) -> None: + self.assert_declined( + relocate(Runner(runner_environment="self-hosted")), + "not a GitHub-hosted runner", + ) + + def test_unknown_runner_environment(self) -> None: + self.assert_declined( + relocate(Runner(runner_environment=None)), "not a GitHub-hosted runner" + ) + + def test_mnt_below_the_floor(self) -> None: + self.assert_declined(relocate(Runner(mnt_free=60, root_free=20)), "below 75G") + + def test_mnt_not_roomier_than_root(self) -> None: + self.assert_declined( + relocate(Runner(mnt_free=80, root_free=90)), "not more than" + ) + + def test_skip_toggle(self) -> None: + self.assert_declined(relocate(Runner(skip="1")), "SKIP_DOCKER_RELOCATE=1") + + +class TestRefusalsBeforeTheStop(unittest.TestCase): + def assert_refused(self, out: Outcome, reason: str) -> None: + """Assert a loud refusal before anything was stopped or written. + + Args: + out: the outcome of the run. + reason: text the error must carry. + """ + self.assertEqual(out.done.returncode, 1) + self.assertIn(reason, out.done.stderr) + self.assertNotIn(STOP, out.calls) + self.assertFalse([c for c in out.calls if c.startswith(("mount ", "install "))]) + self.assertFalse(out.bound) + + def test_act_even_with_a_faked_github_runner(self) -> None: + for act in ("true", "1"): + with self.subTest(act=act): + self.assert_refused( + relocate(Runner(act=act)), "refuses to run under act" + ) + + def test_non_numeric_floor(self) -> None: + self.assert_refused(relocate(Runner(min_free_gb="forty")), "positive integer") + + def test_missing_containerd_unit(self) -> None: + self.assert_refused( + relocate(Runner(containerd_unit_rc=1)), "containerd.service is missing" + ) + + def test_containerd_root_elsewhere(self) -> None: + self.assert_refused( + relocate(Runner(dump="version = 3\nroot = '/data/containerd'\n")), + "/data/containerd", + ) + + def test_existing_containers(self) -> None: + self.assert_refused(relocate(Runner(containers="abc123\n")), "abc123") + + def test_existing_images(self) -> None: + self.assert_refused(relocate(Runner(images="sha256feed\n")), "sha256feed") + + def test_existing_volumes(self) -> None: + self.assert_refused( + relocate(Runner(volumes="core_infinito\n")), "core_infinito" + ) + + def test_existing_build_cache(self) -> None: + self.assert_refused(relocate(Runner(build_cache="3")), "build cache") + + def test_containers_in_another_namespace(self) -> None: + out = relocate( + Runner(namespaces="moby\nk8s.io\n", foreign_containers="pause1\n") + ) + self.assert_refused(out, "k8s.io holds containers: pause1") + + def test_tasks_in_another_namespace(self) -> None: + out = relocate(Runner(namespaces="moby\nk8s.io\n", foreign_tasks="task7\n")) + self.assert_refused(out, "k8s.io runs tasks: task7") + + def test_running_shims(self) -> None: + out = relocate(Runner(shims="4242 containerd-shim-runc-v2 -id x\n", shim_rc=0)) + self.assert_refused(out, "containerd shims are running") + + def test_shim_listing_error(self) -> None: + self.assert_refused( + relocate(Runner(shim_rc=2)), "could not list containerd shims" + ) + + def test_broken_sudo(self) -> None: + out = relocate(Runner(sudo_rc=1)) + self.assertEqual(out.done.returncode, 1) + self.assertNotIn(STOP, out.calls) + + def test_unreadable_daemon_json_is_never_overwritten(self) -> None: + out = relocate(Runner(daemon_exists=True, daemon_in="{}", daemon_cat_rc=1)) + self.assertEqual(out.done.returncode, 1) + self.assertNotIn(STOP, out.calls) + self.assertFalse([c for c in out.calls if c.startswith("install ")]) + self.assertEqual(out.daemon, {}) + + def test_invalid_daemon_json_is_never_overwritten(self) -> None: + out = relocate(Runner(daemon_exists=True, daemon_in="{not json")) + self.assertEqual(out.done.returncode, 1) + self.assertIn("is not valid JSON", out.done.stderr) + self.assertNotIn(STOP, out.calls) + self.assertFalse([c for c in out.calls if c.startswith("install ")]) + self.assertEqual(out.daemon_raw, "{not json") + + +class TestRollback(unittest.TestCase): + def assert_rolled_back(self, out: Outcome, reason: str) -> None: + """Assert the rollback: no bind left, daemons restarted last, exit 1. + + Args: + out: the outcome of the run. + reason: text the rollback error must carry. + """ + self.assertEqual(out.done.returncode, 1, out.done.stderr) + self.assertIn("rolling Docker and containerd back onto /", out.done.stderr) + self.assertIn(reason, out.done.stderr) + self.assertFalse(out.bound) + self.assertIn(RESTART, out.calls) + self.assertTrue([c for c in out.calls if c.startswith("journalctl ")]) + + def test_failed_bind_mount_removes_the_daemon_json_it_never_had(self) -> None: + out = relocate(Runner(mount_rc=32)) + self.assert_rolled_back(out, "mount --bind") + self.assertIsNone(out.daemon) + + def test_failed_containerd_start_unbinds_before_the_restart(self) -> None: + out = relocate( + Runner( + daemon_exists=True, + daemon_in='{"debug": true}', + fail_systemctl="start containerd.service", + ) + ) + self.assert_rolled_back(out, "systemctl start containerd.service") + self.assertLess(out.calls.index(UNBIND), len(out.calls) - 1) + self.assertLess(out.calls.index(UNBIND), out.calls.index(RESTART)) + self.assertEqual(out.daemon, {"debug": True}) + + def test_cancel_during_the_move_rolls_back(self) -> None: + out = relocate(Runner(term_on="mkdir")) + self.assert_rolled_back(out, "interrupted") + + def test_down_units_still_yield_the_journal(self) -> None: + out = relocate(Runner(mount_rc=32, status_rc=3)) + self.assert_rolled_back(out, "mount --bind") + + def test_containerd_without_metadata_on_the_spare_disk(self) -> None: + out = relocate(Runner(metadata=False)) + self.assert_rolled_back(out, "without its metadata") + self.assertIn(UNBIND, out.calls) + self.assertIsNone(out.daemon) + + def test_docker_root_elsewhere(self) -> None: + out = relocate(Runner(docker_root="/var/lib/docker")) + self.assert_rolled_back(out, "expected '/mnt/docker-host'") + + def test_docker_root_off_the_spare_disk(self) -> None: + out = relocate(Runner(docker_root_dev="66305")) + self.assert_rolled_back(out, "not on the /mnt filesystem") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/tests/swarm/test_run_storage_roots.py b/tests/unit/python/utils/tests/swarm/test_run_storage_roots.py new file mode 100644 index 0000000000..f74647a43c --- /dev/null +++ b/tests/unit/python/utils/tests/swarm/test_run_storage_roots.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import shutil +import unittest +import unittest.mock as mock +from pathlib import Path +from typing import NamedTuple + +from utils.tests.swarm import run + + +class Usage(NamedTuple): + total: int + used: int + free: int + + +def lowest(free_gib: dict[str, int], existing: set[str]) -> tuple[int, str]: + """Call lowest_free_mb with faked disks. + + Args: + free_gib: free GiB per storage root. + existing: storage roots that exist as directories. + + Returns: + What lowest_free_mb reports. + """ + with ( + mock.patch.object( + shutil, + "disk_usage", + side_effect=lambda root: Usage(0, 0, free_gib[root] * 2**30), + ), + mock.patch.object( + Path, "is_dir", autospec=True, side_effect=lambda p: str(p) in existing + ), + ): + return run.lowest_free_mb() + + +class TestLowestFreeMb(unittest.TestCase): + def test_a_full_spare_disk_under_containerd_is_seen(self) -> None: + free, root = lowest( + {"/": 40, "/var/lib/containerd": 5}, {"/", "/var/lib/containerd"} + ) + self.assertEqual((free, root), (5 * 1024, "/var/lib/containerd")) + self.assertLess(free, run.DISK_FLOOR_MB) + + def test_a_full_root_is_still_seen(self) -> None: + free, root = lowest( + {"/": 5, "/var/lib/containerd": 150}, {"/", "/var/lib/containerd"} + ) + self.assertEqual((free, root), (5 * 1024, "/")) + + def test_a_host_without_containerd_measures_root_only(self) -> None: + free, root = lowest({"/": 30}, {"/"}) + self.assertEqual((free, root), (30 * 1024, "/")) + + +def watch( + waits: list[object], free: tuple[int, str] +) -> tuple[int, mock.MagicMock, str]: + """Run run_watched over a fake step. + + Args: + waits: successive outcomes of the step's wait(): an exception to raise + or an exit code to return. + free: what lowest_free_mb reports on every sample. + + Returns: + The exit code, the fake process, and everything printed. + """ + proc = mock.MagicMock() + proc.wait.side_effect = waits + printed = mock.MagicMock() + with ( + mock.patch.object(run.subprocess, "Popen", return_value=proc), + mock.patch.object(run.subprocess, "run"), + mock.patch.object(run, "lowest_free_mb", return_value=free), + mock.patch.object(run, "mem_available_mb", return_value=12000), + mock.patch.object(run, "mem_total_mb", return_value=16000), + mock.patch.object(run, "mem_stall_pct", return_value=0.0), + mock.patch("builtins.print", printed), + ): + rc = run.run_watched(["step"], env={}) + text = " ".join(str(call.args[0]) for call in printed.call_args_list) + return rc, proc, text + + +class TestRunWatchedSampling(unittest.TestCase): + def test_samples_every_ten_seconds(self) -> None: + timeout = run.subprocess.TimeoutExpired("step", run.SAMPLE_INTERVAL_S) + rc, proc, _ = watch([timeout, timeout, 0], (50_000, "/")) + self.assertEqual(rc, 0) + self.assertEqual(run.SAMPLE_INTERVAL_S, 10) + self.assertEqual( + [call.kwargs["timeout"] for call in proc.wait.call_args_list], + [10, 10, 10], + ) + + def test_a_full_containerd_root_aborts_the_step(self) -> None: + timeout = run.subprocess.TimeoutExpired("step", run.SAMPLE_INTERVAL_S) + rc, proc, text = watch([timeout, 0], (1_000, "/var/lib/containerd")) + self.assertEqual(rc, 75) + proc.terminate.assert_called_once_with() + self.assertIn("free on /var/lib/containerd", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/tests/swarm/run.py b/utils/tests/swarm/run.py index 042e7fcb96..405e4c8775 100644 --- a/utils/tests/swarm/run.py +++ b/utils/tests/swarm/run.py @@ -3,12 +3,28 @@ from __future__ import annotations import shutil import subprocess import time +from pathlib import Path from utils import PROJECT_ROOT from utils.env.runtime import mem_available_mb, mem_stall_pct, mem_total_mb DISK_FLOOR_MB = 6 * 2**10 MEM_FLOOR_RATIO = 0.06 +STORAGE_ROOTS = ("/", "/var/lib/containerd") +SAMPLE_INTERVAL_S = 10 + + +def lowest_free_mb() -> tuple[int, str]: + """Measure the storage root closest to running full. + + Returns: + Free MB on the fullest existing root in STORAGE_ROOTS, and that root. + """ + return min( + (shutil.disk_usage(root).free // 2**20, root) + for root in STORAGE_ROOTS + if Path(root).is_dir() + ) def _abort(proc: subprocess.Popen[bytes], banner: str, probe: list[str]) -> int: @@ -61,22 +77,22 @@ def run_watched(cmd: list[str], *, env: dict[str, str]) -> int: proc = subprocess.Popen(cmd, cwd=str(PROJECT_ROOT), env=env) while True: try: - return int(proc.wait(timeout=30)) + return int(proc.wait(timeout=SAMPLE_INTERVAL_S)) except subprocess.TimeoutExpired: - free_mb = shutil.disk_usage("/").free // 2**20 + free_mb, root = lowest_free_mb() if free_mb < DISK_FLOOR_MB: return _abort( proc, "=== swarm-matrix: DISK EXHAUSTION IMMINENT " - f"(<{DISK_FLOOR_MB}M free on /) - aborting step ===", - ["df", "-h", "/"], + f"(<{DISK_FLOOR_MB}M free on {root}) - aborting step ===", + ["df", "-h", *STORAGE_ROOTS], ) avail = mem_available_mb() total = mem_total_mb() stall = mem_stall_pct() print( f"=== swarm-matrix: host mem {avail}M/{total}M available, " - f"disk {free_mb}M free on /, stall60 {stall:.1f}% ===", + f"disk {free_mb}M free on {root}, stall60 {stall:.1f}% ===", flush=True, ) if total and avail < total * MEM_FLOOR_RATIO: