diff --git a/.github/workflows/call-test-deploy.yml b/.github/workflows/call-test-deploy.yml index dcdf313fdc..d20d798d3c 100644 --- a/.github/workflows/call-test-deploy.yml +++ b/.github/workflows/call-test-deploy.yml @@ -298,16 +298,21 @@ jobs: shell: bash run: grep '^INFINITO_RESCUE_DIAGNOSTICS_BASE=' default.env >> "$GITHUB_ENV" + - name: 💽 Mount the empty spare disk at /mnt + if: ${{ !env.ACT }} + shell: bash + run: ./scripts/github/runner/mount_spare_disk.sh /mnt 75 + + - name: 📦 Relocate containerd and the Docker data-root onto /mnt when roomier + if: ${{ !env.ACT }} + shell: bash + run: ./scripts/github/runner/relocate_docker_dataroot.sh 75 + - name: 💾 Enlarge swap space if: ${{ !env.ACT }} shell: bash run: ./scripts/github/runner/enlarge_swap.sh - - name: 📦 Relocate Docker data-root onto /mnt when roomier - if: ${{ !env.ACT }} - shell: bash - run: ./scripts/github/runner/relocate_docker_dataroot.sh - - name: 🛡️ Tear down host AppArmor profiles (dev-host parity) if: ${{ !env.ACT }} shell: bash diff --git a/docs/contributing/artefact/files/github/workflow.md b/docs/contributing/artefact/files/github/workflow.md index 383e8e7961..a186b9c65c 100644 --- a/docs/contributing/artefact/files/github/workflow.md +++ b/docs/contributing/artefact/files/github/workflow.md @@ -78,11 +78,39 @@ Deploy test workflows use the `jlumbroso/free-disk-space` action to reclaim runn Set an option back to `false` only when a new host-side step in the same workflow genuinely needs the removed payload. +## Spare disk 💽 + +Some GitHub-hosted runners have a 72 GB root disk plus a second, unformatted NVMe disk. [mount_spare_disk.sh](../../../../../scripts/github/runner/mount_spare_disk.sh) formats that disk as ext4 and mounts it at `/mnt`. The nested Docker volume (`/mnt/docker`) and the loop image of [docker_dataroot.sh](../../../../../scripts/tests/deploy/utils/filesystem/docker_dataroot.sh) (`/mnt/docker-fs.img`) then land on it. + +When `/mnt` has at least `MIN_FREE_GB` free and more than `/`, [relocate_docker_dataroot.sh](../../../../../scripts/github/runner/relocate_docker_dataroot.sh) stops Docker and containerd, bind-mounts `/mnt/containerd` over `/var/lib/containerd` (images, container layers, the swarm node containers), sets the Docker data-root to `/mnt/docker-host` (named volumes), restarts both and verifies that containerd's metadata and the Docker root sit on `/mnt`. Before the stop it fails if Docker already holds containers, images, volumes or build cache, if another containerd namespace holds containers or tasks, if containerd shims run, or if `daemon.json` cannot be read as JSON. Any failure after the stop, and a cancel, unmounts the bind, restores `daemon.json`, restarts both daemons and fails the step. It refuses under act and changes nothing on a runner that is not GitHub-hosted. Every decline prints a `::notice::` annotation. [run.py](../../../../../utils/tests/swarm/run.py) then watches the free space of both `/` and `/var/lib/containerd`. + +Step order: spare disk, relocation, swap. With containerd on `/mnt`, the swapfile goes to `/`. + +The script formats a disk only when all of these hold; otherwise the runner stays as it is: + +- `ACT` is unset (the script fails under act) and `RUNNER_ENVIRONMENT` is `github-hosted`. +- `/mnt` is an existing, empty directory and no mount. +- The disk is a whole disk other than the root disk, with no partitions or holders, no filesystem or partition-table signature (`lsblk`, `blkid -p` and `wipefs`), no mount or swap use, not read-only, and at least `MIN_SIZE_GB` large. + +Workflow step invocations, in this order and gated alike: + +```yaml +- name: Mount the empty spare disk at /mnt + if: ${{ !env.ACT }} + shell: bash + run: ./scripts/github/runner/mount_spare_disk.sh /mnt 75 + +- name: Relocate containerd and the Docker data-root onto /mnt when roomier + if: ${{ !env.ACT }} + shell: bash + run: ./scripts/github/runner/relocate_docker_dataroot.sh 75 +``` + ## Swap 💾 Deploy test workflows enlarge host swap via [enlarge_swap.sh](../../../../../scripts/github/runner/enlarge_swap.sh) to absorb transient memory spikes (e.g. PeerTube plugin install [#162](https://github.com/infinito-nexus/core/issues/162)) that would otherwise trip the host OOM-killer on the 16 GB GitHub-hosted runner. -The script **prefers `/mnt`** when it is a separate partition (classic `ubuntu-latest` layout with `/dev/sdb` mounted at `/mnt`) and falls back to `/` otherwise. Current public runners no longer expose a separate `/mnt`, so the swapfile lands on `/` and must not starve the root filesystem. +The script puts the swapfile on `/` when containerd already lives on `/mnt`, **prefers `/mnt`** otherwise when it is a separate partition (classic `ubuntu-latest` layout with `/dev/sdb` mounted at `/mnt`), and falls back to `/`. Current public runners no longer expose a separate `/mnt`, so the swapfile lands on `/` and must not starve the root filesystem. Swap size is a **fixed constant**, not a "free space minus buffer" calculation. An earlier version took ~99 GB on a 145 GB root partition, leaving so little headroom that later Docker layer writes (Playwright test image pull) failed with `no space left on device`. A fixed cap is large enough to absorb the peertube spike and small enough that Docker layer writes keep succeeding throughout the 30+ min deploy. diff --git a/scripts/github/runner/dump_state.sh b/scripts/github/runner/dump_state.sh index 93f41a74ac..b8c9ce617f 100755 --- a/scripts/github/runner/dump_state.sh +++ b/scripts/github/runner/dump_state.sh @@ -18,14 +18,24 @@ date -u +'%Y-%m-%dT%H:%M:%SZ' echo "--- disk (bytes) ---" df -h / /mnt 2>/dev/null || df -h / +echo "--- disk per storage root (/, /mnt, containerd root, Docker data root) ---" +docker_root="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null)" || docker_root="" +df -h --output=source,size,used,avail,target / /mnt /var/lib/containerd ${docker_root:+"${docker_root}"} || + echo "df could not read every storage root" + +containerd --version 2>/dev/null || echo "containerd not on PATH" + echo "--- disk (inodes) ---" df -hi / /mnt 2>/dev/null || df -hi / -echo "--- block devices ---" -lsblk -f || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error +echo "--- disks (bytes, model: Amazon EC2 NVMe Instance Storage versus Amazon Elastic Block Store) ---" +lsblk -d -b -o NAME,SIZE,MODEL,ROTA,TYPE || echo "lsblk could not list the disks" -echo "--- mounts (/, /mnt, /var/lib/docker, /mnt/docker) ---" -mount | grep -E ' / | /mnt | /var/lib/docker | /mnt/docker ' || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error +echo "--- block devices ---" +lsblk -o NAME,TYPE,SIZE,FSTYPE,FSVER,LABEL,UUID,FSAVAIL,FSUSE%,MOUNTPOINTS || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error + +echo "--- mounts (/, /mnt, /var/lib/docker, /var/lib/containerd, /mnt/docker) ---" +mount | grep -E ' / | /mnt | /var/lib/docker | /var/lib/containerd | /mnt/docker ' || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error echo "--- docker ---" docker info 2>/dev/null | diff --git a/scripts/github/runner/enlarge_swap.sh b/scripts/github/runner/enlarge_swap.sh index 48b56e1473..f61a347c36 100755 --- a/scripts/github/runner/enlarge_swap.sh +++ b/scripts/github/runner/enlarge_swap.sh @@ -4,7 +4,8 @@ # spikes (e.g. peertube plugin install, issue #162) don't trip the host # OOM-killer, WITHOUT starving the rest of the job of disk space. # -# Prefers /mnt when it is a separate partition (classic ubuntu-latest +# Uses / when containerd already lives on /mnt (relocate_docker_dataroot.sh +# runs first), otherwise prefers /mnt when it is a separate partition (classic ubuntu-latest # layout with /dev/sdb mounted at /mnt). On current public runners # there is no separate /mnt partition — everything lives on /, so we # place the swapfile there. The swap size is a hard constant, never a @@ -46,7 +47,16 @@ root_free=$(df --output=avail -B1G / | tail -n1 | tr -d ' ') echo "Target swap: ${SWAP_GB}G. Free: / ${root_free}G, /mnt ${mnt_free}G (separate mount: $(mountpoint -q /mnt 2>/dev/null && echo yes || echo no))" -if [ "$mnt_free" -ge "$MIN_FREE_GB" ]; then +storage_on_mnt=no +if mountpoint -q /mnt 2>/dev/null && containerd_dev="$(stat -c %d /var/lib/containerd 2>/dev/null)" && + [ "$containerd_dev" = "$(stat -c %d /mnt)" ]; then + storage_on_mnt=yes +fi + +if [ "$storage_on_mnt" = yes ] && [ "$root_free" -ge "$MIN_FREE_GB" ]; then + target=/swapfile + chosen="/ (containerd and Docker live on /mnt)" +elif [ "$mnt_free" -ge "$MIN_FREE_GB" ]; then target=/mnt/swapfile chosen="/mnt (separate partition)" elif [ "$root_free" -ge "$MIN_FREE_GB" ]; then diff --git a/scripts/github/runner/mount_spare_disk.sh b/scripts/github/runner/mount_spare_disk.sh new file mode 100755 index 0000000000..0d52e941ba --- /dev/null +++ b/scripts/github/runner/mount_spare_disk.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Usage: mount_spare_disk.sh MOUNT_POINT MIN_SIZE_GB +# MOUNT_POINT where the spare disk lands; enlarge_swap.sh and +# relocate_docker_dataroot.sh pick it up when it is /mnt +# MIN_SIZE_GB smallest spare disk worth mounting; smaller disks leave the runner as it is +set -euo pipefail + +MOUNT_POINT="${1:?MOUNT_POINT is required - state where the spare disk is mounted at the call site}" +MIN_SIZE_GB="${2:?MIN_SIZE_GB is required - state the smallest spare disk in GB worth mounting at the call site}" + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DUMP="${HERE}/dump_state.sh" + +if [ -n "${ACT:-}" ]; then + echo "::error::mount_spare_disk.sh refuses to run under act (ACT=${ACT}): it formats block devices and must never touch the operator's machine" >&2 + exit 1 +fi + +if ! [[ "${MIN_SIZE_GB}" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::MIN_SIZE_GB must be a positive integer, got '${MIN_SIZE_GB}'" >&2 + exit 1 +fi + +if [ "${RUNNER_ENVIRONMENT:-}" != github-hosted ]; then + echo "::notice::mount_spare_disk.sh mounted no spare disk: not a GitHub-hosted runner (RUNNER_ENVIRONMENT='${RUNNER_ENVIRONMENT:-}'), every block device stays untouched" + exit 0 +fi + +if mountpoint -q "${MOUNT_POINT}"; then + echo "::notice::mount_spare_disk.sh mounted no spare disk: ${MOUNT_POINT} is already a separate mount" + exit 0 +fi + +if [ ! -d "${MOUNT_POINT}" ]; then + echo "::error::mount point ${MOUNT_POINT} does not exist" >&2 + exit 1 +fi + +occupant="$(find "${MOUNT_POINT}" -mindepth 1 -maxdepth 1 -print -quit)" +if [ -n "${occupant}" ]; then + echo "::warning::${MOUNT_POINT} is not empty; mounting over it would hide its content, so no spare disk is mounted" + exit 0 +fi + +min_bytes=$((MIN_SIZE_GB * 1024 * 1024 * 1024)) +tree="$(lsblk -J -b -o NAME,TYPE,SIZE,RO,FSTYPE,PTTYPE,MOUNTPOINTS)" + +if ! jq -e ' + [.. | objects | select(has("name"))] + | all( + ((.mountpoints | type) == "array") + and ((.ro | type) == "boolean" or .ro == "0" or .ro == "1") + )' <<<"${tree}" >/dev/null; then + echo "::error::lsblk did not report mountpoints and ro for every device; refusing to pick a spare disk" >&2 + exit 1 +fi + +root_disks="$(jq -c '[ + .blockdevices[] + | select([.. | objects | .mountpoints[] | select(. == "/")] | length > 0) + | .name +]' <<<"${tree}")" +if [ "${root_disks}" = "[]" ]; then + echo "::error::no block device carries / according to lsblk; refusing to pick a spare disk" >&2 + exit 1 +fi + +disk="$(jq -r --argjson roots "${root_disks}" --argjson min "${min_bytes}" ' + [ + .blockdevices[] + | select(.type == "disk") + | select(.name | IN($roots[]) | not) + | select(.name | test("^(nvme[0-9]+n[0-9]+|sd[a-z]+|vd[a-z]+|xvd[a-z]+)$")) + | select(.ro == false or .ro == "0") + | select((.children // []) | length == 0) + | select((.fstype // "") == "" and (.pttype // "") == "") + | select([.mountpoints[] | select(. != null)] | length == 0) + | select((.size | tonumber) >= $min) + ] + | max_by(.size | tonumber) + | .name // empty' <<<"${tree}")" + +if [ -z "${disk}" ]; then + echo "::notice::mount_spare_disk.sh mounted no spare disk: No empty, unpartitioned, unmounted disk of at least ${MIN_SIZE_GB}G besides the root disk ${root_disks}" + exit 0 +fi + +probe_rc=0 +probe="$(sudo blkid -p "/dev/${disk}" 2>&1)" || probe_rc=$? +if [ "${probe_rc}" = 2 ] && [ -n "${probe}" ]; then + echo "::error::blkid -p /dev/${disk} found nothing but complained: ${probe}" >&2 + exit 1 +fi +case "${probe_rc}" in +2) ;; +0) + echo "::notice::mount_spare_disk.sh mounted no spare disk: /dev/${disk} carries a signature (${probe}), leaving it untouched" + exit 0 + ;; +*) + echo "::error::blkid -p /dev/${disk} could not tell whether the disk is empty (rc=${probe_rc}): ${probe}" >&2 + exit 1 + ;; +esac + +signatures="$(sudo wipefs --no-act --noheadings "/dev/${disk}")" +if [ -n "${signatures}" ]; then + echo "::notice::mount_spare_disk.sh mounted no spare disk: /dev/${disk} carries signatures blkid missed (${signatures}), leaving it untouched" + exit 0 +fi + +echo "Formatting the empty spare disk /dev/${disk} and mounting it at ${MOUNT_POINT}" +sudo mkfs.ext4 -q -m 0 -L infinito-spare "/dev/${disk}" +sudo mount "/dev/${disk}" "${MOUNT_POINT}" + +source_dev="$(findmnt -n -o SOURCE "${MOUNT_POINT}")" +if [ "${source_dev}" != "/dev/${disk}" ]; then + echo "::error::${MOUNT_POINT} is backed by '${source_dev}' after mounting /dev/${disk}" >&2 + exit 1 +fi + +"$DUMP" "after spare-disk mount -> /dev/${disk} at ${MOUNT_POINT}" diff --git a/scripts/github/runner/relocate_docker_dataroot.sh b/scripts/github/runner/relocate_docker_dataroot.sh index d83a7638c9..1296fcc5e7 100755 --- a/scripts/github/runner/relocate_docker_dataroot.sh +++ b/scripts/github/runner/relocate_docker_dataroot.sh @@ -1,98 +1,173 @@ #!/usr/bin/env bash -# Move the host Docker data-root onto /mnt when it is a separate, roomier -# partition. Image/volume storage (swarm's nested DinD-node daemons, the -# DR-drill LUKS loop image, per-node registry pulls) otherwise piles onto -# the ~50 GB root fs and trips the <6 GB abort guard in -# utils/tests/swarm/matrix.py. On public runners without a separate /mnt -# this is a no-op and Docker stays on /. -# -# Best-effort: relocation is an optimization, never a requirement. Any -# failure rolls Docker back onto / and the step still exits 0, so a hiccup -# here can never turn an otherwise-green deploy red. Runs BEFORE any Docker -# usage; the free-disk-space step already pruned the preinstalled images, so -# /var/lib/docker is ~empty and nothing is copied. -# -# Usage: relocate_docker_dataroot.sh [min-free-gb] -# Default floor: 40 GB (swarm peak footprint; below it /mnt buys nothing). -# -# Debug toggle: SKIP_DOCKER_RELOCATE=1 leaves the data-root on /. +# Usage: relocate_docker_dataroot.sh MIN_FREE_GB +# MIN_FREE_GB smallest free space on /mnt that makes the move worth it +# SKIP_DOCKER_RELOCATE=1 leave Docker and containerd on / set -euo pipefail +MIN_FREE_GB="${1:?MIN_FREE_GB is required - state the smallest free GB on /mnt worth relocating to at the call site}" + HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DUMP="${HERE}/dump_state.sh" +DATA_ROOT=/mnt/docker-host +CONTAINERD_ROOT=/var/lib/containerd +CONTAINERD_TARGET=/mnt/containerd +DAEMON_JSON=/etc/docker/daemon.json + +# Param: $1 why Docker and containerd stay on / +decline() { + echo "::notice::relocate_docker_dataroot.sh left Docker and containerd on /: $1" + "$DUMP" "docker-relocate skipped ($1)" + exit 0 +} + +# Param: $1 what was found that the relocation would lose or orphan +refuse() { + echo "::error::relocate_docker_dataroot.sh refuses to move Docker and containerd: $1" >&2 + exit 1 +} + if [ "${SKIP_DOCKER_RELOCATE:-0}" = "1" ]; then - echo "SKIP_DOCKER_RELOCATE=1 — leaving Docker data-root on /." - "$DUMP" "docker-relocate skipped (SKIP_DOCKER_RELOCATE=1)" + decline "SKIP_DOCKER_RELOCATE=1" +fi + +if [ -n "${ACT:-}" ]; then + echo "::error::relocate_docker_dataroot.sh refuses to run under act (ACT=${ACT}): it restarts Docker and containerd and must never touch the operator's machine" >&2 + exit 1 +fi + +if ! [[ "${MIN_FREE_GB}" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::MIN_FREE_GB must be a positive integer, got '${MIN_FREE_GB}'" >&2 + exit 1 +fi + +if [ "${RUNNER_ENVIRONMENT:-}" != github-hosted ]; then + echo "::notice::relocate_docker_dataroot.sh left Docker and containerd where they are: not a GitHub-hosted runner (RUNNER_ENVIRONMENT='${RUNNER_ENVIRONMENT:-}')" exit 0 fi -TARGET_DIR=/mnt/docker -MIN_FREE_GB="${1:-40}" - -if ! mountpoint -q /mnt 2>/dev/null; then - echo "/mnt is not a separate mount — leaving Docker data-root on /." - "$DUMP" "docker-relocate skipped (no separate /mnt)" - exit 0 +if ! mountpoint -q /mnt; then + decline "/mnt is not a separate mount" fi mnt_free=$(df --output=avail -B1G /mnt | tail -n1 | tr -d ' ') root_free=$(df --output=avail -B1G / | tail -n1 | tr -d ' ') echo "Free: / ${root_free}G, /mnt ${mnt_free}G; need >= ${MIN_FREE_GB}G on /mnt and roomier than /" -if [ "$mnt_free" -lt "$MIN_FREE_GB" ] || [ "$mnt_free" -le "$root_free" ]; then - echo "/mnt below floor or not roomier than / — leaving Docker data-root on /." - "$DUMP" "docker-relocate skipped (/mnt not roomier)" - exit 0 +if [ "$mnt_free" -lt "$MIN_FREE_GB" ]; then + decline "/mnt has ${mnt_free}G free, below ${MIN_FREE_GB}G" +fi +if [ "$mnt_free" -le "$root_free" ]; then + decline "/mnt has ${mnt_free}G free, not more than the ${root_free}G on /" fi -backup="" -if [ -s /etc/docker/daemon.json ]; then - backup="$(mktemp)" - cp /etc/docker/daemon.json "$backup" +if ! systemctl cat containerd.service >/dev/null; then + refuse "containerd.service is missing" fi -restore_and_continue() { - echo "WARN: Docker data-root relocation failed — keeping Docker on /." >&2 - if [ -n "$backup" ]; then - sudo install -m 0644 "$backup" /etc/docker/daemon.json || true # nocheck: shell-or-true -- rollback must reach the docker restart even if the restore fails - else - sudo rm -f /etc/docker/daemon.json || true # nocheck: shell-or-true -- rollback must reach the docker restart even if the cleanup fails - fi - sudo systemctl start docker.service || sudo systemctl restart docker.service || true # nocheck: shell-or-true -- last-ditch daemon start; rollback still exits 0 by design - "$DUMP" "docker-relocate rolled back (kept on /)" - exit 0 -} - -echo "Relocating Docker data-root to ${TARGET_DIR}" -sudo systemctl stop docker.service docker.socket || true # nocheck: shell-or-true -- inactive/missing units may exit non-zero; a wedged daemon is caught by the post-start root-dir check -sudo mkdir -p "$TARGET_DIR" /etc/docker || restore_and_continue - -tmp="$(mktemp)" -if [ -n "$backup" ]; then - jq --arg dr "$TARGET_DIR" '. + {"data-root": $dr}' "$backup" >"$tmp" || restore_and_continue -else - printf '{"data-root": "%s"}\n' "$TARGET_DIR" >"$tmp" +configured_root="$(sudo containerd config dump | sed -nE "s/^root = ['\"](.*)['\"]\$/\1/p")" +if [ "${configured_root}" != "${CONTAINERD_ROOT}" ]; then + refuse "containerd root is '${configured_root}', expected ${CONTAINERD_ROOT}" fi -sudo install -m 0644 "$tmp" /etc/docker/daemon.json || restore_and_continue -rm -f "$tmp" -sudo systemctl start docker.service || restore_and_continue +containers="$(docker ps -aq)" +[ -z "${containers}" ] || refuse "Docker containers exist: ${containers//$'\n'/ }" +images="$(docker images -q)" +[ -z "${images}" ] || refuse "Docker images exist: ${images//$'\n'/ }" +volumes="$(docker volume ls -q)" +[ -z "${volumes}" ] || refuse "Docker volumes exist: ${volumes//$'\n'/ }" +build_cache="$(docker system df --format '{{.Type}}={{.TotalCount}}' | sed -n 's/^Build Cache=//p')" +[ "${build_cache}" = 0 ] || refuse "the build cache holds '${build_cache}' records" -root_dir="" -attempt=0 -while [ "$attempt" -lt 30 ]; do - root_dir="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)" - [ -n "$root_dir" ] && break - attempt=$((attempt + 1)) - sleep 1 +namespaces="$(sudo ctr namespaces ls -q)" +for namespace in ${namespaces}; do + [ "${namespace}" != moby ] || continue + foreign="$(sudo ctr -n "${namespace}" containers ls -q)" + [ -z "${foreign}" ] || refuse "containerd namespace ${namespace} holds containers: ${foreign//$'\n'/ }" + tasks="$(sudo ctr -n "${namespace}" tasks ls -q)" + [ -z "${tasks}" ] || refuse "containerd namespace ${namespace} runs tasks: ${tasks//$'\n'/ }" done -if [ "$root_dir" != "$TARGET_DIR" ]; then - echo "Docker came up with root '${root_dir}', expected '${TARGET_DIR}'." >&2 - restore_and_continue +shim_rc=0 +shims="$(pgrep -a containerd-shim)" || shim_rc=$? +case "${shim_rc}" in +0) refuse "containerd shims are running: ${shims//$'\n'/; }" ;; +1) ;; +*) refuse "pgrep could not list containerd shims (rc=${shim_rc})" ;; +esac + +sudo true +daemon_backup="$(mktemp)" +daemon_new="$(mktemp)" +if sudo test -e "${DAEMON_JSON}"; then + daemon_existed=yes + # shellcheck disable=SC2024 + sudo cat "${DAEMON_JSON}" >"${daemon_backup}" + if [ -s "${daemon_backup}" ] && ! jq empty "${daemon_backup}"; then + refuse "${DAEMON_JSON} is not valid JSON" + fi +else + daemon_existed=no +fi +if [ -s "${daemon_backup}" ]; then + jq --arg dr "${DATA_ROOT}" '. + {"data-root": $dr}' "${daemon_backup}" >"${daemon_new}" +else + jq -n --arg dr "${DATA_ROOT}" '{"data-root": $dr}' >"${daemon_new}" fi -echo "Docker Root Dir: ${root_dir}" -[ -n "$backup" ] && rm -f "$backup" -"$DUMP" "after docker-relocate -> ${TARGET_DIR}" +left_behind="$(sudo du -sxh "${CONTAINERD_ROOT}")" +echo "Left behind on /: ${left_behind%%[[:space:]]*} in ${CONTAINERD_ROOT}" +echo "Relocating the Docker data-root to ${DATA_ROOT} and the containerd root to ${CONTAINERD_TARGET}" + +# Param: $1 what failed +rollback() { + trap - ERR INT TERM + set +e + echo "::error::relocation failed ($1); rolling Docker and containerd back onto /" >&2 + if ! sudo systemctl stop docker.socket docker.service containerd.service; then + echo "::error::could not stop Docker and containerd for the rollback" >&2 + fi + if mountpoint -q "${CONTAINERD_ROOT}" && ! sudo umount "${CONTAINERD_ROOT}"; then + echo "::error::could not unmount the bind over ${CONTAINERD_ROOT}" >&2 + fi + if [ "${daemon_existed}" = yes ]; then + if ! sudo install -m 0644 "${daemon_backup}" "${DAEMON_JSON}"; then + echo "::error::could not restore ${DAEMON_JSON}" >&2 + fi + elif ! sudo rm -f "${DAEMON_JSON}"; then + echo "::error::could not remove the ${DAEMON_JSON} this script wrote" >&2 + fi + if ! sudo systemctl start containerd.service docker.socket docker.service; then + echo "::error::Docker or containerd did not come back up" >&2 + fi + sudo systemctl status --no-pager containerd.service docker.service >&2 + sudo journalctl --no-pager -n 80 -u containerd.service -u docker.service >&2 + exit 1 +} + +trap 'rollback "command failed: ${BASH_COMMAND}"' ERR +trap 'rollback "interrupted"' INT TERM +sudo systemctl stop docker.socket docker.service containerd.service +sudo mkdir -p "${DATA_ROOT}" "${CONTAINERD_TARGET}" "$(dirname "${DAEMON_JSON}")" +sudo mount --bind "${CONTAINERD_TARGET}" "${CONTAINERD_ROOT}" +sudo install -m 0644 "${daemon_new}" "${DAEMON_JSON}" +sudo systemctl start containerd.service +sudo systemctl start docker.socket docker.service +sudo ctr version >/dev/null + +if ! sudo test -f "${CONTAINERD_TARGET}/io.containerd.metadata.v1.bolt/meta.db"; then + rollback "containerd came up without its metadata in ${CONTAINERD_TARGET}" +fi +root_dir="$(docker info --format '{{.DockerRootDir}}')" +if [ "${root_dir}" != "${DATA_ROOT}" ]; then + rollback "Docker came up with root '${root_dir}', expected '${DATA_ROOT}'" +fi +mnt_dev="$(stat -c %d /mnt)" +if [ "$(stat -c %d "${root_dir}")" != "${mnt_dev}" ]; then + rollback "Docker root ${root_dir} is not on the /mnt filesystem" +fi + +trap - ERR INT TERM +rm -f "${daemon_backup}" "${daemon_new}" +echo "Docker Root Dir: ${root_dir}; containerd root ${CONTAINERD_ROOT} bound from ${CONTAINERD_TARGET}" +"$DUMP" "after docker-relocate -> ${DATA_ROOT}, containerd -> ${CONTAINERD_TARGET}" diff --git a/scripts/tests/deploy/utils/filesystem/docker_dataroot.sh b/scripts/tests/deploy/utils/filesystem/docker_dataroot.sh index 7f810384fc..204f9d0413 100755 --- a/scripts/tests/deploy/utils/filesystem/docker_dataroot.sh +++ b/scripts/tests/deploy/utils/filesystem/docker_dataroot.sh @@ -25,7 +25,11 @@ SIZE="${3:?SIZE is required - state the loop image size (e.g. 30G) at the call s IDENTITY="${4:?IDENTITY is required - state what this machine is called at the call site}" MOUNT=/mnt/docker-fs -IMAGE=/var/tmp/docker-fs.img +if mountpoint -q /mnt; then + IMAGE=/mnt/docker-fs.img +else + IMAGE=/var/tmp/docker-fs.img +fi POOL="infinito_ci_$(printf '%s' "${IDENTITY}" | tr -dc 'A-Za-z0-9_.:-')" DAEMON=/etc/docker/daemon.json MISC_MAJOR=10 diff --git a/tests/unit/python/scripts/github/runner/__init__.py b/tests/unit/python/scripts/github/runner/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/scripts/github/runner/test_enlarge_swap_placement.py b/tests/unit/python/scripts/github/runner/test_enlarge_swap_placement.py new file mode 100644 index 0000000000..e3ddc8a5a4 --- /dev/null +++ b/tests/unit/python/scripts/github/runner/test_enlarge_swap_placement.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils.cache.files import PROJECT_ROOT + +SCRIPT = PROJECT_ROOT / "scripts" / "github" / "runner" / "enlarge_swap.sh" +SUDO_STUB = '#!/usr/bin/env bash\nprintf "%s\\n" "$*" >>"${STUB_LOG}"\n' +DF_STUB = ( + "#!/usr/bin/env bash\n" + 'echo "Avail"\n' + 'if [ "${*: -1}" = /mnt ]; then echo "${STUB_MNT_FREE}"; else echo "${STUB_ROOT_FREE}"; fi\n' +) +MOUNTPOINT_STUB = '#!/usr/bin/env bash\n[ "${STUB_MNT_MOUNTED}" = 1 ]\n' +STAT_STUB = ( + "#!/usr/bin/env bash\n" + 'case "${*: -1}" in\n' + "/mnt) echo 2049 ;;\n" + '/var/lib/containerd) if [ "${STUB_CONTAINERD_DEV}" = missing ]; then exit 1; fi\n' + ' echo "${STUB_CONTAINERD_DEV}" ;;\n' + "*) exit 1 ;;\n" + "esac\n" +) +DUMP_STUB = "#!/usr/bin/env bash\nexit 0\n" + + +def enlarge( + *, mounted: bool, mnt_free: int, root_free: int, containerd_dev: str +) -> tuple[subprocess.CompletedProcess, list[str]]: + """Run enlarge_swap.sh against stubbed disks. + + Args: + mounted: whether /mnt is a separate mount. + mnt_free: GB df reports free on /mnt. + root_free: GB df reports free on /. + containerd_dev: device id stat reports for /var/lib/containerd + (2049 is the /mnt device). + + Returns: + The finished process and every sudo call in order. + """ + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + runner_dir = root / "runner" + runner_dir.mkdir() + (runner_dir / "enlarge_swap.sh").symlink_to(SCRIPT) + bin_dir = root / "bin" + bin_dir.mkdir() + for path, body in ( + (runner_dir / "dump_state.sh", DUMP_STUB), + (bin_dir / "sudo", SUDO_STUB), + (bin_dir / "df", DF_STUB), + (bin_dir / "mountpoint", MOUNTPOINT_STUB), + (bin_dir / "stat", STAT_STUB), + ): + path.write_text(body) + path.chmod(0o755) + log = root / "sudo.log" + log.touch() + env = { + k: v for k, v in os.environ.items() if k not in ("BASH_ENV", "SKIP_SWAP") + } + env.update( + PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + STUB_LOG=str(log), + STUB_MNT_MOUNTED="1" if mounted else "0", + STUB_MNT_FREE=str(mnt_free), + STUB_ROOT_FREE=str(root_free), + STUB_CONTAINERD_DEV=containerd_dev, + ) + done = subprocess.run( + ["bash", str(runner_dir / "enlarge_swap.sh")], + env=env, + capture_output=True, + text=True, + check=False, + ) + return done, log.read_text().splitlines() # nocheck: cache-read stub call log + + +def swapfile(calls: list[str]) -> list[str]: + """Return the fallocate calls among the recorded sudo calls. + + Args: + calls: every recorded sudo call. + + Returns: + The calls that create the swapfile. + """ + return [call for call in calls if call.startswith("fallocate ")] + + +class TestEnlargeSwapPlacement(unittest.TestCase): + def test_swap_goes_to_root_when_containerd_lives_on_the_spare_disk(self) -> None: + done, calls = enlarge( + mounted=True, mnt_free=200, root_free=54, containerd_dev="2049" + ) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(swapfile(calls), ["fallocate -l 16G /swapfile"]) + + def test_swap_goes_to_mnt_when_containerd_stayed_on_root(self) -> None: + done, calls = enlarge( + mounted=True, mnt_free=60, root_free=54, containerd_dev="66305" + ) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(swapfile(calls), ["fallocate -l 16G /mnt/swapfile"]) + + def test_large_shape_without_mnt_keeps_swap_on_root(self) -> None: + done, calls = enlarge( + mounted=False, mnt_free=0, root_free=123, containerd_dev="66305" + ) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(swapfile(calls), ["fallocate -l 16G /swapfile"]) + + def test_missing_containerd_root_keeps_the_mnt_preference(self) -> None: + done, calls = enlarge( + mounted=True, mnt_free=60, root_free=54, containerd_dev="missing" + ) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(swapfile(calls), ["fallocate -l 16G /mnt/swapfile"]) + + def test_full_root_still_uses_the_spare_disk(self) -> None: + done, calls = enlarge( + mounted=True, mnt_free=200, root_free=10, containerd_dev="2049" + ) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(swapfile(calls), ["fallocate -l 16G /mnt/swapfile"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/scripts/github/runner/test_mount_spare_disk.py b/tests/unit/python/scripts/github/runner/test_mount_spare_disk.py new file mode 100644 index 0000000000..a22fb980ba --- /dev/null +++ b/tests/unit/python/scripts/github/runner/test_mount_spare_disk.py @@ -0,0 +1,457 @@ +from __future__ import annotations + +import copy +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils.cache.files import PROJECT_ROOT +from utils.cache.yaml import load_yaml + +SCRIPT = PROJECT_ROOT / "scripts" / "github" / "runner" / "mount_spare_disk.sh" +WORKFLOW = PROJECT_ROOT / ".github" / "workflows" / "call-test-deploy.yml" +MIN_SIZE_GB = "75" +GIB = 1024**3 +SPARE = "nvme9n9" +SPARE_INDEX = 2 +DESTRUCTIVE = ("mkfs.ext4", "blkid", "wipefs", "mount", "mkswap") + +SUDO_STUB = ( + "#!/usr/bin/env bash\n" + 'printf "%s\\n" "$*" >>"${STUB_LOG}"\n' + 'if [ -x "${STUB_BIN}/$1" ]; then exec "${STUB_BIN}/$1" "${@:2}"; fi\n' + 'echo "unstubbed sudo $1" >&2\n' + "exit 97\n" +) +UNPRIVILEGED_STUB = ( + '#!/usr/bin/env bash\necho "$(basename "$0") ran without sudo" >&2\nexit 98\n' +) +LSBLK_STUB = '#!/usr/bin/env bash\ncat "${STUB_LSBLK_JSON}"\n' +MOUNTPOINT_STUB = '#!/usr/bin/env bash\n[ "${STUB_MOUNTED}" = 1 ]\n' +BLKID_STUB = ( + '#!/usr/bin/env bash\nprintf "%s" "${STUB_BLKID_OUT}"\nexit "${STUB_BLKID_RC}"\n' +) +WIPEFS_STUB = '#!/usr/bin/env bash\nprintf "%s" "${STUB_WIPEFS_OUT}"\n' +MKFS_STUB = "#!/usr/bin/env bash\nexit 0\n" +MOUNT_STUB = ( + '#!/usr/bin/env bash\nprintf "%s" "${STUB_MOUNT_AS:-$1}" >"${STUB_STATE}"\n' +) +FINDMNT_STUB = '#!/usr/bin/env bash\ncat "${STUB_STATE}"\n' +DUMP_STUB = "#!/usr/bin/env bash\nexit 0\n" + + +def part(name: str, size_gib: float, fstype: str | None, mount: str | None) -> dict: + """Build one lsblk partition entry. + + Args: + name: kernel name of the partition. + size_gib: size in GiB. + fstype: filesystem type, None when unformatted. + mount: mount point, None when unmounted. + + Returns: + The entry as lsblk -J -b prints it. + """ + return { + "name": name, + "type": "part", + "size": int(size_gib * GIB), + "ro": False, + "fstype": fstype, + "pttype": "gpt", + "mountpoints": [mount], + } + + +def snap_loops() -> list[dict]: + """Return the two snap squashfs loop devices every runner shape carries. + + Returns: + The loop entries as lsblk -J -b prints them. + """ + return [ + { + "name": f"loop{index}", + "type": "loop", + "size": 77594624, + "ro": True, + "fstype": "squashfs", + "pttype": None, + "mountpoints": [mount], + } + for index, mount in enumerate(("/snap/core22/2955", "/snap/snapd/28254")) + ] + + +def root_disk(name: str, prefix: str, size_gib: int) -> dict: + """Build the partitioned root disk of a GitHub runner image. + + Args: + name: kernel name of the disk. + prefix: partition name prefix (``sda`` or ``nvme0n1p``). + size_gib: size in GiB. + + Returns: + The disk entry with its four partitions. + """ + return { + "name": name, + "type": "disk", + "size": size_gib * GIB, + "ro": False, + "fstype": None, + "pttype": "gpt", + "mountpoints": [None], + "children": [ + part(f"{prefix}1", size_gib - 1.2, "ext4", "/"), + part(f"{prefix}13", 1, "ext4", "/boot"), + part(f"{prefix}14", 0.004, None, None), + part(f"{prefix}15", 0.1, "vfat", "/boot/efi"), + ], + } + + +def spare_disk(size_gib: int) -> dict: + """Build the empty second NVMe disk of the small runner shape. + + Args: + size_gib: size in GiB. + + Returns: + The disk entry without filesystem, partition table or mount. + """ + return { + "name": SPARE, + "type": "disk", + "size": size_gib * GIB, + "ro": False, + "fstype": None, + "pttype": None, + "mountpoints": [None], + } + + +LARGE_SHAPE = {"blockdevices": [*snap_loops(), root_disk("sda", "sda", 145)]} +SMALL_SHAPE = { + "blockdevices": [ + *snap_loops(), + spare_disk(128), + root_disk("nvme0n1", "nvme0n1p", 72), + ] +} + + +def with_spare(**changes: object) -> dict: + """Return the small shape with fields of the spare disk replaced. + + Args: + **changes: lsblk fields to overwrite on the spare disk. + + Returns: + A modified deep copy of SMALL_SHAPE. + """ + tree = copy.deepcopy(SMALL_SHAPE) + tree["blockdevices"][SPARE_INDEX].update(changes) + return tree + + +def run( + tree: dict, + *, + runner_environment: str | None = "github-hosted", + act: str | None = None, + mounted: bool = False, + blkid_rc: int = 2, + blkid_out: str = "", + wipefs_out: str = "", + mount_as: str | None = None, + occupied: bool = False, + min_size_gb: str = MIN_SIZE_GB, +) -> tuple[subprocess.CompletedProcess, list[str]]: + """Run mount_spare_disk.sh against a faked block-device layout. + + Args: + tree: what ``lsblk -J -b`` reports. + runner_environment: RUNNER_ENVIRONMENT, None to leave it unset. + act: ACT, None to leave it unset. + mounted: whether the mount point already is a mount. + blkid_rc: exit code of ``blkid -p`` (2 means no signature). + blkid_out: what ``blkid -p`` prints. + wipefs_out: what ``wipefs --no-act`` prints. + mount_as: device findmnt reports after the mount, None for the mounted one. + occupied: whether the mount point already holds a file. + min_size_gb: the MIN_SIZE_GB argument. + + Returns: + The finished process and every privileged call in order. + """ + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + runner_dir = root / "runner" + runner_dir.mkdir() + (runner_dir / "mount_spare_disk.sh").symlink_to(SCRIPT) + bin_dir = root / "bin" + bin_dir.mkdir() + stub_bin = root / "privileged" + stub_bin.mkdir() + stubs = [ + (runner_dir / "dump_state.sh", DUMP_STUB), + (bin_dir / "sudo", SUDO_STUB), + (bin_dir / "lsblk", LSBLK_STUB), + (bin_dir / "mountpoint", MOUNTPOINT_STUB), + (bin_dir / "findmnt", FINDMNT_STUB), + (stub_bin / "blkid", BLKID_STUB), + (stub_bin / "wipefs", WIPEFS_STUB), + (stub_bin / "mkfs.ext4", MKFS_STUB), + (stub_bin / "mount", MOUNT_STUB), + ] + stubs += [(bin_dir / name, UNPRIVILEGED_STUB) for name in DESTRUCTIVE] + for path, body in stubs: + path.write_text(body) + path.chmod(0o755) + mount_point = root / "mnt" + mount_point.mkdir() + if occupied: + (mount_point / "keep.txt").write_text("data\n") + lsblk_json = root / "lsblk.json" + lsblk_json.write_text(json.dumps(tree)) + log = root / "sudo.log" + log.touch() + env = { + k: v + for k, v in os.environ.items() + if k not in ("BASH_ENV", "ACT", "RUNNER_ENVIRONMENT") + } + env.update( + PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + STUB_LOG=str(log), + STUB_BIN=str(stub_bin), + STUB_LSBLK_JSON=str(lsblk_json), + STUB_MOUNTED="1" if mounted else "0", + STUB_BLKID_RC=str(blkid_rc), + STUB_BLKID_OUT=blkid_out, + STUB_WIPEFS_OUT=wipefs_out, + STUB_STATE=str(root / "mounted-source"), + ) + if mount_as is not None: + env["STUB_MOUNT_AS"] = mount_as + if runner_environment is not None: + env["RUNNER_ENVIRONMENT"] = runner_environment + if act is not None: + env["ACT"] = act + done = subprocess.run( + [ + "bash", + str(runner_dir / "mount_spare_disk.sh"), + str(mount_point), + min_size_gb, + ], + env=env, + capture_output=True, + text=True, + check=False, + ) + return done, log.read_text().splitlines() # nocheck: cache-read stub call log + + +def formatted(calls: list[str]) -> bool: + """Tell whether any privileged call formatted or mounted a disk. + + Args: + calls: every recorded sudo call. + + Returns: + True when mkfs or mount ran. + """ + return any(call.startswith(("mkfs", "mount ")) for call in calls) + + +class TestMountSpareDisk(unittest.TestCase): + def assert_untouched(self, done: subprocess.CompletedProcess, reason: str) -> None: + """Assert a declined run: exit 0, no privileged call, the stated reason. + + Args: + done: the finished script. + reason: text the script must print for this decline. + """ + self.assertEqual(done.returncode, 0, done.stderr) + self.assertIn(reason, done.stdout + done.stderr) + self.assertRegex(done.stdout, r"::(notice|warning)::") + + def test_small_shape_formats_and_mounts_the_empty_nvme_disk(self) -> None: + done, calls = run(SMALL_SHAPE) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual( + calls[:3], + [ + f"blkid -p /dev/{SPARE}", + f"wipefs --no-act --noheadings /dev/{SPARE}", + f"mkfs.ext4 -q -m 0 -L infinito-spare /dev/{SPARE}", + ], + ) + self.assertTrue(calls[3].startswith(f"mount /dev/{SPARE} "), calls) + self.assertEqual(len(calls), 4, calls) + + def test_large_shape_without_a_spare_disk_stays_as_it_is(self) -> None: + done, calls = run(LARGE_SHAPE) + self.assert_untouched(done, '["sda"]') + self.assertEqual(calls, []) + + def test_self_hosted_runner_is_never_formatted(self) -> None: + done, calls = run(SMALL_SHAPE, runner_environment="self-hosted") + self.assert_untouched(done, "not a GitHub-hosted runner") + self.assertEqual(calls, []) + + def test_unknown_runner_environment_is_never_formatted(self) -> None: + done, calls = run(SMALL_SHAPE, runner_environment=None) + self.assert_untouched(done, "not a GitHub-hosted runner") + self.assertEqual(calls, []) + + def test_act_refuses_loudly_even_with_a_faked_github_runner(self) -> None: + for act in ("true", "1", "TRUE"): + with self.subTest(act=act): + done, calls = run(SMALL_SHAPE, act=act) + self.assertEqual(done.returncode, 1) + self.assertEqual(calls, []) + self.assertIn("refuses to run under act", done.stderr) + + def test_disk_with_a_filesystem_is_never_formatted(self) -> None: + done, calls = run(with_spare(fstype="ext4")) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_disk_with_partitions_is_never_formatted(self) -> None: + done, calls = run(with_spare(children=[part(f"{SPARE}p1", 127, None, None)])) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_disk_held_by_lvm_is_never_formatted(self) -> None: + holder = { + "name": "vg-data", + "type": "lvm", + "size": 127 * GIB, + "ro": False, + "fstype": None, + "pttype": None, + "mountpoints": [None], + } + done, calls = run(with_spare(children=[holder])) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_disk_with_a_bare_partition_table_is_never_formatted(self) -> None: + done, calls = run(with_spare(pttype="dos")) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_mounted_disk_is_never_formatted(self) -> None: + done, calls = run(with_spare(mountpoints=["/data"])) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_disk_used_as_swap_is_never_formatted(self) -> None: + done, calls = run(with_spare(mountpoints=["[SWAP]"])) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_read_only_disk_is_never_formatted(self) -> None: + done, calls = run(with_spare(ro=True)) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_disk_below_the_size_floor_stays_as_it_is(self) -> None: + done, calls = run(with_spare(size=32 * GIB)) + self.assert_untouched(done, "No empty, unpartitioned, unmounted disk") + self.assertEqual(calls, []) + + def test_signature_lsblk_missed_is_never_formatted(self) -> None: + done, calls = run(SMALL_SHAPE, blkid_rc=0, blkid_out='TYPE="xfs"') + self.assert_untouched(done, "carries a signature") + self.assertFalse(formatted(calls), calls) + + def test_signature_only_wipefs_sees_is_never_formatted(self) -> None: + done, calls = run(SMALL_SHAPE, wipefs_out="0x218 LVM2_member") + self.assert_untouched(done, "carries signatures blkid missed") + self.assertFalse(formatted(calls), calls) + + def test_inconclusive_probe_fails_loudly_without_formatting(self) -> None: + done, calls = run(SMALL_SHAPE, blkid_rc=8, blkid_out="ambivalent result") + self.assertEqual(done.returncode, 1) + self.assertFalse(formatted(calls), calls) + + def test_probe_that_found_nothing_but_complained_fails_loudly(self) -> None: + done, calls = run(SMALL_SHAPE, blkid_rc=2, blkid_out="Input/output error") + self.assertEqual(done.returncode, 1) + self.assertFalse(formatted(calls), calls) + + def test_lsblk_without_mountpoints_fails_loudly(self) -> None: + tree = copy.deepcopy(SMALL_SHAPE) + spare = tree["blockdevices"][SPARE_INDEX] + spare["mountpoint"] = spare.pop("mountpoints")[0] + done, calls = run(tree) + self.assertEqual(done.returncode, 1) + self.assertEqual(calls, []) + + def test_layout_without_a_root_device_fails_loudly(self) -> None: + done, calls = run({"blockdevices": [spare_disk(128)]}) + self.assertEqual(done.returncode, 1) + self.assertEqual(calls, []) + + def test_non_numeric_size_floor_fails_loudly(self) -> None: + done, calls = run(SMALL_SHAPE, min_size_gb="RANDOM") + self.assertEqual(done.returncode, 1) + self.assertEqual(calls, []) + + def test_existing_mount_stays_as_it_is(self) -> None: + done, calls = run(SMALL_SHAPE, mounted=True) + self.assert_untouched(done, "already a separate mount") + self.assertEqual(calls, []) + + def test_occupied_mount_point_stays_as_it_is(self) -> None: + done, calls = run(SMALL_SHAPE, occupied=True) + self.assert_untouched(done, "is not empty") + self.assertEqual(calls, []) + + def test_mount_landing_on_another_device_fails_loudly(self) -> None: + done, _ = run(SMALL_SHAPE, mount_as="/dev/nvme0n1p1") + self.assertEqual(done.returncode, 1) + self.assertIn("backed by '/dev/nvme0n1p1'", done.stderr) + + +class TestWorkflowWiring(unittest.TestCase): + def test_spare_disk_step_is_act_gated_and_runs_before_swap_and_relocation( + self, + ) -> None: + workflow = load_yaml(str(WORKFLOW)) + steps = next( + job["steps"] + for job in workflow["jobs"].values() + if any( + "mount_spare_disk.sh" in str(step.get("run", "")) + for step in job.get("steps", []) + ) + ) + runs = [str(step.get("run", "")) for step in steps] + index = next(i for i, cmd in enumerate(runs) if "mount_spare_disk.sh" in cmd) + self.assertEqual( + runs[index], + f"./scripts/github/runner/mount_spare_disk.sh /mnt {MIN_SIZE_GB}", + ) + self.assertEqual(steps[index].get("if"), "${{ !env.ACT }}") + swap = next(i for i, cmd in enumerate(runs) if "enlarge_swap.sh" in cmd) + relocate = next( + i for i, cmd in enumerate(runs) if "relocate_docker_dataroot.sh" in cmd + ) + self.assertLess(index, relocate) + self.assertLess(relocate, swap) + self.assertEqual( + runs[relocate], "./scripts/github/runner/relocate_docker_dataroot.sh 75" + ) + self.assertEqual(steps[relocate].get("if"), "${{ !env.ACT }}") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/scripts/github/runner/test_relocate_docker_dataroot.py b/tests/unit/python/scripts/github/runner/test_relocate_docker_dataroot.py new file mode 100644 index 0000000000..f60d9ab24a --- /dev/null +++ b/tests/unit/python/scripts/github/runner/test_relocate_docker_dataroot.py @@ -0,0 +1,524 @@ +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import unittest +from dataclasses import dataclass, field +from pathlib import Path + +from utils.cache.files import PROJECT_ROOT + +SCRIPT = PROJECT_ROOT / "scripts" / "github" / "runner" / "relocate_docker_dataroot.sh" +MIN_FREE_GB = "75" +DUMP_V2 = "version = 3\nroot = '/var/lib/containerd'\nstate = '/run/containerd'\n" +DUMP_V1 = 'version = 2\nroot = "/var/lib/containerd"\nstate = "/run/containerd"\n' +STOP = "systemctl stop docker.socket docker.service containerd.service" +BIND = "mount --bind /mnt/containerd /var/lib/containerd" +UNBIND = "umount /var/lib/containerd" +RESTART = "systemctl start containerd.service docker.socket docker.service" + +SUDO_STUB = ( + "#!/usr/bin/env bash\n" + 'printf "%s\\n" "$*" >>"${STUB_LOG}"\n' + 'if [ -x "${STUB_BIN}/$1" ]; then exec "${STUB_BIN}/$1" "${@:2}"; fi\n' + 'echo "unstubbed sudo $1" >&2\n' + "exit 97\n" +) +UNPRIVILEGED_STUB = ( + '#!/usr/bin/env bash\necho "$(basename "$0") ran without sudo" >&2\nexit 98\n' +) +TERM_HOOK = ( + 'if [ "$(basename "$0")" = "${STUB_TERM_ON}" ]; then kill -TERM "${PPID}"; fi\n' +) +UNPRIVILEGED = { + "mountpoint": ( + "#!/usr/bin/env bash\n" + 'case "${*: -1}" in\n' + '/mnt) [ "${STUB_MNT_MOUNTED}" = 1 ] ;;\n' + '/var/lib/containerd) [ -e "${STUB_STATE}/bound" ] ;;\n' + "*) exit 1 ;;\n" + "esac\n" + ), + "df": ( + "#!/usr/bin/env bash\n" + 'echo "Avail"\n' + 'if [ "${*: -1}" = /mnt ]; then echo "${STUB_MNT_FREE}"; ' + 'else echo "${STUB_ROOT_FREE}"; fi\n' + ), + "systemctl": '#!/usr/bin/env bash\nexit "${STUB_CONTAINERD_UNIT_RC}"\n', + "docker": ( + "#!/usr/bin/env bash\n" + 'case "$1" in\n' + 'ps) printf "%s" "${STUB_CONTAINERS}" ;;\n' + 'images) printf "%s" "${STUB_IMAGES}" ;;\n' + 'volume) printf "%s" "${STUB_VOLUMES}" ;;\n' + 'system) printf "Images=0\\nContainers=0\\nLocal Volumes=0\\n' + 'Build Cache=%s\\n" "${STUB_BUILD_CACHE}" ;;\n' + 'info) echo "${STUB_DOCKER_ROOT}" ;;\n' + "*) exit 96 ;;\n" + "esac\n" + ), + "stat": ( + "#!/usr/bin/env bash\n" + 'case "${*: -1}" in\n' + "/mnt) echo 2049 ;;\n" + '/mnt/docker-host) echo "${STUB_DOCKER_ROOT_DEV}" ;;\n' + "*) echo 66305 ;;\n" + "esac\n" + ), + "pgrep": '#!/usr/bin/env bash\nprintf "%s" "${STUB_SHIMS}"\nexit "${STUB_SHIM_RC}"\n', +} +PRIVILEGED = { + "systemctl": ( + "#!/usr/bin/env bash\n" + 'if [ "$1" = status ]; then exit "${STUB_STATUS_RC}"; fi\n' + 'if [ "$*" = "${STUB_FAIL_SYSTEMCTL}" ]; then exit 1; fi\n' + ), + "containerd": '#!/usr/bin/env bash\nprintf "%s" "${STUB_CONTAINERD_DUMP}"\n', + "ctr": ( + "#!/usr/bin/env bash\n" + 'if [ "$1" = namespaces ]; then printf "%s" "${STUB_NAMESPACES}"; exit; fi\n' + 'if [ "$1" = -n ] && [ "$2" = "${STUB_FOREIGN_NS}" ]; then\n' + ' if [ "$3" = containers ]; then printf "%s" "${STUB_FOREIGN_CONTAINERS}"; fi\n' + ' if [ "$3" = tasks ]; then printf "%s" "${STUB_FOREIGN_TASKS}"; fi\n' + "fi\n" + ), + "du": '#!/usr/bin/env bash\nprintf "1.2G\\t%s\\n" "${*: -1}"\n', + "true": '#!/usr/bin/env bash\nexit "${STUB_SUDO_RC}"\n', + "test": ( + "#!/usr/bin/env bash\n" + 'if [ "$1" = -e ]; then [ "${STUB_DAEMON_EXISTS}" = 1 ]; exit; fi\n' + '[ "${STUB_METADATA}" = 1 ]\n' + ), + "cat": ( + "#!/usr/bin/env bash\n" + 'if [ "${STUB_DAEMON_CAT_RC}" != 0 ]; then exit "${STUB_DAEMON_CAT_RC}"; fi\n' + 'printf "%s" "${STUB_DAEMON_IN}"\n' + ), + "mkdir": "#!/usr/bin/env bash\n" + TERM_HOOK, + "mount": ( + "#!/usr/bin/env bash\n" + 'if [ "${STUB_MOUNT_RC}" != 0 ]; then exit "${STUB_MOUNT_RC}"; fi\n' + 'touch "${STUB_STATE}/bound"\n' + ), + "umount": '#!/usr/bin/env bash\nrm -f "${STUB_STATE}/bound"\n', + "install": '#!/usr/bin/env bash\ncp "${*: -2:1}" "${STUB_DAEMON_OUT}"\n', + "rm": '#!/usr/bin/env bash\n/bin/rm -f "${STUB_DAEMON_OUT}"\n', + "journalctl": "#!/usr/bin/env bash\nexit 0\n", +} +DESTRUCTIVE = ("mount", "umount", "install", "ctr", "containerd", "journalctl", "mkdir") + + +@dataclass +class Runner: + """The faked runner state relocate_docker_dataroot.sh meets. + + Args: + runner_environment: RUNNER_ENVIRONMENT, None to leave it unset. + act: ACT, None to leave it unset. + skip: SKIP_DOCKER_RELOCATE, None to leave it unset. + mounted: whether /mnt is a separate mount. + mnt_free: GB df reports free on /mnt. + root_free: GB df reports free on /. + dump: what ``containerd config dump`` prints. + containerd_unit_rc: exit code of ``systemctl cat containerd.service``. + containers: what ``docker ps -aq`` prints. + images: what ``docker images -q`` prints. + volumes: what ``docker volume ls -q`` prints. + build_cache: Build Cache record count in ``docker system df``. + namespaces: what ``ctr namespaces ls -q`` prints. + foreign_ns: namespace whose containers and tasks are faked. + foreign_containers: containers in foreign_ns. + foreign_tasks: tasks in foreign_ns. + shims: what ``pgrep -a containerd-shim`` prints. + shim_rc: exit code of pgrep (1 means no shim). + sudo_rc: exit code of ``sudo true``. + daemon_exists: whether daemon.json exists. + daemon_in: content of the existing daemon.json. + daemon_cat_rc: exit code of ``sudo cat`` on daemon.json. + mount_rc: exit code of the bind mount. + fail_systemctl: the one systemctl argument string that fails. + status_rc: exit code of ``systemctl status``. + term_on: privileged stub that sends SIGTERM to the script. + metadata: whether containerd's meta.db shows up under /mnt/containerd. + docker_root: what ``docker info`` reports as DockerRootDir. + docker_root_dev: device id stat reports for /mnt/docker-host. + min_free_gb: the MIN_FREE_GB argument. + """ + + runner_environment: str | None = "github-hosted" + act: str | None = None + skip: str | None = None + mounted: bool = True + mnt_free: int = 137 + root_free: int = 51 + dump: str = DUMP_V2 + containerd_unit_rc: int = 0 + containers: str = "" + images: str = "" + volumes: str = "" + build_cache: str = "0" + namespaces: str = "moby\n" + foreign_ns: str = "k8s.io" + foreign_containers: str = "" + foreign_tasks: str = "" + shims: str = "" + shim_rc: int = 1 + sudo_rc: int = 0 + daemon_exists: bool = False + daemon_in: str = "" + daemon_cat_rc: int = 0 + mount_rc: int = 0 + fail_systemctl: str = "" + status_rc: int = 0 + term_on: str = "" + metadata: bool = True + docker_root: str = "/mnt/docker-host" + docker_root_dev: str = "2049" + min_free_gb: str = MIN_FREE_GB + extra: dict[str, str] = field(default_factory=dict) + + +@dataclass +class Outcome: + """What a relocation run left behind. + + Args: + done: the finished script. + calls: every privileged call in order. + daemon: the daemon.json on disk afterwards, None when there is none + or it is no valid JSON. + daemon_raw: the daemon.json bytes on disk afterwards, None when there is none. + bound: whether the containerd bind mount is still in place. + """ + + done: subprocess.CompletedProcess + calls: list[str] + daemon: dict | None + daemon_raw: str | None + bound: bool + + +def relocate(runner: Runner) -> Outcome: + """Run relocate_docker_dataroot.sh against a faked runner. + + Args: + runner: the faked services, disks and files. + + Returns: + The outcome of the run. + """ + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + runner_dir = root / "runner" + bin_dir = root / "bin" + stub_bin = root / "privileged" + state = root / "state" + for directory in (runner_dir, bin_dir, stub_bin, state): + directory.mkdir() + (runner_dir / "relocate_docker_dataroot.sh").symlink_to(SCRIPT) + stubs = [(runner_dir / "dump_state.sh", "#!/usr/bin/env bash\nexit 0\n")] + stubs += [(bin_dir / "sudo", SUDO_STUB)] + stubs += [(bin_dir / name, UNPRIVILEGED_STUB) for name in DESTRUCTIVE] + stubs += [(bin_dir / name, body) for name, body in UNPRIVILEGED.items()] + stubs += [(stub_bin / name, body) for name, body in PRIVILEGED.items()] + for path, body in stubs: + path.write_text(body) + path.chmod(0o755) + log = root / "calls.log" + log.touch() + daemon_out = root / "daemon.json" + if runner.daemon_exists: + daemon_out.write_text(runner.daemon_in) + env = { + k: v + for k, v in os.environ.items() + if k + not in ("BASH_ENV", "ACT", "RUNNER_ENVIRONMENT", "SKIP_DOCKER_RELOCATE") + } + env.update( + PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + STUB_LOG=str(log), + STUB_BIN=str(stub_bin), + STUB_STATE=str(state), + STUB_MNT_MOUNTED="1" if runner.mounted else "0", + STUB_MNT_FREE=str(runner.mnt_free), + STUB_ROOT_FREE=str(runner.root_free), + STUB_CONTAINERD_DUMP=runner.dump, + STUB_CONTAINERD_UNIT_RC=str(runner.containerd_unit_rc), + STUB_CONTAINERS=runner.containers, + STUB_IMAGES=runner.images, + STUB_VOLUMES=runner.volumes, + STUB_BUILD_CACHE=runner.build_cache, + STUB_NAMESPACES=runner.namespaces, + STUB_FOREIGN_NS=runner.foreign_ns, + STUB_FOREIGN_CONTAINERS=runner.foreign_containers, + STUB_FOREIGN_TASKS=runner.foreign_tasks, + STUB_SHIMS=runner.shims, + STUB_SHIM_RC=str(runner.shim_rc), + STUB_SUDO_RC=str(runner.sudo_rc), + STUB_DAEMON_EXISTS="1" if runner.daemon_exists else "0", + STUB_DAEMON_IN=runner.daemon_in, + STUB_DAEMON_CAT_RC=str(runner.daemon_cat_rc), + STUB_DAEMON_OUT=str(daemon_out), + STUB_MOUNT_RC=str(runner.mount_rc), + STUB_FAIL_SYSTEMCTL=runner.fail_systemctl, + STUB_STATUS_RC=str(runner.status_rc), + STUB_TERM_ON=runner.term_on, + STUB_METADATA="1" if runner.metadata else "0", + STUB_DOCKER_ROOT=runner.docker_root, + STUB_DOCKER_ROOT_DEV=runner.docker_root_dev, + **runner.extra, + ) + optional = { + "RUNNER_ENVIRONMENT": runner.runner_environment, + "ACT": runner.act, + "SKIP_DOCKER_RELOCATE": runner.skip, + } + env.update({k: v for k, v in optional.items() if v is not None}) + done = subprocess.run( + [ + "bash", + str(runner_dir / "relocate_docker_dataroot.sh"), + runner.min_free_gb, + ], + env=env, + capture_output=True, + text=True, + check=False, + ) + calls = log.read_text().splitlines() # nocheck: cache-read stub call log + content = ( + daemon_out.read_text() # nocheck: cache-read stub output + if daemon_out.exists() + else None + ) + try: + daemon = json.loads(content) if content else None + except json.JSONDecodeError: + daemon = None + return Outcome( + done=done, + calls=calls, + daemon=daemon, + daemon_raw=content, + bound=(state / "bound").exists(), + ) + + +class TestRelocation(unittest.TestCase): + def test_small_shape_moves_containerd_and_docker_onto_the_spare_disk(self) -> None: + out = relocate(Runner()) + self.assertEqual(out.done.returncode, 0, out.done.stderr) + stop, bind = out.calls.index(STOP), out.calls.index(BIND) + start = out.calls.index("systemctl start containerd.service") + self.assertLess(stop, bind) + self.assertLess(bind, start) + self.assertIn("systemctl start docker.socket docker.service", out.calls) + self.assertIn( + "test -f /mnt/containerd/io.containerd.metadata.v1.bolt/meta.db", out.calls + ) + self.assertEqual(out.daemon, {"data-root": "/mnt/docker-host"}) + self.assertTrue(out.bound) + + def test_containerd_1x_dump_with_double_quotes_is_understood(self) -> None: + out = relocate(Runner(dump=DUMP_V1)) + self.assertEqual(out.done.returncode, 0, out.done.stderr) + self.assertIn(BIND, out.calls) + + def test_existing_daemon_json_keeps_its_settings(self) -> None: + out = relocate(Runner(daemon_exists=True, daemon_in='{"debug": true}')) + self.assertEqual(out.done.returncode, 0, out.done.stderr) + self.assertEqual(out.daemon, {"debug": True, "data-root": "/mnt/docker-host"}) + + +class TestDeclines(unittest.TestCase): + def assert_declined(self, out: Outcome, reason: str) -> None: + """Assert a visible no-op: exit 0, a notice with the reason, no call. + + Args: + out: the outcome of the run. + reason: text the notice must carry. + """ + self.assertEqual(out.done.returncode, 0, out.done.stderr) + self.assertIn("::notice::", out.done.stdout) + self.assertIn(reason, out.done.stdout) + self.assertEqual(out.calls, []) + + def test_large_shape_without_a_mnt_mount(self) -> None: + self.assert_declined( + relocate(Runner(mounted=False)), "/mnt is not a separate mount" + ) + + def test_self_hosted_runner(self) -> None: + self.assert_declined( + relocate(Runner(runner_environment="self-hosted")), + "not a GitHub-hosted runner", + ) + + def test_unknown_runner_environment(self) -> None: + self.assert_declined( + relocate(Runner(runner_environment=None)), "not a GitHub-hosted runner" + ) + + def test_mnt_below_the_floor(self) -> None: + self.assert_declined(relocate(Runner(mnt_free=60, root_free=20)), "below 75G") + + def test_mnt_not_roomier_than_root(self) -> None: + self.assert_declined( + relocate(Runner(mnt_free=80, root_free=90)), "not more than" + ) + + def test_skip_toggle(self) -> None: + self.assert_declined(relocate(Runner(skip="1")), "SKIP_DOCKER_RELOCATE=1") + + +class TestRefusalsBeforeTheStop(unittest.TestCase): + def assert_refused(self, out: Outcome, reason: str) -> None: + """Assert a loud refusal before anything was stopped or written. + + Args: + out: the outcome of the run. + reason: text the error must carry. + """ + self.assertEqual(out.done.returncode, 1) + self.assertIn(reason, out.done.stderr) + self.assertNotIn(STOP, out.calls) + self.assertFalse([c for c in out.calls if c.startswith(("mount ", "install "))]) + self.assertFalse(out.bound) + + def test_act_even_with_a_faked_github_runner(self) -> None: + for act in ("true", "1"): + with self.subTest(act=act): + self.assert_refused( + relocate(Runner(act=act)), "refuses to run under act" + ) + + def test_non_numeric_floor(self) -> None: + self.assert_refused(relocate(Runner(min_free_gb="forty")), "positive integer") + + def test_missing_containerd_unit(self) -> None: + self.assert_refused( + relocate(Runner(containerd_unit_rc=1)), "containerd.service is missing" + ) + + def test_containerd_root_elsewhere(self) -> None: + self.assert_refused( + relocate(Runner(dump="version = 3\nroot = '/data/containerd'\n")), + "/data/containerd", + ) + + def test_existing_containers(self) -> None: + self.assert_refused(relocate(Runner(containers="abc123\n")), "abc123") + + def test_existing_images(self) -> None: + self.assert_refused(relocate(Runner(images="sha256feed\n")), "sha256feed") + + def test_existing_volumes(self) -> None: + self.assert_refused( + relocate(Runner(volumes="core_infinito\n")), "core_infinito" + ) + + def test_existing_build_cache(self) -> None: + self.assert_refused(relocate(Runner(build_cache="3")), "build cache") + + def test_containers_in_another_namespace(self) -> None: + out = relocate( + Runner(namespaces="moby\nk8s.io\n", foreign_containers="pause1\n") + ) + self.assert_refused(out, "k8s.io holds containers: pause1") + + def test_tasks_in_another_namespace(self) -> None: + out = relocate(Runner(namespaces="moby\nk8s.io\n", foreign_tasks="task7\n")) + self.assert_refused(out, "k8s.io runs tasks: task7") + + def test_running_shims(self) -> None: + out = relocate(Runner(shims="4242 containerd-shim-runc-v2 -id x\n", shim_rc=0)) + self.assert_refused(out, "containerd shims are running") + + def test_shim_listing_error(self) -> None: + self.assert_refused( + relocate(Runner(shim_rc=2)), "could not list containerd shims" + ) + + def test_broken_sudo(self) -> None: + out = relocate(Runner(sudo_rc=1)) + self.assertEqual(out.done.returncode, 1) + self.assertNotIn(STOP, out.calls) + + def test_unreadable_daemon_json_is_never_overwritten(self) -> None: + out = relocate(Runner(daemon_exists=True, daemon_in="{}", daemon_cat_rc=1)) + self.assertEqual(out.done.returncode, 1) + self.assertNotIn(STOP, out.calls) + self.assertFalse([c for c in out.calls if c.startswith("install ")]) + self.assertEqual(out.daemon, {}) + + def test_invalid_daemon_json_is_never_overwritten(self) -> None: + out = relocate(Runner(daemon_exists=True, daemon_in="{not json")) + self.assertEqual(out.done.returncode, 1) + self.assertIn("is not valid JSON", out.done.stderr) + self.assertNotIn(STOP, out.calls) + self.assertFalse([c for c in out.calls if c.startswith("install ")]) + self.assertEqual(out.daemon_raw, "{not json") + + +class TestRollback(unittest.TestCase): + def assert_rolled_back(self, out: Outcome, reason: str) -> None: + """Assert the rollback: no bind left, daemons restarted last, exit 1. + + Args: + out: the outcome of the run. + reason: text the rollback error must carry. + """ + self.assertEqual(out.done.returncode, 1, out.done.stderr) + self.assertIn("rolling Docker and containerd back onto /", out.done.stderr) + self.assertIn(reason, out.done.stderr) + self.assertFalse(out.bound) + self.assertIn(RESTART, out.calls) + self.assertTrue([c for c in out.calls if c.startswith("journalctl ")]) + + def test_failed_bind_mount_removes_the_daemon_json_it_never_had(self) -> None: + out = relocate(Runner(mount_rc=32)) + self.assert_rolled_back(out, "mount --bind") + self.assertIsNone(out.daemon) + + def test_failed_containerd_start_unbinds_before_the_restart(self) -> None: + out = relocate( + Runner( + daemon_exists=True, + daemon_in='{"debug": true}', + fail_systemctl="start containerd.service", + ) + ) + self.assert_rolled_back(out, "systemctl start containerd.service") + self.assertLess(out.calls.index(UNBIND), len(out.calls) - 1) + self.assertLess(out.calls.index(UNBIND), out.calls.index(RESTART)) + self.assertEqual(out.daemon, {"debug": True}) + + def test_cancel_during_the_move_rolls_back(self) -> None: + out = relocate(Runner(term_on="mkdir")) + self.assert_rolled_back(out, "interrupted") + + def test_down_units_still_yield_the_journal(self) -> None: + out = relocate(Runner(mount_rc=32, status_rc=3)) + self.assert_rolled_back(out, "mount --bind") + + def test_containerd_without_metadata_on_the_spare_disk(self) -> None: + out = relocate(Runner(metadata=False)) + self.assert_rolled_back(out, "without its metadata") + self.assertIn(UNBIND, out.calls) + self.assertIsNone(out.daemon) + + def test_docker_root_elsewhere(self) -> None: + out = relocate(Runner(docker_root="/var/lib/docker")) + self.assert_rolled_back(out, "expected '/mnt/docker-host'") + + def test_docker_root_off_the_spare_disk(self) -> None: + out = relocate(Runner(docker_root_dev="66305")) + self.assert_rolled_back(out, "not on the /mnt filesystem") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/tests/swarm/test_run_storage_roots.py b/tests/unit/python/utils/tests/swarm/test_run_storage_roots.py new file mode 100644 index 0000000000..f74647a43c --- /dev/null +++ b/tests/unit/python/utils/tests/swarm/test_run_storage_roots.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import shutil +import unittest +import unittest.mock as mock +from pathlib import Path +from typing import NamedTuple + +from utils.tests.swarm import run + + +class Usage(NamedTuple): + total: int + used: int + free: int + + +def lowest(free_gib: dict[str, int], existing: set[str]) -> tuple[int, str]: + """Call lowest_free_mb with faked disks. + + Args: + free_gib: free GiB per storage root. + existing: storage roots that exist as directories. + + Returns: + What lowest_free_mb reports. + """ + with ( + mock.patch.object( + shutil, + "disk_usage", + side_effect=lambda root: Usage(0, 0, free_gib[root] * 2**30), + ), + mock.patch.object( + Path, "is_dir", autospec=True, side_effect=lambda p: str(p) in existing + ), + ): + return run.lowest_free_mb() + + +class TestLowestFreeMb(unittest.TestCase): + def test_a_full_spare_disk_under_containerd_is_seen(self) -> None: + free, root = lowest( + {"/": 40, "/var/lib/containerd": 5}, {"/", "/var/lib/containerd"} + ) + self.assertEqual((free, root), (5 * 1024, "/var/lib/containerd")) + self.assertLess(free, run.DISK_FLOOR_MB) + + def test_a_full_root_is_still_seen(self) -> None: + free, root = lowest( + {"/": 5, "/var/lib/containerd": 150}, {"/", "/var/lib/containerd"} + ) + self.assertEqual((free, root), (5 * 1024, "/")) + + def test_a_host_without_containerd_measures_root_only(self) -> None: + free, root = lowest({"/": 30}, {"/"}) + self.assertEqual((free, root), (30 * 1024, "/")) + + +def watch( + waits: list[object], free: tuple[int, str] +) -> tuple[int, mock.MagicMock, str]: + """Run run_watched over a fake step. + + Args: + waits: successive outcomes of the step's wait(): an exception to raise + or an exit code to return. + free: what lowest_free_mb reports on every sample. + + Returns: + The exit code, the fake process, and everything printed. + """ + proc = mock.MagicMock() + proc.wait.side_effect = waits + printed = mock.MagicMock() + with ( + mock.patch.object(run.subprocess, "Popen", return_value=proc), + mock.patch.object(run.subprocess, "run"), + mock.patch.object(run, "lowest_free_mb", return_value=free), + mock.patch.object(run, "mem_available_mb", return_value=12000), + mock.patch.object(run, "mem_total_mb", return_value=16000), + mock.patch.object(run, "mem_stall_pct", return_value=0.0), + mock.patch("builtins.print", printed), + ): + rc = run.run_watched(["step"], env={}) + text = " ".join(str(call.args[0]) for call in printed.call_args_list) + return rc, proc, text + + +class TestRunWatchedSampling(unittest.TestCase): + def test_samples_every_ten_seconds(self) -> None: + timeout = run.subprocess.TimeoutExpired("step", run.SAMPLE_INTERVAL_S) + rc, proc, _ = watch([timeout, timeout, 0], (50_000, "/")) + self.assertEqual(rc, 0) + self.assertEqual(run.SAMPLE_INTERVAL_S, 10) + self.assertEqual( + [call.kwargs["timeout"] for call in proc.wait.call_args_list], + [10, 10, 10], + ) + + def test_a_full_containerd_root_aborts_the_step(self) -> None: + timeout = run.subprocess.TimeoutExpired("step", run.SAMPLE_INTERVAL_S) + rc, proc, text = watch([timeout, 0], (1_000, "/var/lib/containerd")) + self.assertEqual(rc, 75) + proc.terminate.assert_called_once_with() + self.assertIn("free on /var/lib/containerd", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/tests/swarm/run.py b/utils/tests/swarm/run.py index 042e7fcb96..405e4c8775 100644 --- a/utils/tests/swarm/run.py +++ b/utils/tests/swarm/run.py @@ -3,12 +3,28 @@ from __future__ import annotations import shutil import subprocess import time +from pathlib import Path from utils import PROJECT_ROOT from utils.env.runtime import mem_available_mb, mem_stall_pct, mem_total_mb DISK_FLOOR_MB = 6 * 2**10 MEM_FLOOR_RATIO = 0.06 +STORAGE_ROOTS = ("/", "/var/lib/containerd") +SAMPLE_INTERVAL_S = 10 + + +def lowest_free_mb() -> tuple[int, str]: + """Measure the storage root closest to running full. + + Returns: + Free MB on the fullest existing root in STORAGE_ROOTS, and that root. + """ + return min( + (shutil.disk_usage(root).free // 2**20, root) + for root in STORAGE_ROOTS + if Path(root).is_dir() + ) def _abort(proc: subprocess.Popen[bytes], banner: str, probe: list[str]) -> int: @@ -61,22 +77,22 @@ def run_watched(cmd: list[str], *, env: dict[str, str]) -> int: proc = subprocess.Popen(cmd, cwd=str(PROJECT_ROOT), env=env) while True: try: - return int(proc.wait(timeout=30)) + return int(proc.wait(timeout=SAMPLE_INTERVAL_S)) except subprocess.TimeoutExpired: - free_mb = shutil.disk_usage("/").free // 2**20 + free_mb, root = lowest_free_mb() if free_mb < DISK_FLOOR_MB: return _abort( proc, "=== swarm-matrix: DISK EXHAUSTION IMMINENT " - f"(<{DISK_FLOOR_MB}M free on /) - aborting step ===", - ["df", "-h", "/"], + f"(<{DISK_FLOOR_MB}M free on {root}) - aborting step ===", + ["df", "-h", *STORAGE_ROOTS], ) avail = mem_available_mb() total = mem_total_mb() stall = mem_stall_pct() print( f"=== swarm-matrix: host mem {avail}M/{total}M available, " - f"disk {free_mb}M free on /, stall60 {stall:.1f}% ===", + f"disk {free_mb}M free on {root}, stall60 {stall:.1f}% ===", flush=True, ) if total and avail < total * MEM_FLOOR_RATIO: