mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 02:54:26 +00:00
kevinveenbirkenbachandClaude Opus 5.5
2d7bf740b0
The small GitHub runner shape (AWS, 72G root) ran out of disk in swarm and compose rows (#1329: WordPress#2 of run 41261531965, Nextcloud#6 and #7 of run 41262286962) while an empty instance-store disk nvme1n1 sat unused. mount_spare_disk.sh formats and mounts it at /mnt, but only on a GitHub-hosted runner outside act, and only a whole unpartitioned disk without any signature, mount or swap use of at least 75G; every decline leaves a notice. relocate_docker_dataroot.sh then bind-mounts /var/lib/containerd from the spare disk and moves the Docker data-root to /mnt/docker-host (not /mnt/docker, which is the nested daemon's volume that dedicated.sh wipes). It refuses before stopping anything when images, volumes, build cache, foreign containerd namespaces or shims exist or daemon.json is unreadable, and rolls back the bind mount and daemon.json on any later failure or cancel. Swap goes on / once containerd lives on /mnt, dump_state.sh logs disk size, model and free space per storage root, and the swarm watchdog samples / and /var/lib/containerd every 10 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
100 lines
4.2 KiB
Bash
Executable File
100 lines
4.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Dump disk, memory, swap and docker state of a GitHub-hosted runner.
|
|
# Intended as a diagnostic helper that can be called from anywhere
|
|
# inside a workflow step (pre/post swap resize, post-deploy failure).
|
|
#
|
|
# Usage: dump_state.sh [label]
|
|
# The label is rendered into the log group title and the step summary.
|
|
|
|
set -euo pipefail
|
|
|
|
LABEL="${1:-runner state}"
|
|
|
|
echo "::group::Runner state — ${LABEL}"
|
|
|
|
echo "--- date ---"
|
|
date -u +'%Y-%m-%dT%H:%M:%SZ'
|
|
|
|
echo "--- disk (bytes) ---"
|
|
df -h / /mnt 2>/dev/null || df -h /
|
|
|
|
echo "--- disk per storage root (/, /mnt, containerd root, Docker data root) ---"
|
|
docker_root="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null)" || docker_root=""
|
|
df -h --output=source,size,used,avail,target / /mnt /var/lib/containerd ${docker_root:+"${docker_root}"} ||
|
|
echo "df could not read every storage root"
|
|
|
|
containerd --version 2>/dev/null || echo "containerd not on PATH"
|
|
|
|
echo "--- disk (inodes) ---"
|
|
df -hi / /mnt 2>/dev/null || df -hi /
|
|
|
|
echo "--- disks (bytes, model: Amazon EC2 NVMe Instance Storage versus Amazon Elastic Block Store) ---"
|
|
lsblk -d -b -o NAME,SIZE,MODEL,ROTA,TYPE || echo "lsblk could not list the disks"
|
|
|
|
echo "--- block devices ---"
|
|
lsblk -o NAME,TYPE,SIZE,FSTYPE,FSVER,LABEL,UUID,FSAVAIL,FSUSE%,MOUNTPOINTS || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo "--- mounts (/, /mnt, /var/lib/docker, /var/lib/containerd, /mnt/docker) ---"
|
|
mount | grep -E ' / | /mnt | /var/lib/docker | /var/lib/containerd | /mnt/docker ' || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo "--- docker ---"
|
|
docker info 2>/dev/null |
|
|
grep -E 'Docker Root Dir|Storage Driver|Containers|Images' ||
|
|
echo "docker daemon not reachable"
|
|
|
|
echo "--- memory ---"
|
|
free -h
|
|
|
|
echo "--- swap ---"
|
|
swapon --show || echo "no swap active"
|
|
|
|
echo "--- largest dirs on / (top 15) ---"
|
|
sudo du -xh --max-depth=2 / 2>/dev/null | sort -h | tail -15 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
if [ -d /mnt ]; then
|
|
echo "--- largest dirs on /mnt (top 15) ---"
|
|
sudo du -xh --max-depth=2 /mnt 2>/dev/null | sort -h | tail -15 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
fi
|
|
|
|
echo "--- infinito container (nested view) ---"
|
|
INFINITO_CONTAINER="$(docker ps --filter 'name=infinito_nexus' --format '{{.Names}}' 2>/dev/null | head -1 || true)"
|
|
if [ -n "${INFINITO_CONTAINER}" ]; then
|
|
echo "container: ${INFINITO_CONTAINER}"
|
|
|
|
echo "--- df inside container ---"
|
|
docker exec "${INFINITO_CONTAINER}" df -h 2>&1 || echo "docker exec df failed"
|
|
|
|
echo "--- df --output=pcent inside container (what hlth-disc-space sees) ---"
|
|
docker exec "${INFINITO_CONTAINER}" df --output=pcent 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo "--- failed systemd units inside container ---"
|
|
docker exec "${INFINITO_CONTAINER}" systemctl --failed --no-pager 2>&1 ||
|
|
echo "systemctl --failed not reachable"
|
|
|
|
echo "--- hlth-disc-space status ---"
|
|
docker exec "${INFINITO_CONTAINER}" \
|
|
systemctl status 'hlth-disc-space*' --no-pager -l 2>&1 ||
|
|
echo "no hlth-disc-space unit (not deployed yet)"
|
|
|
|
echo "--- hlth-disc-space journal (last 80 lines) ---"
|
|
docker exec "${INFINITO_CONTAINER}" \
|
|
journalctl -u 'hlth-disc-space*' -n 80 --no-pager 2>&1 ||
|
|
echo "journal for hlth-disc-space not accessible"
|
|
else
|
|
echo "no infinito container running (expected before 'make compose-up')"
|
|
fi
|
|
|
|
echo "::endgroup::"
|
|
|
|
if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
|
|
root_free=$(df --output=avail -B1G / 2>/dev/null | tail -n1 | tr -d ' ' || echo '?')
|
|
mnt_free=$(df --output=avail -B1G /mnt 2>/dev/null | tail -n1 | tr -d ' ' || echo '?')
|
|
mem_free=$(free -g | awk '/^Mem:/ {print $4"G"}')
|
|
swap_total=$(free -g | awk '/^Swap:/ {print $2"G"}')
|
|
{
|
|
echo "### Runner state — ${LABEL}"
|
|
echo "- / free: **${root_free}G** · /mnt free: **${mnt_free}G**"
|
|
echo "- mem free: **${mem_free}** · swap total: **${swap_total}**"
|
|
} >>"$GITHUB_STEP_SUMMARY"
|
|
fi
|