mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 10:59:59 +00:00
kevinveenbirkenbachandClaude Opus 5.5
2d7bf740b0
The small GitHub runner shape (AWS, 72G root) ran out of disk in swarm and compose rows (#1329: WordPress#2 of run 41261531965, Nextcloud#6 and #7 of run 41262286962) while an empty instance-store disk nvme1n1 sat unused. mount_spare_disk.sh formats and mounts it at /mnt, but only on a GitHub-hosted runner outside act, and only a whole unpartitioned disk without any signature, mount or swap use of at least 75G; every decline leaves a notice. relocate_docker_dataroot.sh then bind-mounts /var/lib/containerd from the spare disk and moves the Docker data-root to /mnt/docker-host (not /mnt/docker, which is the nested daemon's volume that dedicated.sh wipes). It refuses before stopping anything when images, volumes, build cache, foreign containerd namespaces or shims exist or daemon.json is unreadable, and rolls back the bind mount and daemon.json on any later failure or cancel. Swap goes on / once containerd lives on /mnt, dump_state.sh logs disk size, model and free space per storage root, and the swarm watchdog samples / and /var/lib/containerd every 10 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
106 lines
3.3 KiB
Python
106 lines
3.3 KiB
Python
from __future__ import annotations
|
|
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from utils import PROJECT_ROOT
|
|
from utils.env.runtime import mem_available_mb, mem_stall_pct, mem_total_mb
|
|
|
|
DISK_FLOOR_MB = 6 * 2**10
|
|
MEM_FLOOR_RATIO = 0.06
|
|
STORAGE_ROOTS = ("/", "/var/lib/containerd")
|
|
SAMPLE_INTERVAL_S = 10
|
|
|
|
|
|
def lowest_free_mb() -> tuple[int, str]:
|
|
"""Measure the storage root closest to running full.
|
|
|
|
Returns:
|
|
Free MB on the fullest existing root in STORAGE_ROOTS, and that root.
|
|
"""
|
|
return min(
|
|
(shutil.disk_usage(root).free // 2**20, root)
|
|
for root in STORAGE_ROOTS
|
|
if Path(root).is_dir()
|
|
)
|
|
|
|
|
|
def _abort(proc: subprocess.Popen[bytes], banner: str, probe: list[str]) -> int:
|
|
print(banner, flush=True)
|
|
subprocess.run(probe, check=False)
|
|
proc.terminate()
|
|
try:
|
|
proc.wait(timeout=30)
|
|
except subprocess.TimeoutExpired:
|
|
proc.kill()
|
|
proc.wait()
|
|
return 75
|
|
|
|
|
|
def run_step(cmd: list[str], *, env: dict[str, str], label: str) -> int:
|
|
"""Announce a matrix step, run it, and report what it cost.
|
|
|
|
Args:
|
|
cmd: argv of the step.
|
|
env: environment the step runs with.
|
|
label: phase name for the banners.
|
|
|
|
Returns:
|
|
The step's exit code.
|
|
"""
|
|
print(f"=== swarm-matrix: {label} ===", flush=True)
|
|
started = time.monotonic()
|
|
rc = run_watched(cmd, env=env)
|
|
print(
|
|
f"=== swarm-matrix: {label} took {time.monotonic() - started:.0f}s (rc={rc}) ===",
|
|
flush=True,
|
|
)
|
|
return rc
|
|
|
|
|
|
def run_watched(cmd: list[str], *, env: dict[str, str]) -> int:
|
|
"""Run a step, aborting visibly before the runner disk or RAM fills.
|
|
|
|
The Actions Worker dies silently on ENOSPC while writing its own logs, so
|
|
a full disk truncates the job without diagnostics; terminating the step at
|
|
DISK_FLOOR_MB keeps enough room for rescue artifacts and the log upload.
|
|
|
|
Args:
|
|
cmd: argv of the step.
|
|
env: environment the step runs with.
|
|
|
|
Returns:
|
|
The step's exit code, or the abort code when a floor was hit.
|
|
"""
|
|
proc = subprocess.Popen(cmd, cwd=str(PROJECT_ROOT), env=env)
|
|
while True:
|
|
try:
|
|
return int(proc.wait(timeout=SAMPLE_INTERVAL_S))
|
|
except subprocess.TimeoutExpired:
|
|
free_mb, root = lowest_free_mb()
|
|
if free_mb < DISK_FLOOR_MB:
|
|
return _abort(
|
|
proc,
|
|
"=== swarm-matrix: DISK EXHAUSTION IMMINENT "
|
|
f"(<{DISK_FLOOR_MB}M free on {root}) - aborting step ===",
|
|
["df", "-h", *STORAGE_ROOTS],
|
|
)
|
|
avail = mem_available_mb()
|
|
total = mem_total_mb()
|
|
stall = mem_stall_pct()
|
|
print(
|
|
f"=== swarm-matrix: host mem {avail}M/{total}M available, "
|
|
f"disk {free_mb}M free on {root}, stall60 {stall:.1f}% ===",
|
|
flush=True,
|
|
)
|
|
if total and avail < total * MEM_FLOOR_RATIO:
|
|
return _abort(
|
|
proc,
|
|
"=== swarm-matrix: MEMORY EXHAUSTION IMMINENT "
|
|
f"({avail}M available, below {MEM_FLOOR_RATIO:.0%} of RAM)"
|
|
" - aborting step ===",
|
|
["free", "-m"],
|
|
)
|