Files
computer-playbook/utils/env/runtime.py
kevinveenbirkenbachandClaude Opus 5 3e9deab6f7 test(swarm): abort the drill before the runner dies
When the host runs out of memory the Actions runner loses its heartbeat. GitHub then reports only 'lost communication': no step completes, so every if-always upload step is skipped and the rescue diagnostics, the inventory artifact and the Playwright results are all lost. Job 92710765494 died exactly that way.

The 30s poll in _run already did this for disk. It now also samples MemAvailable, MemTotal and the memory PSI stall, prints one line per poll, and aborts below 6 percent of RAM. The abort returns 75, which propagates to the EXIT trap in 00_one.sh, so the existing post-mortem apparatus runs normally instead of dying with the runner.

The sample line is the main point: artifacts provably do not survive a lost runner, streamed log lines largely do, and the numbers discriminate memory thrash from a kernel wedge on the next occurrence. The floor is relative to MemTotal so a dev box does not inherit a runner-sized threshold; measured margin across four real drills is 9-15x. PSI is sampled but not used as a trigger, since no drill has ever measured it.

mem_total_mb and mem_stall_pct join the existing host probes in utils/env/runtime.py and share the /proc/meminfo parser. The probes read only /proc, never the host-side NFS mount, which would block in D-state under the very condition they exist to catch.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-07 17:05:49 +02:00

152 lines
4.5 KiB
Python

"""Runtime-context lookups for the .env generator.
Each helper resolves a single piece of information from the host the
generator runs on (disk free space, RAM, hostname, /proc/version,
GitHub Actions env vars, ...). They are intentionally tiny and
side-effect-free apart from the OS read they need to perform, so they
can be exercised independently in tests.
"""
from __future__ import annotations
import os
import re
import socket
import subprocess
from pathlib import Path
def is_user_writable(path: str) -> bool:
"""Return True when `path` (or its closest existing ancestor) can
be written to by the current user without privilege escalation.
Used to decide between a system path (e.g. ``/opt/venvs``) and a
user-local fallback (``$HOME/.venvs``) when the system path is not
yet prepared on a fresh dev machine."""
p = Path(path)
while not p.exists() and str(p) not in ("/", "."):
p = p.parent
return os.access(p, os.W_OK)
def df_avail_gb(path: str) -> int:
"""Return free space in GB at `path` (or the closest existing
ancestor). 0 when `df` fails or the output is unparseable."""
p = Path(path)
while not p.exists() and str(p) not in ("/", "."):
p = p.parent
try:
out = subprocess.check_output(
["df", "--output=avail", "-B1G", str(p)],
text=True,
stderr=subprocess.DEVNULL,
)
except (subprocess.CalledProcessError, OSError):
return 0
lines = out.strip().splitlines()
if len(lines) < 2:
return 0
try:
return int(lines[1].strip())
except ValueError:
return 0
def _meminfo_mb(field: str) -> int:
"""`field` from /proc/meminfo in MB. 0 on failure."""
try:
text = Path(
"/proc/meminfo"
).read_text() # nocheck: cache-read -- live kernel pseudo-file
except OSError:
return 0
for line in text.splitlines():
if line.startswith(field):
try:
kb = int(line.split()[1])
return kb // 1024
except (IndexError, ValueError):
return 0
return 0
def mem_available_mb() -> int:
"""`MemAvailable` from /proc/meminfo in MB. 0 on failure."""
return _meminfo_mb("MemAvailable:")
def mem_total_mb() -> int:
"""`MemTotal` from /proc/meminfo in MB. 0 on failure."""
return _meminfo_mb("MemTotal:")
def mem_stall_pct() -> float:
"""`full avg60` from /proc/pressure/memory in percent. 0.0 on failure."""
try:
text = Path(
"/proc/pressure/memory"
).read_text() # nocheck: cache-read -- live kernel pseudo-file
except OSError:
return 0.0
for line in text.splitlines():
if not line.startswith("full "):
continue
for field in line.split():
if field.startswith("avg60="):
try:
return float(field.removeprefix("avg60="))
except ValueError:
return 0.0
return 0.0
def is_wsl2() -> bool:
"""True when /proc/version mentions microsoft or wsl. False otherwise."""
try:
text = Path(
"/proc/version"
).read_text() # nocheck: cache-read -- live kernel pseudo-file
except OSError:
return False
return bool(re.search(r"microsoft|wsl", text, re.IGNORECASE))
def detect_gha_act() -> tuple[bool, bool]:
"""Return (running_on_github, running_on_act).
Mirrors the legacy scripts/meta/env/runtime.sh logic:
- GITHUB_ACTIONS=true AND ACT=true -> act (local nektos act)
- GITHUB_ACTIONS=true AND not ACT -> real github actions
"""
gha = os.environ.get("GITHUB_ACTIONS") == "true"
act = gha and os.environ.get("ACT") == "true"
return (gha and not act, act)
def hostname() -> str:
"""`socket.gethostname()` with a `local` fallback for sandboxed hosts
that fail the syscall."""
try:
name = socket.gethostname()
except OSError:
return "local"
return name or "local"
def run_helper(
cmd: list[str],
cwd: Path,
extra_env: dict[str, str] | None = None,
) -> str:
"""Run an existing shell helper, return stripped stdout. Empty
string on failure. Helpers run with `cwd` as their working
directory."""
env = {**os.environ, **(extra_env or {})}
try:
out = subprocess.check_output(
cmd, cwd=str(cwd), env=env, text=True, stderr=subprocess.DEVNULL
)
return out.strip()
except (subprocess.CalledProcessError, OSError, FileNotFoundError):
return ""