Files
computer-playbook/utils/env/runtime.py
T
kevinveenbirkenbachandClaude Opus 5 676a91773e fix(ci): sample I/O pressure, and give swarm a post-mortem at all
Two jobs of run 39040406916 died with the runner's own annotation, 'The
hosted runner lost communication with the server'. Both left every
diagnostic step pending, so no artifact exists for either. For the swarm
one the in-band sampler ruled starvation out — 9324M of 15947M available,
stall60 0.0%, 25 seconds before the cut — and nothing in the repository
can tell the remaining causes apart.

A stuck NFS mount wedges tasks in uninterruptible D state, which shows as
I/O pressure while memory pressure stays flat, and this drill runs its own
NFS node. /proc/pressure/io and /proc/pressure/cpu were read nowhere in
the repository. They are now sampled beside memory, with the D-state count,
every ten seconds instead of thirty, and an I/O stall at or above half the
minute aborts the step while the runner is still alive to upload the
banner.

cpu reads 'some', not 'full': the full line is pinned at 0.00 system-wide,
measured 13.50 against 0.00 on this host, so reading it like memory would
have shipped a metric that can never move.

Dump runner state was switched off for swarm, so the one post-mortem that
survives a step failure never ran there. It runs now, and the swarm drill
gets the step timeout its compose sibling has had all along — 350, the
number test_distro_budget_fits_timeout already derives the distro budget
from.

The sampler moved to its own module: the additions took matrix.py past the
file-size cap, and the lint asks for a split rather than a marker.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-10-17 21:31:12 +02:00

215 lines
6.6 KiB
Python

"""Runtime-context lookups for the .env generator.
Each helper resolves a single piece of information from the host the
generator runs on (disk free space, RAM, hostname, /proc/version,
GitHub Actions env vars, ...). They are intentionally tiny and
side-effect-free apart from the OS read they need to perform, so they
can be exercised independently in tests.
"""
from __future__ import annotations
import os
import re
import socket
import subprocess
from pathlib import Path
def is_user_writable(path: str) -> bool:
"""Return True when `path` (or its closest existing ancestor) can
be written to by the current user without privilege escalation.
Used to decide between a system path (e.g. ``/opt/venvs``) and a
user-local fallback (``$HOME/.venvs``) when the system path is not
yet prepared on a fresh dev machine."""
p = Path(path)
while not p.exists() and str(p) not in ("/", "."):
p = p.parent
return os.access(p, os.W_OK)
def df_avail_gb(path: str) -> int:
"""Return free space in GB at `path` (or the closest existing
ancestor). 0 when `df` fails or the output is unparseable."""
p = Path(path)
while not p.exists() and str(p) not in ("/", "."):
p = p.parent
try:
out = subprocess.check_output(
["df", "--output=avail", "-B1G", str(p)],
text=True,
stderr=subprocess.DEVNULL,
)
except (subprocess.CalledProcessError, OSError):
return 0
lines = out.strip().splitlines()
if len(lines) < 2:
return 0
try:
return int(lines[1].strip())
except ValueError:
return 0
def _meminfo_mb(field: str) -> int:
"""`field` from /proc/meminfo in MB. 0 on failure."""
try:
text = Path(
"/proc/meminfo"
).read_text() # nocheck: cache-read -- live kernel pseudo-file
except OSError:
return 0
for line in text.splitlines():
if line.startswith(field):
try:
kb = int(line.split()[1])
return kb // 1024
except (IndexError, ValueError):
return 0
return 0
def mem_available_mb() -> int:
"""`MemAvailable` from /proc/meminfo in MB. 0 on failure."""
return _meminfo_mb("MemAvailable:")
def mem_total_mb() -> int:
"""`MemTotal` from /proc/meminfo in MB. 0 on failure."""
return _meminfo_mb("MemTotal:")
def _pressure_pct(resource: str, kind: str) -> float:
"""`<kind> avg60` from /proc/pressure/<resource> in percent.
Args:
resource: PSI resource file name (``memory``, ``io``, ``cpu``).
kind: pressure line to read (``some`` or ``full``).
Returns:
The avg60 percentage, 0.0 when the file or the field is unreadable.
"""
try:
text = Path(
f"/proc/pressure/{resource}"
).read_text() # nocheck: cache-read -- live kernel pseudo-file
except OSError:
return 0.0
for line in text.splitlines():
if not line.startswith(f"{kind} "):
continue
for field in line.split():
if field.startswith("avg60="):
try:
return float(field.removeprefix("avg60="))
except ValueError:
return 0.0
return 0.0
def mem_stall_pct() -> float:
"""`full avg60` from /proc/pressure/memory in percent. 0.0 on failure."""
return _pressure_pct("memory", "full")
def io_stall_pct() -> float:
"""`full avg60` from /proc/pressure/io in percent. 0.0 on failure.
`full` is the share of the last minute in which every non-idle task was
blocked on I/O, which is the signature of a wedged mount rather than of
a busy one."""
return _pressure_pct("io", "full")
def cpu_stall_pct() -> float:
"""`some avg60` from /proc/pressure/cpu in percent. 0.0 on failure.
The kernel emits a system-level `full` line for cpu but keeps it pinned
at 0, so reading cpu like its memory and io siblings would report a
constant zero."""
return _pressure_pct("cpu", "some")
def uninterruptible_procs() -> int:
"""Count the processes in uninterruptible (`D`) state via /proc/<pid>/stat.
A process stuck in `D` cannot be killed and is the host-visible trace of
a wedged mount. Only the state field is evaluated; the command name is
never retained, so the count can be logged without leaking process
command lines. The comm field may itself contain spaces and `)`, so the
state is read after the last closing parenthesis rather than by index.
Returns:
The number of `D`-state processes, 0 when /proc is unreadable.
"""
try:
entries = os.listdir("/proc")
except OSError:
return 0
count = 0
for entry in entries:
if not entry.isdigit():
continue
try:
stat = Path(
f"/proc/{entry}/stat"
).read_text() # nocheck: cache-read -- live kernel pseudo-file
except OSError:
continue
fields = stat.rpartition(")")[2].split()
if fields and fields[0] == "D":
count += 1
return count
def is_wsl2() -> bool:
"""True when /proc/version mentions microsoft or wsl. False otherwise."""
try:
text = Path(
"/proc/version"
).read_text() # nocheck: cache-read -- live kernel pseudo-file
except OSError:
return False
return bool(re.search(r"microsoft|wsl", text, re.IGNORECASE))
def detect_gha_act() -> tuple[bool, bool]:
"""Return (running_on_github, running_on_act).
Mirrors the legacy scripts/meta/env/runtime.sh logic:
- GITHUB_ACTIONS=true AND ACT=true -> act (local nektos act)
- GITHUB_ACTIONS=true AND not ACT -> real github actions
"""
gha = os.environ.get("GITHUB_ACTIONS") == "true"
act = gha and os.environ.get("ACT") == "true"
return (gha and not act, act)
def hostname() -> str:
"""`socket.gethostname()` with a `local` fallback for sandboxed hosts
that fail the syscall."""
try:
name = socket.gethostname()
except OSError:
return "local"
return name or "local"
def run_helper(
cmd: list[str],
cwd: Path,
extra_env: dict[str, str] | None = None,
) -> str:
"""Run an existing shell helper, return stripped stdout. Empty
string on failure. Helpers run with `cwd` as their working
directory."""
env = {**os.environ, **(extra_env or {})}
try:
out = subprocess.check_output(
cmd, cwd=str(cwd), env=env, text=True, stderr=subprocess.DEVNULL
)
return out.strip()
except (subprocess.CalledProcessError, OSError, FileNotFoundError):
return ""