mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-08 19:56:53 +00:00
Two jobs of run 39040406916 died with the runner's own annotation, 'The hosted runner lost communication with the server'. Both left every diagnostic step pending, so no artifact exists for either. For the swarm one the in-band sampler ruled starvation out — 9324M of 15947M available, stall60 0.0%, 25 seconds before the cut — and nothing in the repository can tell the remaining causes apart. A stuck NFS mount wedges tasks in uninterruptible D state, which shows as I/O pressure while memory pressure stays flat, and this drill runs its own NFS node. /proc/pressure/io and /proc/pressure/cpu were read nowhere in the repository. They are now sampled beside memory, with the D-state count, every ten seconds instead of thirty, and an I/O stall at or above half the minute aborts the step while the runner is still alive to upload the banner. cpu reads 'some', not 'full': the full line is pinned at 0.00 system-wide, measured 13.50 against 0.00 on this host, so reading it like memory would have shipped a metric that can never move. Dump runner state was switched off for swarm, so the one post-mortem that survives a step failure never ran there. It runs now, and the swarm drill gets the step timeout its compose sibling has had all along — 350, the number test_distro_budget_fits_timeout already derives the distro budget from. The sampler moved to its own module: the additions took matrix.py past the file-size cap, and the lint asks for a split rather than a marker. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
215 lines
6.6 KiB
Python
215 lines
6.6 KiB
Python
"""Runtime-context lookups for the .env generator.
|
|
|
|
Each helper resolves a single piece of information from the host the
|
|
generator runs on (disk free space, RAM, hostname, /proc/version,
|
|
GitHub Actions env vars, ...). They are intentionally tiny and
|
|
side-effect-free apart from the OS read they need to perform, so they
|
|
can be exercised independently in tests.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
import socket
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
|
|
def is_user_writable(path: str) -> bool:
|
|
"""Return True when `path` (or its closest existing ancestor) can
|
|
be written to by the current user without privilege escalation.
|
|
|
|
Used to decide between a system path (e.g. ``/opt/venvs``) and a
|
|
user-local fallback (``$HOME/.venvs``) when the system path is not
|
|
yet prepared on a fresh dev machine."""
|
|
p = Path(path)
|
|
while not p.exists() and str(p) not in ("/", "."):
|
|
p = p.parent
|
|
return os.access(p, os.W_OK)
|
|
|
|
|
|
def df_avail_gb(path: str) -> int:
|
|
"""Return free space in GB at `path` (or the closest existing
|
|
ancestor). 0 when `df` fails or the output is unparseable."""
|
|
p = Path(path)
|
|
while not p.exists() and str(p) not in ("/", "."):
|
|
p = p.parent
|
|
try:
|
|
out = subprocess.check_output(
|
|
["df", "--output=avail", "-B1G", str(p)],
|
|
text=True,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
except (subprocess.CalledProcessError, OSError):
|
|
return 0
|
|
lines = out.strip().splitlines()
|
|
if len(lines) < 2:
|
|
return 0
|
|
try:
|
|
return int(lines[1].strip())
|
|
except ValueError:
|
|
return 0
|
|
|
|
|
|
def _meminfo_mb(field: str) -> int:
|
|
"""`field` from /proc/meminfo in MB. 0 on failure."""
|
|
try:
|
|
text = Path(
|
|
"/proc/meminfo"
|
|
).read_text() # nocheck: cache-read -- live kernel pseudo-file
|
|
except OSError:
|
|
return 0
|
|
for line in text.splitlines():
|
|
if line.startswith(field):
|
|
try:
|
|
kb = int(line.split()[1])
|
|
return kb // 1024
|
|
except (IndexError, ValueError):
|
|
return 0
|
|
return 0
|
|
|
|
|
|
def mem_available_mb() -> int:
|
|
"""`MemAvailable` from /proc/meminfo in MB. 0 on failure."""
|
|
return _meminfo_mb("MemAvailable:")
|
|
|
|
|
|
def mem_total_mb() -> int:
|
|
"""`MemTotal` from /proc/meminfo in MB. 0 on failure."""
|
|
return _meminfo_mb("MemTotal:")
|
|
|
|
|
|
def _pressure_pct(resource: str, kind: str) -> float:
|
|
"""`<kind> avg60` from /proc/pressure/<resource> in percent.
|
|
|
|
Args:
|
|
resource: PSI resource file name (``memory``, ``io``, ``cpu``).
|
|
kind: pressure line to read (``some`` or ``full``).
|
|
|
|
Returns:
|
|
The avg60 percentage, 0.0 when the file or the field is unreadable.
|
|
"""
|
|
try:
|
|
text = Path(
|
|
f"/proc/pressure/{resource}"
|
|
).read_text() # nocheck: cache-read -- live kernel pseudo-file
|
|
except OSError:
|
|
return 0.0
|
|
for line in text.splitlines():
|
|
if not line.startswith(f"{kind} "):
|
|
continue
|
|
for field in line.split():
|
|
if field.startswith("avg60="):
|
|
try:
|
|
return float(field.removeprefix("avg60="))
|
|
except ValueError:
|
|
return 0.0
|
|
return 0.0
|
|
|
|
|
|
def mem_stall_pct() -> float:
|
|
"""`full avg60` from /proc/pressure/memory in percent. 0.0 on failure."""
|
|
return _pressure_pct("memory", "full")
|
|
|
|
|
|
def io_stall_pct() -> float:
|
|
"""`full avg60` from /proc/pressure/io in percent. 0.0 on failure.
|
|
|
|
`full` is the share of the last minute in which every non-idle task was
|
|
blocked on I/O, which is the signature of a wedged mount rather than of
|
|
a busy one."""
|
|
return _pressure_pct("io", "full")
|
|
|
|
|
|
def cpu_stall_pct() -> float:
|
|
"""`some avg60` from /proc/pressure/cpu in percent. 0.0 on failure.
|
|
|
|
The kernel emits a system-level `full` line for cpu but keeps it pinned
|
|
at 0, so reading cpu like its memory and io siblings would report a
|
|
constant zero."""
|
|
return _pressure_pct("cpu", "some")
|
|
|
|
|
|
def uninterruptible_procs() -> int:
|
|
"""Count the processes in uninterruptible (`D`) state via /proc/<pid>/stat.
|
|
|
|
A process stuck in `D` cannot be killed and is the host-visible trace of
|
|
a wedged mount. Only the state field is evaluated; the command name is
|
|
never retained, so the count can be logged without leaking process
|
|
command lines. The comm field may itself contain spaces and `)`, so the
|
|
state is read after the last closing parenthesis rather than by index.
|
|
|
|
Returns:
|
|
The number of `D`-state processes, 0 when /proc is unreadable.
|
|
"""
|
|
try:
|
|
entries = os.listdir("/proc")
|
|
except OSError:
|
|
return 0
|
|
count = 0
|
|
for entry in entries:
|
|
if not entry.isdigit():
|
|
continue
|
|
try:
|
|
stat = Path(
|
|
f"/proc/{entry}/stat"
|
|
).read_text() # nocheck: cache-read -- live kernel pseudo-file
|
|
except OSError:
|
|
continue
|
|
fields = stat.rpartition(")")[2].split()
|
|
if fields and fields[0] == "D":
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def is_wsl2() -> bool:
|
|
"""True when /proc/version mentions microsoft or wsl. False otherwise."""
|
|
try:
|
|
text = Path(
|
|
"/proc/version"
|
|
).read_text() # nocheck: cache-read -- live kernel pseudo-file
|
|
except OSError:
|
|
return False
|
|
return bool(re.search(r"microsoft|wsl", text, re.IGNORECASE))
|
|
|
|
|
|
def detect_gha_act() -> tuple[bool, bool]:
|
|
"""Return (running_on_github, running_on_act).
|
|
|
|
Mirrors the legacy scripts/meta/env/runtime.sh logic:
|
|
- GITHUB_ACTIONS=true AND ACT=true -> act (local nektos act)
|
|
- GITHUB_ACTIONS=true AND not ACT -> real github actions
|
|
"""
|
|
gha = os.environ.get("GITHUB_ACTIONS") == "true"
|
|
act = gha and os.environ.get("ACT") == "true"
|
|
return (gha and not act, act)
|
|
|
|
|
|
def hostname() -> str:
|
|
"""`socket.gethostname()` with a `local` fallback for sandboxed hosts
|
|
that fail the syscall."""
|
|
try:
|
|
name = socket.gethostname()
|
|
except OSError:
|
|
return "local"
|
|
return name or "local"
|
|
|
|
|
|
def run_helper(
|
|
cmd: list[str],
|
|
cwd: Path,
|
|
extra_env: dict[str, str] | None = None,
|
|
) -> str:
|
|
"""Run an existing shell helper, return stripped stdout. Empty
|
|
string on failure. Helpers run with `cwd` as their working
|
|
directory."""
|
|
env = {**os.environ, **(extra_env or {})}
|
|
try:
|
|
out = subprocess.check_output(
|
|
cmd, cwd=str(cwd), env=env, text=True, stderr=subprocess.DEVNULL
|
|
)
|
|
return out.strip()
|
|
except (subprocess.CalledProcessError, OSError, FileNotFoundError):
|
|
return ""
|