test(swarm): abort the drill before the runner dies

When the host runs out of memory the Actions runner loses its heartbeat. GitHub then reports only 'lost communication': no step completes, so every if-always upload step is skipped and the rescue diagnostics, the inventory artifact and the Playwright results are all lost. Job 92710765494 died exactly that way.

The 30s poll in _run already did this for disk. It now also samples MemAvailable, MemTotal and the memory PSI stall, prints one line per poll, and aborts below 6 percent of RAM. The abort returns 75, which propagates to the EXIT trap in 00_one.sh, so the existing post-mortem apparatus runs normally instead of dying with the runner.

The sample line is the main point: artifacts provably do not survive a lost runner, streamed log lines largely do, and the numbers discriminate memory thrash from a kernel wedge on the next occurrence. The floor is relative to MemTotal so a dev box does not inherit a runner-sized threshold; measured margin across four real drills is 9-15x. PSI is sampled but not used as a trigger, since no drill has ever measured it.

mem_total_mb and mem_stall_pct join the existing host probes in utils/env/runtime.py and share the /proc/meminfo parser. The probes read only /proc, never the host-side NFS mount, which would block in D-state under the very condition they exist to catch.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

utils/env/runtime.py

@@ -52,8 +52,8 @@ def df_avail_gb(path: str) -> int:
         return 0
 
 
-def mem_available_mb() -> int:
-    """`MemAvailable` from /proc/meminfo in MB. 0 on failure."""
+def _meminfo_mb(field: str) -> int:
+    """`field` from /proc/meminfo in MB. 0 on failure."""
     try:
         text = Path(
             "/proc/meminfo"
@@ -61,7 +61,7 @@ def mem_available_mb() -> int:
     except OSError:
         return 0
     for line in text.splitlines():
-        if line.startswith("MemAvailable:"):
+        if line.startswith(field):
             try:
                 kb = int(line.split()[1])
                 return kb // 1024
@@ -70,6 +70,36 @@ def mem_available_mb() -> int:
     return 0
 
 
+def mem_available_mb() -> int:
+    """`MemAvailable` from /proc/meminfo in MB. 0 on failure."""
+    return _meminfo_mb("MemAvailable:")
+
+
+def mem_total_mb() -> int:
+    """`MemTotal` from /proc/meminfo in MB. 0 on failure."""
+    return _meminfo_mb("MemTotal:")
+
+
+def mem_stall_pct() -> float:
+    """`full avg60` from /proc/pressure/memory in percent. 0.0 on failure."""
+    try:
+        text = Path(
+            "/proc/pressure/memory"
+        ).read_text()  # nocheck: cache-read -- live kernel pseudo-file
+    except OSError:
+        return 0.0
+    for line in text.splitlines():
+        if not line.startswith("full "):
+            continue
+        for field in line.split():
+            if field.startswith("avg60="):
+                try:
+                    return float(field.removeprefix("avg60="))
+                except ValueError:
+                    return 0.0
+    return 0.0
+
+
 def is_wsl2() -> bool:
     """True when /proc/version mentions microsoft or wsl. False otherwise."""
     try:

utils/tests/swarm/matrix.py

@@ -26,6 +26,7 @@ import subprocess
 import sys
 
 from utils import PROJECT_ROOT
+from utils.env.runtime import mem_available_mb, mem_stall_pct, mem_total_mb
 from utils.storage.constrained import host_storage_constrained
 
 _SWARM_DIR = PROJECT_ROOT / "scripts" / "tests" / "deploy" / "swarm"
@@ -34,10 +35,23 @@ _ROLES_DIR = str(PROJECT_ROOT / "roles")
 _SWARM_EXTRAS_VARS = "inventories/development/swarm.yml"
 _DEFAULT_INVENTORY_DIR = "/tmp/inv"  # noqa: S108 - ephemeral swarm-test inventory base in CI
 DISK_FLOOR_MB = 6 * 2**10
+MEM_FLOOR_RATIO = 0.06
+
+
+def _abort(proc: subprocess.Popen[bytes], banner: str, probe: list[str]) -> int:
+    print(banner, flush=True)
+    subprocess.run(probe, check=False)
+    proc.terminate()
+    try:
+        proc.wait(timeout=30)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+        proc.wait()
+    return 75
 
 
 def _run(cmd: list[str], *, env: dict[str, str], label: str) -> int:
-    """Run a matrix step, aborting visibly before the runner disk fills.
+    """Run a matrix step, aborting visibly before the runner disk or RAM fills.
 
     The Actions Worker dies silently on ENOSPC while writing its own logs, so
     a full disk truncates the job without diagnostics; terminating the step at
@@ -50,19 +64,28 @@ def _run(cmd: list[str], *, env: dict[str, str], label: str) -> int:
             return int(proc.wait(timeout=30))
         except subprocess.TimeoutExpired:
             if shutil.disk_usage("/").free < DISK_FLOOR_MB * 2**20:
-                print(
+                return _abort(
+                    proc,
                     "=== swarm-matrix: DISK EXHAUSTION IMMINENT "
                     f"(<{DISK_FLOOR_MB}M free on /) - aborting step ===",
-                    flush=True,
+                    ["df", "-h", "/"],
+                )
+            avail = mem_available_mb()
+            total = mem_total_mb()
+            stall = mem_stall_pct()
+            print(
+                f"=== swarm-matrix: host mem {avail}M/{total}M available, "
+                f"stall60 {stall:.1f}% ===",
+                flush=True,
+            )
+            if total and avail < total * MEM_FLOOR_RATIO:
+                return _abort(
+                    proc,
+                    "=== swarm-matrix: MEMORY EXHAUSTION IMMINENT "
+                    f"({avail}M available, below {MEM_FLOOR_RATIO:.0%} of RAM)"
+                    " - aborting step ===",
+                    ["free", "-m"],
                 )
-                subprocess.run(["df", "-h", "/"], check=False)
-                proc.terminate()
-                try:
-                    proc.wait(timeout=30)
-                except subprocess.TimeoutExpired:
-                    proc.kill()
-                    proc.wait()
-                return 75
 
 
 def _provision(