fix(ci): let the ZFS data-root pools hand freed blocks back to their images

docker_dataroot.sh backs each node's ZFS pool with a sparse loop image
and created the pool without autotrim. ZFS never discarded freed blocks
to the loop device, so the image only grew: in WordPress#0 of run
40625679270 the runner still held 118G after the teardown had removed
every container ("Containers: 0") while only 47G of files remained, and
the pools held ~38G of live data. The step had already aborted at
"DISK EXHAUSTION IMMINENT (<6144M free on /)" with 139G of 145G used
(rc=75). The other ZFS swarm jobs of the run show the same df-minus-du
gap (wordpress1 73G, odoo1 73G, magento0 82G, weblate1 55G, gitlab2
44G); ext4 and btrfs jobs show none.

The pool is now created with -o autotrim=on, so ZFS discards freed
ranges to the loop device, which punches holes into the image file.

test_docker_dataroot runs the script's zfs path against stubs of every
host tool and asserts the single zpool create carries -o autotrim=on;
against the previous script that assertion fails.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
kevinveenbirkenbachandClaude Opus 5.5 committed 2026-10-28 23:07:23 +01:00
1 parent a0bcd59150
commit a5de245bf9
2 files changed
+99 -1

No files matched your search

@@ -0,0 +1,98 @@
from __future__ import annotations
import os
import subprocess
import tempfile
import unittest
from itertools import pairwise
from pathlib import Path
from utils.cache.files import PROJECT_ROOT, read_text
SCRIPT = (
PROJECT_ROOT
/ "scripts"
/ "tests"
/ "deploy"
/ "utils"
/ "filesystem"
/ "docker_dataroot.sh"
)
STUB = (
"#!/usr/bin/env bash\n"
'name="$(basename "$0")"\n'
'printf "%s %s\\n" "${name}" "$*" >>"${STUB_LOG}"\n'
'case "${name}" in\n'
" id) echo 0 ;;\n"
" awk) echo 10 ;;\n"
" losetup) echo /dev/infinito-test-loop ;;\n"
" mountpoint) exit 1 ;;\n"
' zpool) [ "$1" != list ] ;;\n'
' systemctl) [ "$1" != cat ] ;;\n'
" python3) cat >/dev/null ;;\n"
"esac\n"
)
STUBBED = (
"id",
"awk",
"losetup",
"mountpoint",
"zpool",
"zfs",
"systemctl",
"python3",
"modprobe",
"mknod",
"mkdir",
"rm",
"truncate",
)
def apply_zfs() -> tuple[subprocess.CompletedProcess, list[list[str]]]:
"""Run the zfs path of docker_dataroot.sh against stubs of every host tool.
Returns:
The finished process and the argv of every stubbed call, tool first.
"""
with tempfile.TemporaryDirectory() as tmp:
bin_dir = Path(tmp) / "bin"
bin_dir.mkdir()
for name in STUBBED:
stub = bin_dir / name
stub.write_text(STUB)
stub.chmod(0o755)
log = Path(tmp) / "calls.log"
log.touch()
env = {
k: v
for k, v in os.environ.items()
if k not in ("BASH_ENV", "GITHUB_STEP_SUMMARY")
}
env.update(PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}", STUB_LOG=str(log))
done = subprocess.run(
["bash", str(SCRIPT), "zfs", "true", "30G", "unit-node"],
env=env,
capture_output=True,
text=True,
check=False,
)
calls = [line.split() for line in read_text(str(log)).splitlines()]
return done, calls
class TestDockerDatarootZfs(unittest.TestCase):
def test_the_stubbed_zfs_path_applies(self) -> None:
done, _ = apply_zfs()
self.assertEqual(done.returncode, 0, done.stdout + done.stderr)
self.assertIn("status=applied-pending-docker", done.stdout)
def test_the_pool_returns_freed_blocks_to_the_sparse_image(self) -> None:
_, calls = apply_zfs()
creates = [argv for argv in calls if argv[:2] == ["zpool", "create"]]
self.assertEqual(len(creates), 1, calls)
self.assertIn(("-o", "autotrim=on"), list(pairwise(creates[0])))
if __name__ == "__main__":
unittest.main()