Merge branch 'feature/shared-stack-caches' into feature/store-i18n

tests/unit/python/cli/administration/deploy/development/test_compose.py

@@ -154,6 +154,8 @@ class TestComposeUpRetries(unittest.TestCase):
             "INFINITO_GIT_COMMON_DIR": "",
             "INFINITO_CACHE_NETWORK": "",
             "INFINITO_CACHE_STACK": "",
+            "INFINITO_CACHE_SHARED_HOST_PATH": "",
+            "INFINITO_CACHE_HOST_PACMAN_PATH": "",
             "INFINITO_GPU_COUNT": "0",
             "INFINITO_GHCR_MIRROR": "",
         },
@@ -273,6 +275,7 @@ class TestComposeUpRetries(unittest.TestCase):
             "INFINITO_GIT_COMMON_DIR": "",
             "INFINITO_CACHE_NETWORK": "",
             "INFINITO_CACHE_STACK": "",
+            "INFINITO_CACHE_SHARED_HOST_PATH": "/var/cache/infinito/core/cache/shared",
             "INFINITO_GHCR_MIRROR": "",
         },
         clear=False,
@@ -285,6 +288,7 @@ class TestComposeUpRetries(unittest.TestCase):
         compose._bootstrap_package_cache = MagicMock()
         compose._generate_package_frontend_certs = MagicMock()
         compose._install_package_frontend_ca_in_runner = MagicMock()
+        compose._layer_pacman_cache_in_runner = MagicMock()
 
         compose.up(run_entry_init=False)
 
@@ -305,6 +309,7 @@ class TestComposeUpRetries(unittest.TestCase):
         compose._bootstrap_package_cache.assert_called_once()
         compose._generate_package_frontend_certs.assert_called_once()
         compose._install_package_frontend_ca_in_runner.assert_called_once()
+        compose._layer_pacman_cache_in_runner.assert_called_once()
 
     @patch.dict(
         os.environ,
@@ -318,6 +323,7 @@ class TestComposeUpRetries(unittest.TestCase):
             "INFINITO_GIT_COMMON_DIR": "/primary/.git",
             "INFINITO_CACHE_NETWORK": "infinito-nexus-core_default",
             "INFINITO_CACHE_STACK": "",
+            "INFINITO_CACHE_SHARED_HOST_PATH": "/var/cache/infinito/core/cache/shared",
         },
         clear=False,
     )
@@ -330,6 +336,7 @@ class TestComposeUpRetries(unittest.TestCase):
         compose._generate_package_frontend_certs = MagicMock()
         compose._apply_shared_cache = MagicMock()
         compose._install_package_frontend_ca_in_runner = MagicMock()
+        compose._layer_pacman_cache_in_runner = MagicMock()
 
         compose.up(run_entry_init=False)
 
@@ -340,6 +347,63 @@ class TestComposeUpRetries(unittest.TestCase):
         compose._bootstrap_package_cache.assert_not_called()
         compose._apply_shared_cache.assert_called_once()
         compose._install_package_frontend_ca_in_runner.assert_called_once()
+        compose._layer_pacman_cache_in_runner.assert_called_once()
+
+    @patch.dict(
+        os.environ,
+        {
+            "INFINITO_BUILD": "1",
+            "INFINITO_IMAGE": "infinito-debian",
+            "INFINITO_PULL_POLICY": "never",
+            "CI": "",
+            "GITHUB_ACTIONS": "",
+            "INFINITO_RUNNING_ON_GITHUB": "",
+            "INFINITO_GIT_COMMON_DIR": "",
+            "INFINITO_CACHE_NETWORK": "",
+            "INFINITO_CACHE_STACK": "",
+            "INFINITO_CACHE_SHARED_HOST_PATH": "",
+            "INFINITO_GHCR_MIRROR": "",
+        },
+        clear=False,
+    )
+    def test_up_skips_the_pacman_layering_without_a_shared_folder(self) -> None:
+        compose = self._compose()
+        compose._render_coredns_corefile = MagicMock()
+        compose._compose_up_with_retries = MagicMock()
+        compose.wait_for_healthy = MagicMock()
+        compose._bootstrap_package_cache = MagicMock()
+        compose._generate_package_frontend_certs = MagicMock()
+        compose._install_package_frontend_ca_in_runner = MagicMock()
+        compose._layer_pacman_cache_in_runner = MagicMock()
+
+        compose.up(run_entry_init=False)
+
+        compose._install_package_frontend_ca_in_runner.assert_called_once()
+        compose._layer_pacman_cache_in_runner.assert_not_called()
+
+    @patch.dict(
+        os.environ,
+        {
+            "INFINITO_CACHE_SHARED_HOST_PATH": "/var/cache/infinito/core/cache/shared",
+            "INFINITO_CACHE_HOST_PACMAN_PATH": "/var/cache/pacman/pkg",
+        },
+        clear=False,
+    )
+    def test_the_runner_layers_pacman_over_both_caches(self) -> None:
+        compose = self._compose()
+        compose.exec = MagicMock()
+
+        compose._layer_pacman_cache_in_runner()
+
+        compose.exec.assert_called_once_with(
+            [
+                "/usr/local/bin/pacman-cache.sh",
+                "/etc/pacman.conf",
+                "/var/cache/infinito/core/cache/shared",
+                "/var/cache/pacman/pkg",
+            ],
+            live=False,
+        )
 
 
 if __name__ == "__main__":  # pragma: no cover

tests/unit/python/cli/administration/deploy/development/test_env_compose_file_args.py

@@ -29,6 +29,8 @@ _LOCAL_PRIMARY = {
     "INFINITO_GIT_COMMON_DIR": "",
     "INFINITO_CACHE_NETWORK": "",
     "INFINITO_CACHE_STACK": "",
+    "INFINITO_CACHE_SHARED_HOST_PATH": "",
+    "INFINITO_CACHE_HOST_PACMAN_PATH": "",
     "INFINITO_GHCR_MIRROR": "",
     "INFINITO_PUBLISH_PORTS": "",
     "INFINITO_GPU_COUNT": "0",
@@ -40,6 +42,9 @@ _LOCAL_PRIMARY = {
     "INFINITO_EXTRA_MOUNTS": "",
 }
 
+_SHARED = "/var/cache/infinito/core/cache/shared"
+_HOST_PACMAN = "/var/cache/pacman/pkg"
+
 _WORKTREE = {
     **_LOCAL_PRIMARY,
     "INFINITO_INSTANCE": "2",
@@ -132,6 +137,111 @@ class TestComposeFileArgs(unittest.TestCase):
             ["-f", "compose.yml", "-f", "compose/tools.override.yml"],
         )
 
+    @patch.dict(
+        os.environ,
+        {**_LOCAL_PRIMARY, "INFINITO_CACHE_SHARED_HOST_PATH": _SHARED},
+        clear=False,
+    )
+    def test_a_shared_cache_folder_adds_its_mount(self) -> None:
+        self.assertEqual(
+            compose_file_args(),
+            [
+                "-f",
+                "compose.yml",
+                "-f",
+                "compose/cache.override.yml",
+                "-f",
+                "compose/shared-cache.override.yml",
+                "-f",
+                "compose/tools.override.yml",
+            ],
+        )
+
+    @patch.dict(
+        os.environ,
+        {
+            **_LOCAL_PRIMARY,
+            "INFINITO_CACHE_SHARED_HOST_PATH": _SHARED,
+            "INFINITO_CACHE_HOST_PACMAN_PATH": _HOST_PACMAN,
+        },
+        clear=False,
+    )
+    def test_a_host_pacman_cache_adds_its_read_only_mount(self) -> None:
+        self.assertEqual(
+            compose_file_args(),
+            [
+                "-f",
+                "compose.yml",
+                "-f",
+                "compose/cache.override.yml",
+                "-f",
+                "compose/shared-cache.override.yml",
+                "-f",
+                "compose/pacman.override.yml",
+                "-f",
+                "compose/tools.override.yml",
+            ],
+        )
+
+    @patch.dict(
+        os.environ,
+        {**_LOCAL_PRIMARY, "INFINITO_CACHE_HOST_PACMAN_PATH": _HOST_PACMAN},
+        clear=False,
+    )
+    def test_a_host_pacman_cache_needs_the_shared_folder(self) -> None:
+        self.assertEqual(
+            compose_file_args(),
+            [
+                "-f",
+                "compose.yml",
+                "-f",
+                "compose/cache.override.yml",
+                "-f",
+                "compose/tools.override.yml",
+            ],
+        )
+
+    @patch.dict(
+        os.environ,
+        {
+            **_LOCAL_PRIMARY,
+            "CI": "true",
+            "INFINITO_CACHE_SHARED_HOST_PATH": _SHARED,
+            "INFINITO_CACHE_HOST_PACMAN_PATH": _HOST_PACMAN,
+        },
+        clear=False,
+    )
+    def test_the_cache_folders_stay_out_without_the_cache_stack(self) -> None:
+        self.assertEqual(
+            compose_file_args(),
+            ["-f", "compose.yml", "-f", "compose/tools.override.yml"],
+        )
+
+    def test_an_env_without_the_cache_keys_layers_neither_override(self) -> None:
+        stale = {
+            k: v
+            for k, v in _LOCAL_PRIMARY.items()
+            if k
+            not in (
+                "INFINITO_CACHE_SHARED_HOST_PATH",
+                "INFINITO_CACHE_HOST_PACMAN_PATH",
+            )
+        }
+        with patch.dict(os.environ, stale, clear=False):
+            os.environ.pop("INFINITO_CACHE_SHARED_HOST_PATH", None)
+            os.environ.pop("INFINITO_CACHE_HOST_PACMAN_PATH", None)
+            self.assertEqual(
+                compose_file_args(),
+                [
+                    "-f",
+                    "compose.yml",
+                    "-f",
+                    "compose/cache.override.yml",
+                    "-f",
+                    "compose/tools.override.yml",
+                ],
+            )
+
     @patch.dict(
         os.environ,
         {**_LOCAL_PRIMARY, "INFINITO_PUBLISH_PORTS": "false"},

tests/unit/python/compose/test_shared_cache_mounts.py

@@ -0,0 +1,123 @@
+"""The cache overrides mount the shared folder at its host path; nothing always layered needs it."""
+
+from __future__ import annotations
+
+import re
+import unittest
+
+from utils.cache.files import PROJECT_ROOT, read_text
+from utils.cache.render import cache_consumer_yml
+from utils.cache.yaml import load_yaml_any
+
+SHARED = "INFINITO_CACHE_SHARED_HOST_PATH"
+HOST_PACMAN = "INFINITO_CACHE_HOST_PACMAN_PATH"
+PACMAN_SCRIPT = "/usr/local/bin/pacman-cache.sh"
+LAB_NODES = (
+    "nfs-server",
+    "swarm-mgr-01",
+    "swarm-wrk-01",
+    "swarm-wrk-02",
+    "backup-host",
+)
+
+
+def _var(text: str) -> str:
+    """Return the variable a compose interpolation names, empty for a literal."""
+    found = re.fullmatch(r"\$\{([A-Z_][A-Z0-9_]*)(?::\?[^}]*)?\}", text.strip())
+    return found.group(1) if found else ""
+
+
+def _short(entry: str) -> list[str]:
+    """Split a short-syntax mount on the colons outside interpolations."""
+    parts, depth, current = [], 0, ""
+    for char in entry:
+        if char == "{" and current.endswith("$"):
+            depth += 1
+        elif char == "}" and depth:
+            depth -= 1
+        if char == ":" and not depth:
+            parts.append(current)
+            current = ""
+            continue
+        current += char
+    return [*parts, current]
+
+
+def _load(*parts: str) -> dict:
+    return load_yaml_any(str(PROJECT_ROOT.joinpath(*parts)))
+
+
+class TestComposeSlot(unittest.TestCase):
+    def setUp(self) -> None:
+        self.runner = _load("compose", "shared-cache.override.yml")["services"][
+            "infinito"
+        ]
+
+    def test_the_shared_folder_is_bound_read_write_at_its_host_path(self) -> None:
+        binds = [
+            v
+            for v in self.runner["volumes"]
+            if isinstance(v, dict) and _var(v["source"]) == SHARED
+        ]
+        self.assertEqual(len(binds), 1, self.runner["volumes"])
+        self.assertEqual(_var(binds[0]["target"]), SHARED)
+        self.assertFalse(binds[0].get("read_only", False))
+
+    def test_pip_and_npm_cache_below_the_shared_folder(self) -> None:
+        env = self.runner["environment"]
+        self.assertRegex(env["PIP_CACHE_DIR"], rf"^\$\{{{SHARED}[^}}]*\}}/pip$")
+        self.assertRegex(env["npm_config_cache"], rf"^\$\{{{SHARED}[^}}]*\}}/npm$")
+
+    def test_the_pacman_script_is_mounted(self) -> None:
+        targets = [_short(v)[1] for v in self.runner["volumes"] if isinstance(v, str)]
+        self.assertIn(PACMAN_SCRIPT, targets)
+
+
+class TestAlwaysLayeredFiles(unittest.TestCase):
+    def test_the_always_layered_files_never_name_the_cache_variables(self) -> None:
+        always = (
+            cache_consumer_yml(),
+            read_text(str(PROJECT_ROOT / "compose.yml")),
+            read_text(str(PROJECT_ROOT / "compose" / "cache.override.yml")),
+            read_text(str(PROJECT_ROOT / "compose" / "swarm" / "cache.override.yml")),
+        )
+        for text in always:
+            self.assertNotIn(SHARED, text)
+            self.assertNotIn(HOST_PACMAN, text)
+
+
+class TestHostPacmanOverride(unittest.TestCase):
+    def test_the_host_pacman_cache_is_mounted_read_only_at_its_host_path(self) -> None:
+        (bind,) = _load("compose", "pacman.override.yml")["services"]["infinito"][
+            "volumes"
+        ]
+        self.assertEqual(_var(bind["source"]), HOST_PACMAN)
+        self.assertEqual(_var(bind["target"]), HOST_PACMAN)
+        self.assertTrue(bind["read_only"])
+
+
+class TestSwarmLabNodes(unittest.TestCase):
+    def test_every_node_binds_the_shared_folder_read_write(self) -> None:
+        services = _load("compose", "swarm", "shared-cache.override.yml")["services"]
+        for node in LAB_NODES:
+            with self.subTest(node=node):
+                mounts = [_short(v) for v in services[node]["volumes"]]
+                shared = [m for m in mounts if _var(m[0]) == SHARED]
+                self.assertEqual(len(shared), 1, mounts)
+                self.assertEqual(_var(shared[0][1]), SHARED)
+                self.assertEqual(len(shared[0]), 2, "read-write: no :ro suffix")
+                self.assertIn(PACMAN_SCRIPT, [m[1] for m in mounts])
+
+    def test_every_node_reads_the_host_pacman_cache_read_only(self) -> None:
+        services = _load("compose", "swarm", "pacman.override.yml")["services"]
+        for node in LAB_NODES:
+            with self.subTest(node=node):
+                (mount,) = [_short(v) for v in services[node]["volumes"]]
+                self.assertEqual(
+                    [_var(mount[0]), _var(mount[1]), mount[2]],
+                    [HOST_PACMAN, HOST_PACMAN, "ro"],
+                )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/python/roles/sys-ctl-cln-disc-space/files/shell/test_script.py

@@ -0,0 +1,57 @@
+"""The disc-space cleanup cleans only the pacman cache directories it can write."""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+from utils.cache.files import PROJECT_ROOT
+
+SCRIPT = (
+    PROJECT_ROOT / "roles" / "sys-ctl-cln-disc-space" / "files" / "shell" / "script.sh"
+)
+STUBS = {
+    "df": '#!/bin/sh\necho "Use%"\n',
+    "find": "#!/bin/sh\nexit 0\n",
+    "systemctl": "#!/bin/sh\nexit 1\n",
+    "pacman": '#!/bin/sh\nprintf "%s\\n" "$*" >>"${STUB_LOG}"\n',
+}
+
+
+class TestPacmanCleanup(unittest.TestCase):
+    def test_a_cache_dir_it_cannot_write_is_skipped(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            bin_dir = Path(tmp) / "bin"
+            bin_dir.mkdir()
+            writable = Path(tmp) / "shared" / "pacman"
+            writable.mkdir(parents=True)
+            unwritable = Path(tmp) / "host-pkg-not-writable"
+            stubs = {
+                **STUBS,
+                "pacman-conf": f'#!/bin/sh\nprintf "%s/\\n%s/\\n" "{unwritable}" "{writable}"\n',
+            }
+            for name, body in stubs.items():
+                stub = bin_dir / name
+                stub.write_text(body)
+                stub.chmod(0o755)
+            for name in ("sed", "yes"):
+                (bin_dir / name).symlink_to(shutil.which(name))
+            log = Path(tmp) / "pacman.log"
+            log.touch()
+            done = subprocess.run(
+                ["/bin/sh", str(SCRIPT), "90", "--force"],
+                env={"PATH": str(bin_dir), "STUB_LOG": str(log)},
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+            calls = log.read_text().splitlines()  # nocheck: cache-read  stub call log
+        self.assertEqual(done.returncode, 0, done.stdout + done.stderr)
+        self.assertEqual(calls, [f"-Sc --cachedir {writable}/"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/python/scripts/docker/cache/test_pacman.py

@@ -0,0 +1,223 @@
+from __future__ import annotations
+
+import shlex
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+from utils.cache.files import PROJECT_ROOT
+
+SCRIPT = PROJECT_ROOT / "scripts" / "docker" / "cache" / "pacman.sh"
+CONF = """\
+[options]
+#CacheDir    = /var/cache/pacman/pkg/
+CacheDir = /var/cache/pacman/pkg/
+HoldPkg     = pacman glibc
+Architecture = auto
+
+[core]
+Include = /etc/pacman.d/mirrorlist
+"""
+HOOK = Path("pacman.d") / "hooks" / "infinito-shared-cache.hook"
+
+
+def layer(conf: Path, shared: Path, host: str) -> subprocess.CompletedProcess:
+    """Run pacman.sh against ``conf``.
+
+    Args:
+        conf: the pacman.conf to rewrite.
+        shared: the shared cache root.
+        host: the read-only host cache, empty for none.
+    """
+    return subprocess.run(
+        ["bash", str(SCRIPT), str(conf), str(shared), host],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+
+class TestPacmanCacheLayering(unittest.TestCase):
+    def setUp(self) -> None:
+        self._tmp = tempfile.TemporaryDirectory()
+        self.tmp = Path(self._tmp.name)
+        self.conf = self.tmp / "pacman.conf"
+        self.conf.write_text(CONF)
+        self.shared = self.tmp / "shared"
+        self.host = self.tmp / "host-pkg"
+        self.host.mkdir()
+
+    def tearDown(self) -> None:
+        self._tmp.cleanup()
+
+    def _text(self) -> str:
+        return self.conf.read_text()  # nocheck: cache-read  the script rewrites it
+
+    def _hook(self) -> str:
+        hook = self.tmp / HOOK
+        return hook.read_text()  # nocheck: cache-read  the script writes it
+
+    def _options(self) -> list[str]:
+        lines = self._text().splitlines()
+        return lines[: lines.index("") if "" in lines else len(lines)]
+
+    def _stage(self, conf: Path) -> Path:
+        prefix = f"CacheDir = {self.shared}/pacman/node-"
+        text = conf.read_text()  # nocheck: cache-read  the script rewrites it
+        lines = text.splitlines()
+        stages = [line for line in lines if line.startswith(prefix)]
+        self.assertEqual(len(stages), 1, lines)
+        self.assertTrue(stages[0].endswith("/"), stages[0])
+        return Path(stages[0].removeprefix("CacheDir = ").rstrip("/"))
+
+    def test_host_cache_comes_first_then_the_node_download_dir_then_the_shared_cache(
+        self,
+    ) -> None:
+        done = layer(self.conf, self.shared, str(self.host))
+        self.assertEqual(done.returncode, 0, done.stderr)
+        self.assertEqual(
+            self._options()[:4],
+            [
+                "[options]",
+                f"CacheDir = {self.host}/",
+                f"CacheDir = {self._stage(self.conf)}/",
+                f"CacheDir = {self.shared}/pacman/",
+            ],
+        )
+
+    def test_every_other_cache_dir_is_dropped_and_the_rest_kept(self) -> None:
+        layer(self.conf, self.shared, str(self.host))
+        text = self._text()
+        self.assertNotIn("CacheDir = /var/cache/pacman/pkg/", text)
+        self.assertIn("#CacheDir    = /var/cache/pacman/pkg/", text)
+        self.assertIn("[core]\nInclude = /etc/pacman.d/mirrorlist\n", text)
+
+    def test_the_shared_pacman_dir_and_a_traversable_node_download_dir_are_created(
+        self,
+    ) -> None:
+        layer(self.conf, self.shared, str(self.host))
+        stage = self._stage(self.conf)
+        self.assertTrue((self.shared / "pacman").is_dir())
+        self.assertTrue(stage.is_dir())
+        self.assertEqual(stage.parent, self.shared / "pacman")
+        self.assertEqual(stage.stat().st_mode & 0o777, 0o755)
+
+    def test_a_second_run_changes_nothing(self) -> None:
+        layer(self.conf, self.shared, str(self.host))
+        first, first_hook = self._text(), self._hook()
+        done = layer(self.conf, self.shared, str(self.host))
+        self.assertEqual(done.returncode, 0, done.stderr)
+        self.assertEqual(self._text(), first)
+        self.assertEqual(self._hook(), first_hook)
+        self.assertEqual(len(list((self.shared / "pacman").glob("node-*"))), 1)
+
+    def test_a_second_run_recreates_a_removed_node_download_dir_under_its_name(
+        self,
+    ) -> None:
+        layer(self.conf, self.shared, str(self.host))
+        stage = self._stage(self.conf)
+        stage.rmdir()
+        first = self._text()
+        layer(self.conf, self.shared, str(self.host))
+        self.assertEqual(self._text(), first)
+        self.assertTrue(stage.is_dir())
+
+    def test_two_nodes_on_one_shared_folder_download_into_different_dirs(self) -> None:
+        other = self.tmp / "other" / "pacman.conf"
+        other.parent.mkdir()
+        other.write_text(CONF)
+        layer(self.conf, self.shared, str(self.host))
+        layer(other, self.shared, str(self.host))
+        self.assertNotEqual(self._stage(self.conf), self._stage(other))
+        self.assertTrue(self._stage(other).is_dir())
+
+    def test_an_empty_host_cache_leaves_the_node_download_dir_and_the_shared_layer(
+        self,
+    ) -> None:
+        layer(self.conf, self.shared, "")
+        self.assertEqual(
+            self._options()[:3],
+            [
+                "[options]",
+                f"CacheDir = {self._stage(self.conf)}/",
+                f"CacheDir = {self.shared}/pacman/",
+            ],
+        )
+        self.assertEqual(
+            sum(line.startswith("CacheDir") for line in self._options()), 2
+        )
+
+    def test_the_hook_runs_after_every_install_and_upgrade(self) -> None:
+        layer(self.conf, self.shared, str(self.host))
+        hook = self._hook().splitlines()
+        for line in (
+            "Operation = Install",
+            "Operation = Upgrade",
+            "Type = Package",
+            "Target = *",
+            "When = PostTransaction",
+        ):
+            self.assertIn(line, hook)
+
+    def test_the_hook_publishes_finished_downloads_and_keeps_partials(self) -> None:
+        layer(self.conf, self.shared, str(self.host))
+        stage = self._stage(self.conf)
+        own = self.shared / "pacman"
+        for name in ("a-1-1-any.pkg.tar.zst", "a-1-1-any.pkg.tar.zst.sig"):
+            (stage / name).write_text(name)
+        (stage / "b-1-1-any.pkg.tar.zst.part").write_text("partial")
+        (own / "a-1-1-any.pkg.tar.zst").write_text("stale")
+        (exec_line,) = [
+            line for line in self._hook().splitlines() if line.startswith("Exec = ")
+        ]
+        done = subprocess.run(
+            shlex.split(exec_line.removeprefix("Exec = ")),
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        self.assertEqual(done.returncode, 0, done.stderr)
+        self.assertEqual(
+            sorted(path.name for path in stage.iterdir()),
+            ["b-1-1-any.pkg.tar.zst.part"],
+        )
+        published = own / "a-1-1-any.pkg.tar.zst"
+        self.assertEqual(
+            published.read_text(),  # nocheck: cache-read  the hook moves it
+            "a-1-1-any.pkg.tar.zst",
+        )
+        self.assertTrue((own / "a-1-1-any.pkg.tar.zst.sig").is_file())
+
+    def test_a_system_without_pacman_conf_is_left_alone(self) -> None:
+        done = layer(self.tmp / "absent.conf", self.shared, str(self.host))
+        self.assertEqual(done.returncode, 0, done.stderr)
+        self.assertFalse(self.shared.exists())
+        self.assertFalse((self.tmp / HOOK).exists())
+
+    def test_a_host_cache_that_is_not_mounted_fails(self) -> None:
+        done = layer(self.conf, self.shared, str(self.tmp / "missing"))
+        self.assertNotEqual(done.returncode, 0)
+        self.assertEqual(self._text(), CONF)
+        self.assertFalse((self.tmp / HOOK).exists())
+
+    def test_a_hook_dir_override_fails(self) -> None:
+        conf = CONF.replace("[options]\n", "[options]\nHookDir = /srv/hooks/\n")
+        self.conf.write_text(conf)
+        done = layer(self.conf, self.shared, str(self.host))
+        self.assertNotEqual(done.returncode, 0)
+        self.assertEqual(self._text(), conf)
+        self.assertFalse(self.shared.exists())
+
+    def test_a_missing_shared_root_fails(self) -> None:
+        done = subprocess.run(
+            ["bash", str(SCRIPT), str(self.conf)],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        self.assertNotEqual(done.returncode, 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/python/scripts/tests/deploy/swarm/utils/test_release.py

@@ -0,0 +1,171 @@
+"""release.sh saves the act job's artefacts, then releases the cluster in every outcome."""
+
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+from utils.cache.files import PROJECT_ROOT, read_text
+
+RELEASE = (
+    PROJECT_ROOT / "scripts" / "tests" / "deploy" / "swarm" / "utils" / "release.sh"
+)
+MAKEFILE = PROJECT_ROOT / "Makefile"
+DOCKER_STUB = (
+    "#!/usr/bin/env bash\n"
+    'printf "docker %s\\n" "$*" >>"${STUB_LOG}"\n'
+    'if [ "$1" = ps ]; then echo act-id; exit 0; fi\n'
+    'if [ "$1" = cp ]; then\n'
+    '  if [ -n "${STUB_CP_ERROR}" ]; then echo "${STUB_CP_ERROR}" >&2; exit 1; fi\n'
+    "  exit 0\n"
+    "fi\n"
+)
+TEARDOWN_STUB = (
+    "#!/usr/bin/env bash\n"
+    'printf "teardown SWARM_NAME=%s KEEP=%s\\n" "${SWARM_NAME}" '
+    '"${INFINITO_KEEP_SWARM_NODES}" >>"${STUB_LOG}"\n'
+)
+DOWN_ACT_STUB = '#!/usr/bin/env bash\nprintf "down_act_outer\\n" >>"${STUB_LOG}"\n'
+
+
+def release(
+    keep: str, drill_rc: int, cp_error: str = ""
+) -> tuple[subprocess.CompletedProcess, list[str]]:
+    """Run release.sh around a drill that exits ``drill_rc``.
+
+    Args:
+        keep: the ``keep`` value make hands over.
+        drill_rc: exit code of the stub drill.
+        cp_error: stderr the stub ``docker cp`` fails with, empty for success.
+
+    Returns:
+        The finished process and every stub call in order.
+    """
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp) / "repo"
+        utils_dir = root / "scripts" / "tests" / "deploy" / "swarm" / "utils"
+        (utils_dir / "clean").mkdir(parents=True)
+        (utils_dir / "release.sh").symlink_to(RELEASE)
+        act_dir = root / "scripts" / "tests" / "deploy" / "act"
+        act_dir.mkdir(parents=True)
+        bin_dir = Path(tmp) / "bin"
+        bin_dir.mkdir()
+        for path, body in (
+            (utils_dir / "clean" / "teardown.sh", TEARDOWN_STUB),
+            (act_dir / "down_act_outer.sh", DOWN_ACT_STUB),
+            (bin_dir / "docker", DOCKER_STUB),
+        ):
+            path.write_text(body)
+            path.chmod(0o755)
+        log = Path(tmp) / "calls.log"
+        log.touch()
+        env = {k: v for k, v in os.environ.items() if k != "BASH_ENV"}
+        env.update(
+            PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
+            STUB_LOG=str(log),
+            STUB_CP_ERROR=cp_error,
+            SWARM_NAME="web-app-demo",
+            keep=keep,
+            INFINITO_CONTAINER="own",
+            INFINITO_RESCUE_DIAGNOSTICS_BASE=str(Path(tmp) / "rescue"),
+        )
+        done = subprocess.run(
+            [
+                "bash",
+                str(utils_dir / "release.sh"),
+                "bash",
+                "-c",
+                f'echo drill >>"{log}"; exit {drill_rc}',
+            ],
+            env=env,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        return done, log.read_text().splitlines()  # nocheck: cache-read  stub call log
+
+
+class TestReleaseAfterRescue(unittest.TestCase):
+    def test_a_green_drill_saves_then_releases(self) -> None:
+        done, calls = release("", 0)
+        self.assertEqual(done.returncode, 0, done.stderr)
+        self.assertEqual(calls[0], "drill")
+        self.assertTrue(calls[2].startswith("docker cp act-id:"), calls)
+        self.assertEqual(calls[-2], "teardown SWARM_NAME=web-app-demo KEEP=false")
+        self.assertEqual(calls[-1], "down_act_outer")
+
+    def test_a_failed_drill_releases_too_and_keeps_its_exit_code(self) -> None:
+        done, calls = release("false", 3)
+        self.assertEqual(done.returncode, 3)
+        self.assertIn("teardown SWARM_NAME=web-app-demo KEEP=false", calls)
+        self.assertEqual(calls[-1], "down_act_outer")
+
+    def test_the_rescue_bundle_is_copied_before_anything_is_removed(self) -> None:
+        _, calls = release("", 1)
+        copies = [i for i, call in enumerate(calls) if call.startswith("docker cp ")]
+        self.assertEqual(len(copies), 2, calls)
+        self.assertLess(
+            max(copies), calls.index(next(c for c in calls if c.startswith("teardown")))
+        )
+
+    def test_keep_true_saves_but_leaves_the_cluster_up(self) -> None:
+        done, calls = release("true", 1)
+        self.assertEqual(done.returncode, 1)
+        self.assertTrue(any(call.startswith("docker cp ") for call in calls))
+        self.assertFalse([call for call in calls if call.startswith("teardown")])
+        self.assertNotIn("down_act_outer", calls)
+
+    def test_an_absent_bundle_does_not_stop_the_release(self) -> None:
+        done, calls = release(
+            "", 0, cp_error="Error: No such container:path: act-id:/tmp/x"
+        )
+        self.assertEqual(done.returncode, 0, done.stderr)
+        self.assertEqual(calls[-1], "down_act_outer")
+
+    def test_a_failed_copy_keeps_the_act_container_and_fails(self) -> None:
+        done, calls = release("", 0, cp_error="no space left on device")
+        self.assertNotEqual(done.returncode, 0)
+        self.assertIn("teardown SWARM_NAME=web-app-demo KEEP=false", calls)
+        self.assertNotIn("down_act_outer", calls)
+        self.assertIn("make swarm-down name=web-app-demo", done.stderr)
+
+    def test_an_unknown_keep_value_runs_nothing(self) -> None:
+        done, calls = release("yes", 0)
+        self.assertEqual(done.returncode, 2)
+        self.assertEqual(calls, [])
+
+
+class TestSwarmZombieWiring(unittest.TestCase):
+    def setUp(self) -> None:
+        text = read_text(str(MAKEFILE))
+        self.recipe = re.search(
+            r"^swarm-zombie:.*?(?=^\.PHONY)", text, re.DOTALL | re.MULTILINE
+        )[0]
+
+    def test_the_drill_runs_inside_release(self) -> None:
+        self.assertIn(
+            "bash scripts/tests/deploy/swarm/utils/release.sh bash scripts/tests/deploy/act/workflow.sh",
+            self.recipe,
+        )
+
+    def test_an_env_without_the_cache_keys_still_starts_the_drill(self) -> None:
+        for key in (
+            "INFINITO_CACHE_SHARED_HOST_PATH",
+            "INFINITO_CACHE_HOST_PACMAN_PATH",
+        ):
+            self.assertIn(f"{key}=$${{{key}:-}};", self.recipe)
+
+    def test_keep_decides_whether_the_act_job_keeps_the_nodes(self) -> None:
+        self.assertIn("keep='$(keep)'", self.recipe)
+        self.assertIn(
+            "INFINITO_KEEP_SWARM_NODES=$(if $(filter true,$(keep)),true,false)",
+            self.recipe,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/python/scripts/tests/deploy/test_roundtrip.py

@@ -41,6 +41,23 @@ def swarm_network(disable: str, *, multi_accepted: bool) -> str:
     Returns:
         The ``network=`` value the roundtrip hands to ``swarm-zombie``.
     """
+    calls = swarm_make_calls(disable, multi_accepted=multi_accepted, keep="true")
+    zombie = next(call for call in calls if " swarm-zombie " in call)
+    return re.search(r"(?:^| )network=(\S*)", zombie).group(1)
+
+
+def swarm_make_calls(disable: str, *, multi_accepted: bool, keep: str) -> list[str]:
+    """Run the roundtrip's swarm step against stubs and return every make call.
+
+    Args:
+        disable: the ``disable`` env the roundtrip runs with.
+        multi_accepted: whether the stubbed validator accepts the role's
+            ``multi`` row.
+        keep: the ``keep`` env the roundtrip runs with.
+
+    Returns:
+        The argument lines the stubbed ``make`` received, in order.
+    """
     with tempfile.TemporaryDirectory() as tmp:
         root = Path(tmp) / "repo"
         script = root / "scripts" / "tests" / "deploy" / "roundtrip.sh"
@@ -69,7 +86,7 @@ def swarm_network(disable: str, *, multi_accepted: bool) -> str:
             STUB_MULTI_RC="0" if multi_accepted else "1",
             apps=APP,
             modes="swarm",
-            keep="true",
+            keep=keep,
             disable=disable,
         )
         done = subprocess.run(
@@ -81,9 +98,7 @@ def swarm_network(disable: str, *, multi_accepted: bool) -> str:
         )
         if done.returncode != 0:
             raise AssertionError(f"roundtrip failed: {done.stderr}")
-        calls = log.read_text().splitlines()  # nocheck: cache-read  stub call log
-    zombie = next(call for call in calls if " swarm-zombie " in call)
-    return re.search(r"(?:^| )network=(\S*)", zombie).group(1)
+        return log.read_text().splitlines()  # nocheck: cache-read  stub call log
 
 
 class TestRoundtripMarker(unittest.TestCase):
@@ -140,5 +155,21 @@ class TestRoundtripSwarmNetwork(unittest.TestCase):
         self.assertEqual(swarm_network("tor-smtp", multi_accepted=True), "multi")
 
 
+class TestRoundtripSwarmRelease(unittest.TestCase):
+    def test_an_empty_keep_reaches_swarm_zombie_which_releases(self) -> None:
+        calls = swarm_make_calls("", multi_accepted=False, keep="")
+        zombie = next(call for call in calls if " swarm-zombie " in call)
+        self.assertRegex(zombie, r"(?:^| )keep=(?: |$)")
+
+    def test_keep_true_reaches_swarm_zombie(self) -> None:
+        calls = swarm_make_calls("", multi_accepted=False, keep="true")
+        zombie = next(call for call in calls if " swarm-zombie " in call)
+        self.assertRegex(zombie, r"(?:^| )keep=true(?: |$)")
+
+    def test_the_roundtrip_runs_no_release_of_its_own(self) -> None:
+        calls = swarm_make_calls("", multi_accepted=False, keep="")
+        self.assertFalse([call for call in calls if " swarm-down " in call], calls)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/unit/python/utils/env/handlers/infinito/test_cache_host_pacman_path.py

@@ -0,0 +1,83 @@
+"""Unit tests for :mod:`utils.env.handlers.infinito.cache.host_pacman_path`."""
+
+from __future__ import annotations
+
+import os
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from utils.env.builder import BuildContext, EnvBuilder
+from utils.env.handlers.infinito.cache import host_pacman_path as handler
+
+HELPER = "utils.env.handlers.infinito.cache.host_pacman_path.run_helper"
+ARCH = 'NAME="Arch Linux"\nID=arch\n'
+MANJARO = 'NAME="Manjaro Linux"\nID=manjaro\nID_LIKE=arch\n'
+ENDEAVOUR = 'NAME="EndeavourOS"\nID="endeavouros"\nID_LIKE="arch"\n'
+
+
+@patch.dict(os.environ, {handler.KEY: ""}, clear=False)
+class TestHostPacmanPath(unittest.TestCase):
+    def setUp(self) -> None:
+        self._tmp = tempfile.TemporaryDirectory()
+        self.tmp = Path(self._tmp.name)
+        self.cache = self.tmp / "pkg"
+        self.cache.mkdir()
+
+    def tearDown(self) -> None:
+        self._tmp.cleanup()
+
+    def _apply(self, os_release: str | None, distro: str, conf_output: str) -> str:
+        path = self.tmp / "os-release"
+        if os_release is not None:
+            path.write_text(os_release)
+        eb = EnvBuilder()
+        eb.set("INFINITO_DISTRO", distro)
+        ctx = BuildContext(
+            static={},
+            static_comments={},
+            repo_root=Path("/repo"),
+            on_gha=False,
+            on_act=False,
+        )
+        with (
+            patch.object(handler, "OS_RELEASE", path),
+            patch(HELPER, return_value=conf_output),
+        ):
+            handler.apply(eb, ctx)
+        return eb.values[handler.KEY]
+
+    def test_an_arch_host_offers_its_first_cache_dir_to_an_arch_stack(self) -> None:
+        self.assertEqual(
+            self._apply(ARCH, "arch", f"{self.cache}/\n/var/cache/other/\n"),
+            str(self.cache),
+        )
+
+    def test_a_manjaro_host_offers_nothing(self) -> None:
+        self.assertEqual(self._apply(MANJARO, "arch", f"{self.cache}/\n"), "")
+
+    def test_an_arch_derivative_offers_nothing(self) -> None:
+        self.assertEqual(self._apply(ENDEAVOUR, "arch", f"{self.cache}/\n"), "")
+
+    def test_a_host_without_os_release_offers_nothing(self) -> None:
+        self.assertEqual(self._apply(None, "arch", f"{self.cache}/\n"), "")
+
+    def test_a_stack_without_pacman_gets_nothing(self) -> None:
+        self.assertEqual(self._apply(ARCH, "debian", f"{self.cache}/\n"), "")
+
+    def test_an_arch_host_without_pacman_conf_offers_nothing(self) -> None:
+        self.assertEqual(self._apply(ARCH, "arch", ""), "")
+
+    def test_a_cache_dir_that_does_not_exist_is_not_offered(self) -> None:
+        self.assertEqual(self._apply(ARCH, "arch", f"{self.cache}/missing/\n"), "")
+
+    def test_an_exported_path_wins(self) -> None:
+        with patch.dict(os.environ, {handler.KEY: "/elsewhere"}, clear=False):
+            self.assertEqual(
+                self._apply(MANJARO, "arch", f"{self.cache}/\n"), "/elsewhere"
+            )
+
+
+if __name__ == "__main__":  # pragma: no cover
+    unittest.main()