feat(cache): serve every checkout's own cache declarations

A checkout that declared a new cache host rendered the artefacts and reached nothing: the Nexus proxy repository, the frontend's leaf certificate and the nginx upstream map all live in the running stack, which only the owning checkout set up. cache-apply installs all three and reloads the frontend, and compose up runs it for a checkout that merely joins the network, so the manual call is left for a declaration changed between two bring-ups.

The map source is no longer selectable. INFINITO_CACHE_CONF_SOURCE existed to let a worktree read the primary checkout's map instead of its own, but the frontend mounts one map and the install replaces it, so the checkout that applied last wins either way: the knob chose which file to point at, never who prevailed. Dropping it also retires the git subprocess that resolved the owning checkout on every env build.

Two checkouts sharing a stack therefore overwrite each other's map, which the documentation now states instead of implying a selector could prevent it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Makefile

@@ -100,6 +100,13 @@ build-no-cache-all:
 		INFINITO_DISTRO="$$d" "$(MAKE)" build-no-cache; \
 	done
 
+.PHONY: cache-apply
+# Install this checkout's cache declarations into the running cache stack.
+# Note: creates the Nexus proxy repos, issues the frontend certs, installs the upstream map the frontend mounts and reloads it.
+# Note: `make compose-up` runs it for a checkout that shares another one's cache stack; call it directly after changing a `cache:` declaration.
+cache-apply:
+	@bash scripts/system/cache/apply.sh
+
 .PHONY: cheat
 # Print the operator prompt cheatsheet from docs/contributing/tools/agents/cheatsheet.md.
 cheat:
@@ -293,10 +300,8 @@ dotenv:
 # Note: avoids stale BASH_ENV INFINITO_* values pinning via setdefault.
 dotenv-force:
 	@rm -f .env
-	@env -i HOME="$${HOME}" PATH="$${PATH}" \
-		INFINITO_CACHE_CONF_SOURCE="$${INFINITO_CACHE_CONF_SOURCE:-}" \
-		python3 -m cli.meta.env
-	@python3 -m cli.meta.cache
+	@env -i HOME="$${HOME}" PATH="$${PATH}" python3 -m cli.meta.env
+	@set -a; . ./.env; set +a; python3 -m cli.meta.cache
 
 .PHONY: environment-bootstrap
 # Bootstrap the local development environment.
@@ -931,7 +936,7 @@ worktree-prune:
 .PHONY: worktree-up
 # Check a branch out into an isolated worktree with its own subnet, ports and container names.
 # Usage: make worktree-up branch=<name> [base=<dir>]
-# Note: the worktree shares the primary checkout's cache stack instead of starting its own.
+# Note: the worktree shares the primary checkout's cache stack instead of starting its own; `make cache-apply` serves its own cache declarations from it.
 # Param branch: branch to check out (required).
 # Param base: parent directory for the worktree (default ~/.local/share/worktrees/<domain>/<account>/<repo>).
 worktree-up:

aliases

@@ -9,6 +9,7 @@ alias i8bumi='m build-missing'
 alias i8buno='m build-no-cache'
 alias i8bunoca='m build-no-cache-all'
 alias i8ca='m act-debug'
+alias i8caap='m cache-apply'
 alias i8caru='m act-runner-image'
 alias i8cawo='m act-workflow'
 alias i8ce='m cheat'

cli/administration/deploy/development/compose.py

@@ -141,6 +141,31 @@ class Compose:
             text=True,
         )
 
+    def _apply_shared_cache(self, env: dict[str, str]) -> None:
+        """Serve this checkout's cache declarations from the shared stack.
+
+        The owner creates the proxy repositories and the certificates while
+        it brings its own containers up. An instance that only joins the
+        network starts none of them, so without this its `cache:` entries
+        exist on disk and nowhere else: the hijacked hostname resolves to a
+        frontend that has no server block for it.
+        """
+        helper = self.repo_root / "scripts" / "system" / "cache" / "apply.sh"
+        print(">>> Applying this checkout's cache declarations to the shared stack")
+        r = subprocess.run(
+            ["bash", str(helper)],
+            cwd=self.repo_root,
+            env=env,
+            check=False,
+            text=True,
+        )
+        if r.returncode != 0:
+            print(
+                f">>> WARNING: cache apply exited rc={r.returncode}; the shared "
+                "frontend keeps serving the map it already had. Start the owning "
+                f"stack and re-run `make cache-apply` ({helper})."
+            )
+
     def _install_package_frontend_ca_in_runner(self) -> None:
         """Install the frontend CA in the runner trust store. Idempotent."""
         print(">>> Installing package-cache-frontend CA into runner trust store")
@@ -213,6 +238,8 @@ class Compose:
 
         if self.profile.owns_cache_stack():
             self._bootstrap_package_cache(env)
+        elif self.profile.cache_stack_enabled():
+            self._apply_shared_cache(env)
         if self.profile.cache_stack_enabled():
             self._install_package_frontend_ca_in_runner()
 

cli/meta/cache/__main__.py

@@ -7,7 +7,7 @@ import sys
 from pathlib import Path
 
 from utils.cache.render import render_artifacts
-from utils.env.handlers.infinito.cache.conf import KEY, RELATIVE, primary_checkout
+from utils.env.handlers.infinito.cache.conf import KEY, RELATIVE
 
 from . import PROJECT_ROOT as REPO_ROOT
 
@@ -21,7 +21,7 @@ def _conf_path() -> Path:
     configured = os.environ.get(KEY, "").strip()
     if configured:
         return Path(configured)
-    return primary_checkout(REPO_ROOT) / RELATIVE
+    return REPO_ROOT / RELATIVE
 
 
 def main() -> int:

default.env

@@ -55,9 +55,6 @@ INFINITO_BUILD=1
 # `1` keeps BuildKit on (default); `0` forces the compose wrapper to use the legacy builder.
 INFINITO_BUILDKIT_ENABLED=1
 
-# Checkout whose nginx upstream map the shared cache frontend mounts: primary | worktree.
-INFINITO_CACHE_CONF_SOURCE=primary
-
 # External docker network carrying the shared cache stack; empty on the primary instance, which owns the stack itself.
 INFINITO_CACHE_NETWORK=
 

docs/contributing/environment/cache.md

@@ -103,6 +103,7 @@ Per-variable defaults and purposes are in [compose.yml.md](../artefact/files/com
 |---|---|
 | Start the stack with caches | `make compose-up` |
 | Stop the stack | `make compose-down` |
+| Install this checkout's `cache:` declarations into the running stack | `make cache-apply` |
 | Wipe local cache state | `make clean-cache` |
 | Manually re-bootstrap Nexus repos | `bash roles/svc-cache-package/files/shell/bootstrap.sh` (after `make dotenv` or sourcing `scripts/meta/env/load.sh`) |
 | Manually regenerate frontend certs | `bash roles/svc-cache-package/files/shell/certs.sh` |
@@ -116,7 +117,11 @@ Cache state persists under `/var/cache/infinito/core/cache/`. Paths are configur
 
 When a new package manager or upstream needs caching:
 
-Declare it under `cache:` in a `meta/networks.yml`, then run `make dotenv`. That file is the single point of truth; the Nexus repo, the leaf cert, the nginx server block, the `extra_hosts` entries, `pip.conf`, `.npmrc` and the apt sources are all derived from it.
+Declare it under `cache:` in a `meta/networks.yml`, then run `make dotenv` followed by `make cache-apply`. That file is the single point of truth; the Nexus repo, the leaf cert, the nginx server block, the `extra_hosts` entries, `pip.conf`, `.npmrc` and the apt sources are all derived from it. `make dotenv` renders them, `make cache-apply` installs them into the running stack: it creates the Nexus proxy repositories, issues the frontend's leaf certificates, copies the upstream map onto the path the frontend mounts and reloads it. Recreate the checkout's runner with `make compose-up` afterwards so the new DNS hijacks reach it.
+
+Every checkout renders and applies its own map, so a branch that declares a new host serves it without waiting for a merge. The frontend mounts one map and `make cache-apply` replaces it, so whichever checkout applied last is the one it serves. Two checkouts sharing a stack therefore overwrite each other: run `make cache-apply` from the one whose declarations you need.
+
+A TLS upstream only reaches an inner image build when that build trusts the frontend's CA: the compose wrapper hands a build the plain-HTTP hosts by default and the full host list only when its Dockerfile runs `package-frontend-ca.sh` (see [svc-ai-ltengine](../../../roles/svc-ai-ltengine/files/Dockerfile) for the three lines and the two staging tasks that go with them).
 
 Put the declaration in the role that needs the upstream. An upstream no single role owns goes in [svc-cache-package](../../../roles/svc-cache-package/meta/networks.yml).
 

scripts/system/cache/apply.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# shellcheck shell=bash
+# nocheck: raw-docker  this runs on the developer's host against the dev cache stack, where the `container` wrapper is not on PATH
+#
+# Install this checkout's cache declarations into the running cache stack:
+# the Nexus proxy repositories, the frontend's leaf certificates, and the
+# nginx upstream map the frontend serves. A checkout that declares a new
+# cache host needs all three before a client reaches it.
+#
+# The frontend belongs to whichever checkout owns the cache stack, so the
+# map is copied onto the path that container actually mounts rather than
+# only rendered here. The map is generated, never tracked, so the owning
+# checkout restores its own set by running this script or `make dotenv`.
+#
+# Use `make cache-apply` rather than calling this script directly.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
+cd "${REPO_ROOT}"
+
+# shellcheck source=scripts/meta/env/load.sh
+source "scripts/meta/env/load.sh"
+
+: "${INFINITO_CACHE_PACKAGE_FRONTEND_CONF:?Run 'make dotenv' first}"
+
+FRONTEND="infinito-package-cache-frontend"
+MOUNT_TARGET="/etc/nginx/conf.d/upstreams.conf"
+
+if ! docker ps --format '{{.Names}}' | grep -qx "${FRONTEND}"; then
+	echo "[cache-apply] ${FRONTEND} is not running; start the cache stack first" >&2
+	exit 1
+fi
+
+echo "[cache-apply] rendering this checkout's cache artefacts"
+python3 -m cli.meta.cache
+
+echo "[cache-apply] creating the Nexus proxy repositories"
+bash roles/svc-cache-package/files/shell/bootstrap.sh
+
+echo "[cache-apply] issuing the frontend leaf certificates"
+bash roles/svc-cache-package/files/shell/certs.sh
+
+mounted="$(docker inspect -f \
+	"{{range .Mounts}}{{if eq .Destination \"${MOUNT_TARGET}\"}}{{.Source}}{{end}}{{end}}" \
+	"${FRONTEND}")"
+
+if [[ -z "${mounted}" ]]; then
+	echo "[cache-apply] ${FRONTEND} mounts no map at ${MOUNT_TARGET}" >&2
+	exit 1
+fi
+
+if [[ "${mounted}" != "${INFINITO_CACHE_PACKAGE_FRONTEND_CONF}" ]]; then
+	echo "[cache-apply] installing this checkout's map into ${mounted}"
+	cp -- "${INFINITO_CACHE_PACKAGE_FRONTEND_CONF}" "${mounted}"
+fi
+
+echo "[cache-apply] reloading ${FRONTEND}"
+docker exec "${FRONTEND}" nginx -t
+docker exec "${FRONTEND}" nginx -s reload
+
+echo "[cache-apply] done. Recreate this checkout's runner to pick up new DNS hijacks:"
+echo "[cache-apply]   make compose-up"

tests/unit/python/cli/administration/deploy/development/test_compose.py

@@ -295,6 +295,41 @@ class TestComposeUpRetries(unittest.TestCase):
         compose._generate_package_frontend_certs.assert_called_once()
         compose._install_package_frontend_ca_in_runner.assert_called_once()
 
+    @patch.dict(
+        os.environ,
+        {
+            "INFINITO_BUILD": "1",
+            "INFINITO_IMAGE": "infinito-debian",
+            "INFINITO_PULL_POLICY": "never",
+            "CI": "",
+            "GITHUB_ACTIONS": "",
+            "INFINITO_RUNNING_ON_GITHUB": "",
+            "INFINITO_GIT_COMMON_DIR": "/primary/.git",
+            "INFINITO_CACHE_NETWORK": "infinito-nexus-core_default",
+            "INFINITO_CACHE_STACK": "",
+        },
+        clear=False,
+    )
+    def test_up_applies_the_declarations_to_a_cache_stack_it_shares(self) -> None:
+        compose = self._compose()
+        compose._render_coredns_corefile = MagicMock()
+        compose._compose_up_with_retries = MagicMock()
+        compose.wait_for_healthy = MagicMock()
+        compose._bootstrap_package_cache = MagicMock()
+        compose._generate_package_frontend_certs = MagicMock()
+        compose._apply_shared_cache = MagicMock()
+        compose._install_package_frontend_ca_in_runner = MagicMock()
+
+        compose.up(run_entry_init=False)
+
+        compose._compose_up_with_retries.assert_called_once_with(
+            ["up", "-d", "coredns", "infinito"], attempts=6, delay_s=30
+        )
+        compose._generate_package_frontend_certs.assert_not_called()
+        compose._bootstrap_package_cache.assert_not_called()
+        compose._apply_shared_cache.assert_called_once()
+        compose._install_package_frontend_ca_in_runner.assert_called_once()
+
 
 if __name__ == "__main__":  # pragma: no cover
     unittest.main()

tests/unit/python/utils/env/handlers/infinito/test_cache_conf.py

@@ -2,22 +2,17 @@
 
 from __future__ import annotations
 
-import os
-import subprocess
 import unittest
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from unittest.mock import patch
 
 from utils.env.builder import BuildContext, EnvBuilder
 from utils.env.handlers.infinito.cache import conf as handler
 
-_BLANK = {handler.SOURCE_KEY: "", handler.KEY: ""}
 
-
-def _ctx(root: Path, static: dict[str, str] | None = None) -> BuildContext:
+def _ctx(root: Path) -> BuildContext:
     return BuildContext(
-        static=static or {},
+        static={},
         static_comments={handler.KEY: handler.COMMENT},
         repo_root=root,
         on_gha=False,
@@ -25,79 +20,18 @@ def _ctx(root: Path, static: dict[str, str] | None = None) -> BuildContext:
     )
 
 
-def _apply(ctx: BuildContext) -> str:
-    eb = EnvBuilder()
-    handler.apply(eb, ctx)
-    return eb.values[handler.KEY]
-
-
-class TestPrimaryCheckout(unittest.TestCase):
-    def setUp(self) -> None:
-        self._tmp = TemporaryDirectory()
-        self.addCleanup(self._tmp.cleanup)
-        self.root = Path(self._tmp.name).resolve()
-        self.primary = self.root / "primary"
-        self.primary.mkdir()
-        self._git("init", "-q", "-b", "main", cwd=self.primary)
-        self._git("config", "user.email", "t@t", cwd=self.primary)
-        self._git("config", "user.name", "t", cwd=self.primary)
-        (self.primary / "seed").write_text("seed\n")
-        self._git("add", "seed", cwd=self.primary)
-        self._git("commit", "-qm", "seed", cwd=self.primary)
-
-    def _git(self, *args: str, cwd: Path) -> None:
-        subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True)
-
-    def test_the_primary_checkout_resolves_to_itself(self) -> None:
-        self.assertEqual(handler.primary_checkout(self.primary), self.primary)
-
-    def test_a_worktree_resolves_to_the_primary_checkout(self) -> None:
-        linked = self.root / "linked"
-        self._git("worktree", "add", "-q", str(linked), "-b", "side", cwd=self.primary)
-
-        self.assertEqual(handler.primary_checkout(linked), self.primary)
-
-    def test_a_directory_outside_git_falls_back_to_itself(self) -> None:
-        outside = self.root / "outside"
-        outside.mkdir()
-
-        self.assertEqual(handler.primary_checkout(outside), outside)
-
-    def test_a_root_git_cannot_be_run_in_falls_back_to_itself(self) -> None:
-        missing = self.root / "gone"
-
-        self.assertEqual(handler.primary_checkout(missing), missing)
-
-
 class TestApply(unittest.TestCase):
     def setUp(self) -> None:
         self._tmp = TemporaryDirectory()
         self.addCleanup(self._tmp.cleanup)
         self.root = Path(self._tmp.name).resolve()
 
-    @patch.dict(os.environ, _BLANK, clear=False)
-    def test_the_default_follows_the_primary_checkout(self) -> None:
-        with patch.object(handler, "primary_checkout", return_value=Path("/primary")):
-            self.assertEqual(
-                _apply(_ctx(self.root)), str(Path("/primary") / handler.RELATIVE)
-            )
+    def test_the_map_is_rendered_in_the_checkout_that_builds_it(self) -> None:
+        eb = EnvBuilder()
 
-    @patch.dict(os.environ, {**_BLANK, handler.SOURCE_KEY: "worktree"}, clear=False)
-    def test_the_environment_can_pin_the_worktree(self) -> None:
-        self.assertEqual(_apply(_ctx(self.root)), str(self.root / handler.RELATIVE))
+        handler.apply(eb, _ctx(self.root))
 
-    @patch.dict(os.environ, _BLANK, clear=False)
-    def test_a_static_entry_can_pin_the_worktree(self) -> None:
-        ctx = _ctx(self.root, {handler.SOURCE_KEY: "Worktree"})
-
-        self.assertEqual(_apply(ctx), str(self.root / handler.RELATIVE))
-
-    @patch.dict(os.environ, {**_BLANK, handler.SOURCE_KEY: "primary"}, clear=False)
-    def test_an_unknown_source_keeps_the_primary_checkout(self) -> None:
-        with patch.object(handler, "primary_checkout", return_value=Path("/primary")):
-            self.assertEqual(
-                _apply(_ctx(self.root)), str(Path("/primary") / handler.RELATIVE)
-            )
+        self.assertEqual(eb.values[handler.KEY], str(self.root / handler.RELATIVE))
 
 
 if __name__ == "__main__":

utils/env/handlers/infinito/cache/conf.py

@@ -1,6 +1,9 @@
 """INFINITO_CACHE_PACKAGE_FRONTEND_CONF: nginx upstream map the shared
-package-cache frontend mounts, taken from the primary checkout unless
-INFINITO_CACHE_CONF_SOURCE says worktree.
+package-cache frontend mounts.
+
+Every checkout renders its own, so a branch that declares a new cache host
+serves it instead of reading a map that cannot know about it. `make
+cache-apply` installs that map into the running frontend.
 
 The map itself is generated, never tracked, so no second copy of a hostname
 can drift; ``utils.cache.render.render_artifacts`` writes it once the
@@ -9,8 +12,6 @@ value-set is built.
 
 from __future__ import annotations
 
-import os
-import subprocess
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -18,41 +19,9 @@ if TYPE_CHECKING:
     from utils.env.builder import BuildContext, EnvBuilder
 
 KEY = "INFINITO_CACHE_PACKAGE_FRONTEND_CONF"
-SOURCE_KEY = "INFINITO_CACHE_CONF_SOURCE"
-COMMENT = (
-    "Upstream map the cache frontend mounts; follows the primary checkout, "
-    f"{SOURCE_KEY}=worktree points it here."
-)
+COMMENT = "Upstream map the cache frontend mounts, rendered by this checkout."
 RELATIVE = Path("compose") / "package-cache-frontend" / "upstreams.conf"
 
 
-def primary_checkout(repo_root: Path) -> Path:
-    """Return the checkout that owns the repository, given any worktree.
-
-    Args:
-        repo_root: the checkout this build runs in.
-
-    Returns:
-        The primary checkout's path, or ``repo_root`` when git cannot say.
-    """
-    try:
-        common = subprocess.run(
-            ["git", "rev-parse", "--path-format=absolute", "--git-common-dir"],
-            cwd=repo_root,
-            capture_output=True,
-            text=True,
-            check=False,
-        ).stdout.strip()
-    except OSError:
-        return repo_root
-    if not common:
-        return repo_root
-    return Path(common).parent
-
-
 def apply(eb: EnvBuilder, ctx: BuildContext) -> None:
-    source = (
-        (os.environ.get(SOURCE_KEY) or ctx.static.get(SOURCE_KEY) or "").strip().lower()
-    )
-    root = ctx.repo_root if source == "worktree" else primary_checkout(ctx.repo_root)
-    eb.setdefault(KEY, str(root / RELATIVE), comment=COMMENT)
+    eb.setdefault(KEY, str(ctx.repo_root / RELATIVE), comment=COMMENT)

utils/env/handlers/passthrough.py

@@ -117,7 +117,6 @@ STATIC_KEYS = (
     "INFINITO_TOR",
     "INFINITO_INSTANCE",
     "INFINITO_GIT_COMMON_DIR",
-    "INFINITO_CACHE_CONF_SOURCE",
     "INFINITO_CACHE_NETWORK",
     "INFINITO_CACHE_STACK",
     "INFINITO_CACHE_REGISTRY_HOST_PATH",