diff --git a/Makefile b/Makefile index b24dfe23dd..096f2e3a7a 100644 --- a/Makefile +++ b/Makefile @@ -100,6 +100,13 @@ build-no-cache-all: INFINITO_DISTRO="$$d" "$(MAKE)" build-no-cache; \ done +.PHONY: cache-apply +# Install this checkout's cache declarations into the running cache stack. +# Note: creates the Nexus proxy repos, issues the frontend certs, installs the upstream map the frontend mounts and reloads it. +# Note: `make compose-up` runs it for a checkout that shares another one's cache stack; call it directly after changing a `cache:` declaration. +cache-apply: + @bash scripts/system/cache/apply.sh + .PHONY: cheat # Print the operator prompt cheatsheet from docs/contributing/tools/agents/cheatsheet.md. cheat: @@ -293,10 +300,8 @@ dotenv: # Note: avoids stale BASH_ENV INFINITO_* values pinning via setdefault. dotenv-force: @rm -f .env - @env -i HOME="$${HOME}" PATH="$${PATH}" \ - INFINITO_CACHE_CONF_SOURCE="$${INFINITO_CACHE_CONF_SOURCE:-}" \ - python3 -m cli.meta.env - @python3 -m cli.meta.cache + @env -i HOME="$${HOME}" PATH="$${PATH}" python3 -m cli.meta.env + @set -a; . ./.env; set +a; python3 -m cli.meta.cache .PHONY: environment-bootstrap # Bootstrap the local development environment. @@ -931,7 +936,7 @@ worktree-prune: .PHONY: worktree-up # Check a branch out into an isolated worktree with its own subnet, ports and container names. # Usage: make worktree-up branch= [base=] -# Note: the worktree shares the primary checkout's cache stack instead of starting its own. +# Note: the worktree shares the primary checkout's cache stack instead of starting its own; `make cache-apply` serves its own cache declarations from it. # Param branch: branch to check out (required). # Param base: parent directory for the worktree (default ~/.local/share/worktrees///). worktree-up: diff --git a/aliases b/aliases index 919e193f31..4b351f27bd 100644 --- a/aliases +++ b/aliases @@ -9,6 +9,7 @@ alias i8bumi='m build-missing' alias i8buno='m build-no-cache' alias i8bunoca='m build-no-cache-all' alias i8ca='m act-debug' +alias i8caap='m cache-apply' alias i8caru='m act-runner-image' alias i8cawo='m act-workflow' alias i8ce='m cheat' diff --git a/cli/administration/deploy/development/compose.py b/cli/administration/deploy/development/compose.py index 74dcf3b980..cb54c37e59 100644 --- a/cli/administration/deploy/development/compose.py +++ b/cli/administration/deploy/development/compose.py @@ -141,6 +141,31 @@ class Compose: text=True, ) + def _apply_shared_cache(self, env: dict[str, str]) -> None: + """Serve this checkout's cache declarations from the shared stack. + + The owner creates the proxy repositories and the certificates while + it brings its own containers up. An instance that only joins the + network starts none of them, so without this its `cache:` entries + exist on disk and nowhere else: the hijacked hostname resolves to a + frontend that has no server block for it. + """ + helper = self.repo_root / "scripts" / "system" / "cache" / "apply.sh" + print(">>> Applying this checkout's cache declarations to the shared stack") + r = subprocess.run( + ["bash", str(helper)], + cwd=self.repo_root, + env=env, + check=False, + text=True, + ) + if r.returncode != 0: + print( + f">>> WARNING: cache apply exited rc={r.returncode}; the shared " + "frontend keeps serving the map it already had. Start the owning " + f"stack and re-run `make cache-apply` ({helper})." + ) + def _install_package_frontend_ca_in_runner(self) -> None: """Install the frontend CA in the runner trust store. Idempotent.""" print(">>> Installing package-cache-frontend CA into runner trust store") @@ -213,6 +238,8 @@ class Compose: if self.profile.owns_cache_stack(): self._bootstrap_package_cache(env) + elif self.profile.cache_stack_enabled(): + self._apply_shared_cache(env) if self.profile.cache_stack_enabled(): self._install_package_frontend_ca_in_runner() diff --git a/cli/meta/cache/__main__.py b/cli/meta/cache/__main__.py index f43bff7a3f..d3fb764aa3 100644 --- a/cli/meta/cache/__main__.py +++ b/cli/meta/cache/__main__.py @@ -7,7 +7,7 @@ import sys from pathlib import Path from utils.cache.render import render_artifacts -from utils.env.handlers.infinito.cache.conf import KEY, RELATIVE, primary_checkout +from utils.env.handlers.infinito.cache.conf import KEY, RELATIVE from . import PROJECT_ROOT as REPO_ROOT @@ -21,7 +21,7 @@ def _conf_path() -> Path: configured = os.environ.get(KEY, "").strip() if configured: return Path(configured) - return primary_checkout(REPO_ROOT) / RELATIVE + return REPO_ROOT / RELATIVE def main() -> int: diff --git a/default.env b/default.env index a097974666..478fddf66c 100644 --- a/default.env +++ b/default.env @@ -55,9 +55,6 @@ INFINITO_BUILD=1 # `1` keeps BuildKit on (default); `0` forces the compose wrapper to use the legacy builder. INFINITO_BUILDKIT_ENABLED=1 -# Checkout whose nginx upstream map the shared cache frontend mounts: primary | worktree. -INFINITO_CACHE_CONF_SOURCE=primary - # External docker network carrying the shared cache stack; empty on the primary instance, which owns the stack itself. INFINITO_CACHE_NETWORK= diff --git a/docs/contributing/environment/cache.md b/docs/contributing/environment/cache.md index 706be93673..a37e905e8d 100644 --- a/docs/contributing/environment/cache.md +++ b/docs/contributing/environment/cache.md @@ -103,6 +103,7 @@ Per-variable defaults and purposes are in [compose.yml.md](../artefact/files/com |---|---| | Start the stack with caches | `make compose-up` | | Stop the stack | `make compose-down` | +| Install this checkout's `cache:` declarations into the running stack | `make cache-apply` | | Wipe local cache state | `make clean-cache` | | Manually re-bootstrap Nexus repos | `bash roles/svc-cache-package/files/shell/bootstrap.sh` (after `make dotenv` or sourcing `scripts/meta/env/load.sh`) | | Manually regenerate frontend certs | `bash roles/svc-cache-package/files/shell/certs.sh` | @@ -116,7 +117,11 @@ Cache state persists under `/var/cache/infinito/core/cache/`. Paths are configur When a new package manager or upstream needs caching: -Declare it under `cache:` in a `meta/networks.yml`, then run `make dotenv`. That file is the single point of truth; the Nexus repo, the leaf cert, the nginx server block, the `extra_hosts` entries, `pip.conf`, `.npmrc` and the apt sources are all derived from it. +Declare it under `cache:` in a `meta/networks.yml`, then run `make dotenv` followed by `make cache-apply`. That file is the single point of truth; the Nexus repo, the leaf cert, the nginx server block, the `extra_hosts` entries, `pip.conf`, `.npmrc` and the apt sources are all derived from it. `make dotenv` renders them, `make cache-apply` installs them into the running stack: it creates the Nexus proxy repositories, issues the frontend's leaf certificates, copies the upstream map onto the path the frontend mounts and reloads it. Recreate the checkout's runner with `make compose-up` afterwards so the new DNS hijacks reach it. + +Every checkout renders and applies its own map, so a branch that declares a new host serves it without waiting for a merge. The frontend mounts one map and `make cache-apply` replaces it, so whichever checkout applied last is the one it serves. Two checkouts sharing a stack therefore overwrite each other: run `make cache-apply` from the one whose declarations you need. + +A TLS upstream only reaches an inner image build when that build trusts the frontend's CA: the compose wrapper hands a build the plain-HTTP hosts by default and the full host list only when its Dockerfile runs `package-frontend-ca.sh` (see [svc-ai-ltengine](../../../roles/svc-ai-ltengine/files/Dockerfile) for the three lines and the two staging tasks that go with them). Put the declaration in the role that needs the upstream. An upstream no single role owns goes in [svc-cache-package](../../../roles/svc-cache-package/meta/networks.yml). diff --git a/scripts/system/cache/apply.sh b/scripts/system/cache/apply.sh new file mode 100755 index 0000000000..d9a79e4864 --- /dev/null +++ b/scripts/system/cache/apply.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# nocheck: raw-docker this runs on the developer's host against the dev cache stack, where the `container` wrapper is not on PATH +# +# Install this checkout's cache declarations into the running cache stack: +# the Nexus proxy repositories, the frontend's leaf certificates, and the +# nginx upstream map the frontend serves. A checkout that declares a new +# cache host needs all three before a client reaches it. +# +# The frontend belongs to whichever checkout owns the cache stack, so the +# map is copied onto the path that container actually mounts rather than +# only rendered here. The map is generated, never tracked, so the owning +# checkout restores its own set by running this script or `make dotenv`. +# +# Use `make cache-apply` rather than calling this script directly. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +cd "${REPO_ROOT}" + +# shellcheck source=scripts/meta/env/load.sh +source "scripts/meta/env/load.sh" + +: "${INFINITO_CACHE_PACKAGE_FRONTEND_CONF:?Run 'make dotenv' first}" + +FRONTEND="infinito-package-cache-frontend" +MOUNT_TARGET="/etc/nginx/conf.d/upstreams.conf" + +if ! docker ps --format '{{.Names}}' | grep -qx "${FRONTEND}"; then + echo "[cache-apply] ${FRONTEND} is not running; start the cache stack first" >&2 + exit 1 +fi + +echo "[cache-apply] rendering this checkout's cache artefacts" +python3 -m cli.meta.cache + +echo "[cache-apply] creating the Nexus proxy repositories" +bash roles/svc-cache-package/files/shell/bootstrap.sh + +echo "[cache-apply] issuing the frontend leaf certificates" +bash roles/svc-cache-package/files/shell/certs.sh + +mounted="$(docker inspect -f \ + "{{range .Mounts}}{{if eq .Destination \"${MOUNT_TARGET}\"}}{{.Source}}{{end}}{{end}}" \ + "${FRONTEND}")" + +if [[ -z "${mounted}" ]]; then + echo "[cache-apply] ${FRONTEND} mounts no map at ${MOUNT_TARGET}" >&2 + exit 1 +fi + +if [[ "${mounted}" != "${INFINITO_CACHE_PACKAGE_FRONTEND_CONF}" ]]; then + echo "[cache-apply] installing this checkout's map into ${mounted}" + cp -- "${INFINITO_CACHE_PACKAGE_FRONTEND_CONF}" "${mounted}" +fi + +echo "[cache-apply] reloading ${FRONTEND}" +docker exec "${FRONTEND}" nginx -t +docker exec "${FRONTEND}" nginx -s reload + +echo "[cache-apply] done. Recreate this checkout's runner to pick up new DNS hijacks:" +echo "[cache-apply] make compose-up" diff --git a/tests/unit/python/cli/administration/deploy/development/test_compose.py b/tests/unit/python/cli/administration/deploy/development/test_compose.py index 6f56e7f082..f18aa20b2b 100644 --- a/tests/unit/python/cli/administration/deploy/development/test_compose.py +++ b/tests/unit/python/cli/administration/deploy/development/test_compose.py @@ -295,6 +295,41 @@ class TestComposeUpRetries(unittest.TestCase): compose._generate_package_frontend_certs.assert_called_once() compose._install_package_frontend_ca_in_runner.assert_called_once() + @patch.dict( + os.environ, + { + "INFINITO_BUILD": "1", + "INFINITO_IMAGE": "infinito-debian", + "INFINITO_PULL_POLICY": "never", + "CI": "", + "GITHUB_ACTIONS": "", + "INFINITO_RUNNING_ON_GITHUB": "", + "INFINITO_GIT_COMMON_DIR": "/primary/.git", + "INFINITO_CACHE_NETWORK": "infinito-nexus-core_default", + "INFINITO_CACHE_STACK": "", + }, + clear=False, + ) + def test_up_applies_the_declarations_to_a_cache_stack_it_shares(self) -> None: + compose = self._compose() + compose._render_coredns_corefile = MagicMock() + compose._compose_up_with_retries = MagicMock() + compose.wait_for_healthy = MagicMock() + compose._bootstrap_package_cache = MagicMock() + compose._generate_package_frontend_certs = MagicMock() + compose._apply_shared_cache = MagicMock() + compose._install_package_frontend_ca_in_runner = MagicMock() + + compose.up(run_entry_init=False) + + compose._compose_up_with_retries.assert_called_once_with( + ["up", "-d", "coredns", "infinito"], attempts=6, delay_s=30 + ) + compose._generate_package_frontend_certs.assert_not_called() + compose._bootstrap_package_cache.assert_not_called() + compose._apply_shared_cache.assert_called_once() + compose._install_package_frontend_ca_in_runner.assert_called_once() + if __name__ == "__main__": # pragma: no cover unittest.main() diff --git a/tests/unit/python/utils/env/handlers/infinito/test_cache_conf.py b/tests/unit/python/utils/env/handlers/infinito/test_cache_conf.py index e0a35a4781..f6ff8ec114 100644 --- a/tests/unit/python/utils/env/handlers/infinito/test_cache_conf.py +++ b/tests/unit/python/utils/env/handlers/infinito/test_cache_conf.py @@ -2,22 +2,17 @@ from __future__ import annotations -import os -import subprocess import unittest from pathlib import Path from tempfile import TemporaryDirectory -from unittest.mock import patch from utils.env.builder import BuildContext, EnvBuilder from utils.env.handlers.infinito.cache import conf as handler -_BLANK = {handler.SOURCE_KEY: "", handler.KEY: ""} - -def _ctx(root: Path, static: dict[str, str] | None = None) -> BuildContext: +def _ctx(root: Path) -> BuildContext: return BuildContext( - static=static or {}, + static={}, static_comments={handler.KEY: handler.COMMENT}, repo_root=root, on_gha=False, @@ -25,79 +20,18 @@ def _ctx(root: Path, static: dict[str, str] | None = None) -> BuildContext: ) -def _apply(ctx: BuildContext) -> str: - eb = EnvBuilder() - handler.apply(eb, ctx) - return eb.values[handler.KEY] - - -class TestPrimaryCheckout(unittest.TestCase): - def setUp(self) -> None: - self._tmp = TemporaryDirectory() - self.addCleanup(self._tmp.cleanup) - self.root = Path(self._tmp.name).resolve() - self.primary = self.root / "primary" - self.primary.mkdir() - self._git("init", "-q", "-b", "main", cwd=self.primary) - self._git("config", "user.email", "t@t", cwd=self.primary) - self._git("config", "user.name", "t", cwd=self.primary) - (self.primary / "seed").write_text("seed\n") - self._git("add", "seed", cwd=self.primary) - self._git("commit", "-qm", "seed", cwd=self.primary) - - def _git(self, *args: str, cwd: Path) -> None: - subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True) - - def test_the_primary_checkout_resolves_to_itself(self) -> None: - self.assertEqual(handler.primary_checkout(self.primary), self.primary) - - def test_a_worktree_resolves_to_the_primary_checkout(self) -> None: - linked = self.root / "linked" - self._git("worktree", "add", "-q", str(linked), "-b", "side", cwd=self.primary) - - self.assertEqual(handler.primary_checkout(linked), self.primary) - - def test_a_directory_outside_git_falls_back_to_itself(self) -> None: - outside = self.root / "outside" - outside.mkdir() - - self.assertEqual(handler.primary_checkout(outside), outside) - - def test_a_root_git_cannot_be_run_in_falls_back_to_itself(self) -> None: - missing = self.root / "gone" - - self.assertEqual(handler.primary_checkout(missing), missing) - - class TestApply(unittest.TestCase): def setUp(self) -> None: self._tmp = TemporaryDirectory() self.addCleanup(self._tmp.cleanup) self.root = Path(self._tmp.name).resolve() - @patch.dict(os.environ, _BLANK, clear=False) - def test_the_default_follows_the_primary_checkout(self) -> None: - with patch.object(handler, "primary_checkout", return_value=Path("/primary")): - self.assertEqual( - _apply(_ctx(self.root)), str(Path("/primary") / handler.RELATIVE) - ) + def test_the_map_is_rendered_in_the_checkout_that_builds_it(self) -> None: + eb = EnvBuilder() - @patch.dict(os.environ, {**_BLANK, handler.SOURCE_KEY: "worktree"}, clear=False) - def test_the_environment_can_pin_the_worktree(self) -> None: - self.assertEqual(_apply(_ctx(self.root)), str(self.root / handler.RELATIVE)) + handler.apply(eb, _ctx(self.root)) - @patch.dict(os.environ, _BLANK, clear=False) - def test_a_static_entry_can_pin_the_worktree(self) -> None: - ctx = _ctx(self.root, {handler.SOURCE_KEY: "Worktree"}) - - self.assertEqual(_apply(ctx), str(self.root / handler.RELATIVE)) - - @patch.dict(os.environ, {**_BLANK, handler.SOURCE_KEY: "primary"}, clear=False) - def test_an_unknown_source_keeps_the_primary_checkout(self) -> None: - with patch.object(handler, "primary_checkout", return_value=Path("/primary")): - self.assertEqual( - _apply(_ctx(self.root)), str(Path("/primary") / handler.RELATIVE) - ) + self.assertEqual(eb.values[handler.KEY], str(self.root / handler.RELATIVE)) if __name__ == "__main__": diff --git a/utils/env/handlers/infinito/cache/conf.py b/utils/env/handlers/infinito/cache/conf.py index c3643803f6..742b9626a6 100644 --- a/utils/env/handlers/infinito/cache/conf.py +++ b/utils/env/handlers/infinito/cache/conf.py @@ -1,6 +1,9 @@ """INFINITO_CACHE_PACKAGE_FRONTEND_CONF: nginx upstream map the shared -package-cache frontend mounts, taken from the primary checkout unless -INFINITO_CACHE_CONF_SOURCE says worktree. +package-cache frontend mounts. + +Every checkout renders its own, so a branch that declares a new cache host +serves it instead of reading a map that cannot know about it. `make +cache-apply` installs that map into the running frontend. The map itself is generated, never tracked, so no second copy of a hostname can drift; ``utils.cache.render.render_artifacts`` writes it once the @@ -9,8 +12,6 @@ value-set is built. from __future__ import annotations -import os -import subprocess from pathlib import Path from typing import TYPE_CHECKING @@ -18,41 +19,9 @@ if TYPE_CHECKING: from utils.env.builder import BuildContext, EnvBuilder KEY = "INFINITO_CACHE_PACKAGE_FRONTEND_CONF" -SOURCE_KEY = "INFINITO_CACHE_CONF_SOURCE" -COMMENT = ( - "Upstream map the cache frontend mounts; follows the primary checkout, " - f"{SOURCE_KEY}=worktree points it here." -) +COMMENT = "Upstream map the cache frontend mounts, rendered by this checkout." RELATIVE = Path("compose") / "package-cache-frontend" / "upstreams.conf" -def primary_checkout(repo_root: Path) -> Path: - """Return the checkout that owns the repository, given any worktree. - - Args: - repo_root: the checkout this build runs in. - - Returns: - The primary checkout's path, or ``repo_root`` when git cannot say. - """ - try: - common = subprocess.run( - ["git", "rev-parse", "--path-format=absolute", "--git-common-dir"], - cwd=repo_root, - capture_output=True, - text=True, - check=False, - ).stdout.strip() - except OSError: - return repo_root - if not common: - return repo_root - return Path(common).parent - - def apply(eb: EnvBuilder, ctx: BuildContext) -> None: - source = ( - (os.environ.get(SOURCE_KEY) or ctx.static.get(SOURCE_KEY) or "").strip().lower() - ) - root = ctx.repo_root if source == "worktree" else primary_checkout(ctx.repo_root) - eb.setdefault(KEY, str(root / RELATIVE), comment=COMMENT) + eb.setdefault(KEY, str(ctx.repo_root / RELATIVE), comment=COMMENT) diff --git a/utils/env/handlers/passthrough.py b/utils/env/handlers/passthrough.py index aff5bffe2f..81096e1e7c 100644 --- a/utils/env/handlers/passthrough.py +++ b/utils/env/handlers/passthrough.py @@ -117,7 +117,6 @@ STATIC_KEYS = ( "INFINITO_TOR", "INFINITO_INSTANCE", "INFINITO_GIT_COMMON_DIR", - "INFINITO_CACHE_CONF_SOURCE", "INFINITO_CACHE_NETWORK", "INFINITO_CACHE_STACK", "INFINITO_CACHE_REGISTRY_HOST_PATH",