diff --git a/Makefile b/Makefile index 08f410e2bb..f17656e3ab 100644 --- a/Makefile +++ b/Makefile @@ -872,8 +872,9 @@ SWARM_DISTROS = $(or $(distros),$${INFINITO_DISTRO:?}) SWARM_ARCH = $(or $(arch),$(shell bash scripts/meta/resolve/architecture.sh)) .PHONY: swarm-zombie -# Run a swarm matrix-app test and leave the cluster alive afterwards for post-mortem inspection. +# Run a swarm matrix-app test, copy the act job's rescue bundle to the host, then release the cluster unless keep=true. # Param app: matrix application id (e.g. web-app-baserow). +# Param keep: true keeps the cluster and its act outer container up for post-mortem inspection; otherwise both are released on success and on failure. # Param distros: optional single distro the cluster runs on (default: INFINITO_DISTRO from .env). # Param variant: optional matrix variant index to deploy (default 0); a multi-variant app runs one cluster per swarm-zombie, so pick the round to validate. # Param disable: optional comma-separated provider keys removed from the test inventory (e.g. matomo,dashboard,prometheus,email,css). @@ -883,16 +884,20 @@ SWARM_ARCH = $(or $(arch),$(shell bash scripts/meta/resolve/architecture.sh)) # Param arch: architecture the cluster deploys on, amd64 | arm64 (default: the host's own). # Param network: optional node network mode pinned on the row (clearnet | tor | multi); empty keeps the run's rotation. # Param step_timeout: optional minute budget for the matrix-deploy step (default 690). -# Note: Use `make swarm-exec` / `make swarm-shell` to inspect, `make swarm-down` to release. +# Note: With keep=true use `make swarm-exec` / `make swarm-shell` to inspect, `make swarm-down` to release. swarm-zombie: install-act - @test -n '$(app)' || { echo 'usage: make swarm-zombie app= [distros=] [variant=] [name=] [disable=] [network=]'; exit 2; } + @test -n '$(app)' || { echo 'usage: make swarm-zombie app= [distros=] [variant=] [name=] [disable=] [network=] [keep=true]'; exit 2; } @"$${PYTHON}" -m cli.meta.ci.validate --modes swarm --whitelist '$(app)#$(or $(variant),0)@swarm$(if $(network),+$(network))' @SWARM_NAME='$(or $(name),$(app))' INFINITO_KEEP_SWARM_NODES=false bash scripts/tests/deploy/swarm/utils/clean/teardown.sh @bash scripts/tests/deploy/swarm/utils/clean/lab_subnet.sh @bash scripts/tests/deploy/act/down_act_outer.sh - @ACT_RM=false \ + @SWARM_NAME='$(or $(name),$(app))' \ + keep='$(keep)' \ + ACT_RM=false \ ACT_BIND=true \ - ACT_ENV="INFINITO_KEEP_SWARM_NODES=true; \ + ACT_ENV="INFINITO_KEEP_SWARM_NODES=$(if $(filter true,$(keep)),true,false); \ + INFINITO_CACHE_SHARED_HOST_PATH=$${INFINITO_CACHE_SHARED_HOST_PATH:-}; \ + INFINITO_CACHE_HOST_PACMAN_PATH=$${INFINITO_CACHE_HOST_PACMAN_PATH:-}; \ INFINITO_APP_DISCOVERY_RUNNER=host; \ INFINITO_DEPLOY_MODE=swarm; \ disable=$(disable); \ @@ -906,7 +911,7 @@ swarm-zombie: install-act ACT_JOB=deploy \ ACT_MATRIX="apps:$(app);variant:$(or $(variant),0);mode:swarm" \ ACT_INPUTS="whitelist=$(app)#$(or $(variant),0)@swarm$(if $(network),+$(network)) distros=$(SWARM_DISTROS) index=0 sweep=0 modes=swarm vpn=$(or $(vpn),disabled) meshes=$(meshes) disable=$(disable) architectures=$(SWARM_ARCH)" \ - bash scripts/tests/deploy/act/workflow.sh + bash scripts/tests/deploy/swarm/utils/release.sh bash scripts/tests/deploy/act/workflow.sh .PHONY: system-purge # Run the broad low-hardware cleanup routine. diff --git a/cli/administration/deploy/development/compose.py b/cli/administration/deploy/development/compose.py index ada608fb5d..bcfbfb8d6f 100644 --- a/cli/administration/deploy/development/compose.py +++ b/cli/administration/deploy/development/compose.py @@ -21,6 +21,9 @@ CACHE_ROLE_SHELL_DIR = CACHE_ROLE_DIR / "files" / "shell" INIT_IMAGE_KEY = "INFINITO_CACHE_PACKAGE_FRONTEND_INIT_IMAGE" +PACMAN_CACHE_SCRIPT = "/usr/local/bin/pacman-cache.sh" +PACMAN_CONF = "/etc/pacman.conf" + class Compose: """Wrapper around `docker compose` for the dev/CI stack.""" @@ -190,6 +193,19 @@ class Compose: "runner manually." ) + def _layer_pacman_cache_in_runner(self) -> None: + """Point the runner's pacman at the host cache and the shared folder.""" + print(">>> Layering the pacman caches in the runner") + self.exec( + [ + PACMAN_CACHE_SCRIPT, + PACMAN_CONF, + self.profile.shared_package_cache(), + self.profile.host_pacman_cache(), + ], + live=False, + ) + def _render_coredns_corefile(self) -> None: renderer = CoreDNSCorefileRenderer(repo_root=self.repo_root) out = renderer.render(show_preview=True, preview_lines=25) @@ -250,6 +266,8 @@ class Compose: self._apply_shared_cache(env) if self.profile.cache_stack_enabled(): self._install_package_frontend_ca_in_runner() + if self.profile.shared_package_cache(): + self._layer_pacman_cache_in_runner() if run_entry_init: print(">>> Running infinito entry.sh init") diff --git a/cli/administration/deploy/development/env.py b/cli/administration/deploy/development/env.py index 49943f9739..5414e45ad6 100644 --- a/cli/administration/deploy/development/env.py +++ b/cli/administration/deploy/development/env.py @@ -34,6 +34,10 @@ def compose_file_args() -> list[str]: out += ["-f", "compose/cache.override.yml"] if profile.shared_cache_network(): out += ["-f", "compose/cache.shared.override.yml"] + if profile.shared_package_cache(): + out += ["-f", "compose/shared-cache.override.yml"] + if profile.host_pacman_cache(): + out += ["-f", "compose/pacman.override.yml"] if profile.gpu_reservable(): out += ["-f", "compose/gpu.override.yml"] if (os.environ.get("INFINITO_PUBLISH_PORTS") or "").strip().lower() == "false": diff --git a/cli/administration/deploy/development/profile.py b/cli/administration/deploy/development/profile.py index 3e80771dac..ab3e80f61d 100644 --- a/cli/administration/deploy/development/profile.py +++ b/cli/administration/deploy/development/profile.py @@ -69,3 +69,11 @@ class Profile: def owns_cache_stack(self) -> bool: """True iff this instance runs the cache stack rather than sharing it.""" return self.cache_stack_enabled() and not self.shared_cache_network() + + def shared_package_cache(self) -> str: + """Shared package and tool cache folder the runner mounts, empty when unset.""" + return (os.environ.get("INFINITO_CACHE_SHARED_HOST_PATH") or "").strip() + + def host_pacman_cache(self) -> str: + """Host pacman cache the runner mounts read-only, empty when there is none.""" + return (os.environ.get("INFINITO_CACHE_HOST_PACMAN_PATH") or "").strip() diff --git a/compose/pacman.override.yml b/compose/pacman.override.yml new file mode 100644 index 0000000000..041765f645 --- /dev/null +++ b/compose/pacman.override.yml @@ -0,0 +1,8 @@ +--- +services: + infinito: + volumes: + - type: bind + source: ${INFINITO_CACHE_HOST_PACMAN_PATH:?Run 'make dotenv' on an arch stack whose host has a pacman cache} + target: ${INFINITO_CACHE_HOST_PACMAN_PATH:?Run 'make dotenv' on an arch stack whose host has a pacman cache} + read_only: true diff --git a/compose/shared-cache.override.yml b/compose/shared-cache.override.yml new file mode 100644 index 0000000000..1bc4698ba5 --- /dev/null +++ b/compose/shared-cache.override.yml @@ -0,0 +1,13 @@ +--- +services: + infinito: + environment: + PIP_CACHE_DIR: ${INFINITO_CACHE_SHARED_HOST_PATH:?Run 'make dotenv' to generate the .env single source of truth}/pip + npm_config_cache: ${INFINITO_CACHE_SHARED_HOST_PATH:?Run 'make dotenv' to generate the .env single source of truth}/npm + volumes: + - type: bind + source: ${INFINITO_CACHE_SHARED_HOST_PATH:?Run 'make dotenv' to generate the .env single source of truth} + target: ${INFINITO_CACHE_SHARED_HOST_PATH:?Run 'make dotenv' to generate the .env single source of truth} + bind: + create_host_path: true + - ./scripts/docker/cache/pacman.sh:/usr/local/bin/pacman-cache.sh:ro diff --git a/compose/swarm/pacman.override.yml b/compose/swarm/pacman.override.yml new file mode 100644 index 0000000000..3bf50e3ae7 --- /dev/null +++ b/compose/swarm/pacman.override.yml @@ -0,0 +1,16 @@ +--- +x-host-pacman: &host_pacman + volumes: + - ${INFINITO_CACHE_HOST_PACMAN_PATH:?Run 'make dotenv' on an arch stack whose host has a pacman cache}:${INFINITO_CACHE_HOST_PACMAN_PATH}:ro + +services: + nfs-server: + <<: *host_pacman + swarm-mgr-01: + <<: *host_pacman + swarm-wrk-01: + <<: *host_pacman + swarm-wrk-02: + <<: *host_pacman + backup-host: + <<: *host_pacman diff --git a/compose/swarm/playbook.yml b/compose/swarm/playbook.yml index e0117d9f25..7a6846194d 100644 --- a/compose/swarm/playbook.yml +++ b/compose/swarm/playbook.yml @@ -32,6 +32,8 @@ lab_filesystem: "{{ lookup('env', 'INFINITO_DOCKER_FILESYSTEM') }}" lab_filesystem_required: "{{ lookup('env', 'INFINITO_DOCKER_FILESYSTEM_REQUIRED') | default('false', true) }}" lab_dataroot_size: "{{ '60G' if lab_is_mgr else '30G' }}" + lab_shared_cache: "{{ lookup('env', 'INFINITO_CACHE_SHARED_HOST_PATH') }}" + lab_host_pacman_cache: "{{ lookup('env', 'INFINITO_CACHE_HOST_PACMAN_PATH') }}" lab_no_proxy: "{{ (NETWORK_LOOPBACK_HOSTS + ['infinito-registry-cache', lab_mgr, lab_wrk1, lab_wrk2, lab_nfs, lab_backup, 'gcr.io', 'storage.googleapis.com', 'googleusercontent.com']) | join(',') }}" tasks: @@ -84,6 +86,23 @@ state: started timeout: 30 + - name: probe the pacman cache script (mounted by shared-cache.override.yml, absent without a shared cache folder) + stat: + path: /usr/local/bin/pacman-cache.sh + timeout: 30 + register: lab_pacman_cache_script + + - name: layer the node pacman caches before the first install, host cache read-only first, the node download dir second and the shared cache third + command: + argv: + - /usr/local/bin/pacman-cache.sh + - /etc/pacman.conf + - "{{ lab_shared_cache }}" + - "{{ lab_host_pacman_cache }}" + timeout: 60 + when: lab_pacman_cache_script.stat.exists + changed_when: true + - name: put the node docker data root on the stated filesystem (before anything is pulled) script: >- {{ lab_repo_root }}/scripts/tests/deploy/utils/filesystem/docker_dataroot.sh diff --git a/compose/swarm/shared-cache.override.yml b/compose/swarm/shared-cache.override.yml new file mode 100644 index 0000000000..6b2adb57ac --- /dev/null +++ b/compose/swarm/shared-cache.override.yml @@ -0,0 +1,17 @@ +--- +x-shared-cache: &shared_cache + volumes: + - ${INFINITO_CACHE_SHARED_HOST_PATH:?Run 'make dotenv' to generate the .env single source of truth}:${INFINITO_CACHE_SHARED_HOST_PATH} + - ../../scripts/docker/cache/pacman.sh:/usr/local/bin/pacman-cache.sh:ro + +services: + nfs-server: + <<: *shared_cache + swarm-mgr-01: + <<: *shared_cache + swarm-wrk-01: + <<: *shared_cache + swarm-wrk-02: + <<: *shared_cache + backup-host: + <<: *shared_cache diff --git a/default.env b/default.env index ca3e548fca..38147ee177 100644 --- a/default.env +++ b/default.env @@ -55,6 +55,9 @@ INFINITO_BUILD=1 # `1` keeps BuildKit on (default); `0` forces the compose wrapper to use the legacy builder. INFINITO_BUILDKIT_ENABLED=1 +# Pacman cache an arch stack mounts read-only at the same path; from pacman-conf on a host whose os-release ID is arch, else empty. +INFINITO_CACHE_HOST_PACMAN_PATH= + # External docker network carrying the shared cache stack; empty on the primary instance, which owns the stack itself. INFINITO_CACHE_NETWORK= @@ -118,6 +121,9 @@ INFINITO_CACHE_REGISTRY_IP=172.30.0.6 # Bind-mounted registry-cache proxy configuration consumed by the cache container. INFINITO_CACHE_REGISTRY_PROXY_CONF=./compose/registry-cache/proxy.conf +# Host folder of package and tool caches every slot, lab node and GUI lane mounts at the same path; derived from DIR_CACHE. +INFINITO_CACHE_SHARED_HOST_PATH= + # True where a persistent pull-through cache stack pays off (local dev). INFINITO_CACHE_STACK=true diff --git a/docs/agents/action/iteration/roundtrip.md b/docs/agents/action/iteration/roundtrip.md index f88464b74a..7db4edc962 100644 --- a/docs/agents/action/iteration/roundtrip.md +++ b/docs/agents/action/iteration/roundtrip.md @@ -14,7 +14,7 @@ For debugging the compose deploy of one role, see [Compose Loop](compose.md); fo - With no `apps=`, the loop defaults to **every application, most-complex first** (the `complexity` CLI: `infinito meta roles applications complexity --sort "desc weight" --format string`). That is a large run (one compose plus one swarm deploy per app), so narrow it with `apps=` while iterating. - The mode sequence defaults to `compose swarm` and is overridable with `modes="compose swarm"`. The order is always compose first, so the cheaper mode fails fast before the expensive swarm rebuild. Append `k8s` here once that mode exists. - Per step the output is streamed to `${TMPDIR:-/tmp}/roundtrip--.log`. Tell the operator the exact `tail -f` path for the running step. -- The compose step runs `compose-deploy mode=reinstall apps= full_cycle=true variant=0`; the swarm step runs `swarm-zombie app=`, pinned to `variant:0` and to a network mode: `clearnet` when `disable=` holds the `tor` key, else `multi` whenever the role can take it, so a multi-variant app validates one cluster (parity with the compose step, not the full per-variant CI matrix that runs one isolated runner per variant). The cluster is named `-swarm-mgr-01` etc. via the mandatory `SWARM_NAME` (the app id is the default cluster id). Each validated swarm cluster is released afterwards unless you pass `keep=true`. +- The compose step runs `compose-deploy mode=reinstall apps= full_cycle=true variant=0`; the swarm step runs `swarm-zombie app=`, pinned to `variant:0` and to a network mode: `clearnet` when `disable=` holds the `tor` key, else `multi` whenever the role can take it, so a multi-variant app validates one cluster (parity with the compose step, not the full per-variant CI matrix that runs one isolated runner per variant). The cluster is named `-swarm-mgr-01` etc. via the mandatory `SWARM_NAME` (the app id is the default cluster id). The roundtrip hands `keep` to `swarm-zombie`, which copies the rescue bundle to the host and then releases every swarm cluster, green or failed, unless you pass `keep=true`. - `make autoformat` and `make test` MUST be green before a roundtrip; the loop does not re-run them per app. - **A blocked mode is NEVER a reason to silently switch to another.** If a mode cannot run (the swarm NFS wedge below, a wedged daemon, a missing host capability), STOP the whole sweep, report the blocker together with the exact recovery command, and wait for it to be cleared. You MUST NOT fall back to a `modes=compose`-only (or any reduced) sweep to look productive: it hides the unmet swarm parity and never satisfies the compose-AND-swarm gate. Resume the full sweep only once the blocker is gone. @@ -24,7 +24,7 @@ The roundtrip is a gate, not a debugger: when a step goes red it stops and leave - Read `${TMPDIR:-/tmp}/roundtrip--.log` to see which app and mode failed, and the first error. - If the **compose** step failed, switch to the [Compose Loop](compose.md): reproduce with `make compose-deploy mode=update apps=`, inspect via `make compose-exec` / `make compose-inner-run`, apply the real fix in the repository, then re-run the roundtrip. -- If the **swarm** step failed, the cluster is left up (fail-fast skips the release). Inspect it with `make swarm-exec node=-swarm-mgr-01 cmd='...'` or `make swarm-shell name=` per the [Swarm Loop](swarm.md), confirm the fix on the live cluster, then re-run. Release it manually with `make swarm-down name=` when done. +- If the **swarm** step failed, read the rescue bundle under `${INFINITO_RESCUE_DIAGNOSTICS_BASE}//` and the Playwright artefacts under `/tmp/playwright-artifacts//` on the host; the cluster is already released. To inspect it live, reproduce with `make swarm-zombie app= keep=true` per the [Swarm Loop](swarm.md), inspect it with `make swarm-exec node=-swarm-mgr-01 cmd='...'` or `make swarm-shell name=`, confirm the fix on the live cluster, then release it with `make swarm-down name=`. A roundtrip run with `keep=true` leaves the failed cluster up the same way. - Validate every candidate fix on live state BEFORE the next roundtrip; a swarm rebuild costs tens of minutes (same rule as the focused loops). In-cluster edits are validation only, the repo change is the real fix. ## Exit diff --git a/docs/agents/action/iteration/swarm.md b/docs/agents/action/iteration/swarm.md index eb8b1fd8fd..d5301fe7ed 100644 --- a/docs/agents/action/iteration/swarm.md +++ b/docs/agents/action/iteration/swarm.md @@ -12,7 +12,8 @@ When Act aborts at "Set up job" on a recent Docker engine, see [Workflow Loop](w ## The loop -- `swarm-zombie app=` builds a full DinD cluster and leaves it up (`INFINITO_KEEP_SWARM_NODES=true`) for inspection. +- `swarm-zombie app= keep=true` builds a full DinD cluster and leaves it up (`INFINITO_KEEP_SWARM_NODES=true`) for inspection. +- Without `keep=true`, `swarm-zombie` copies the act job's rescue diagnostics to `${INFINITO_RESCUE_DIAGNOSTICS_BASE}` and its Playwright artefacts to `/tmp/playwright-artifacts` on the host, then releases the cluster and its act outer container, on success and on failure. - Inspect the live cluster with `make swarm-exec` / `make swarm-shell`, then release it with `make swarm-down`. - Apply the real fix in the repository, then rebuild with a new `swarm-zombie` run. @@ -21,7 +22,7 @@ When Act aborts at "Set up job" on a recent Docker engine, see [Workflow Loop](w An `swarm-zombie` run rebuilds a full DinD cluster over tens of minutes, so you MUST confirm a fix on the live cluster before redeploying. - Before every redeploy you MUST fully resolve the failure in swarm and reach at least **95% confidence that your fix actually fixes it**. That confidence MUST come from inspecting the live cluster in-container: reproduce the failure, walk the fix path, and confirm the corrected behaviour on the running services. Never redeploy on a guess — a swarm redeploy costs tens of minutes. -- The run leaves the cluster up (`INFINITO_KEEP_SWARM_NODES=true`). You MUST reproduce the failure on it via `make swarm-exec` / `make swarm-shell`, then release it with `make swarm-down`. +- A run with `keep=true` leaves the cluster up (`INFINITO_KEEP_SWARM_NODES=true`). You MUST reproduce the failure on it via `make swarm-exec` / `make swarm-shell`, then release it with `make swarm-down`. A run without `keep=true` leaves only the rescue bundle on the host. - You MAY confirm an early-stage fix (config render, realm import) while the run is still going. - In-cluster edits are validation only. You MUST apply the real fix in the repository before the next redeploy. - You MUST NOT commit during an active iteration, because a live deploy makes the pre-commit hook stash the files it reads. If a commit is unavoidable, you MUST pass `--no-verify`. diff --git a/docs/contributing/artefact/files/compose.yml.md b/docs/contributing/artefact/files/compose.yml.md index a7a79e1a56..3e3604f70f 100644 --- a/docs/contributing/artefact/files/compose.yml.md +++ b/docs/contributing/artefact/files/compose.yml.md @@ -82,6 +82,8 @@ The env-var contracts each service expects strictly via `${VAR:?…}` (consumed | `INFINITO_CACHE_PACKAGE_FRONTEND_CERTS_DIR` | Host directory for per-hostname leaf certs. | | `INFINITO_CACHE_PACKAGE_FRONTEND_IP` | Static IPv4 address for the frontend on the compose default network. | | `INFINITO_CACHE_PACKAGE_FRONTEND_CA_FILE` | Bind source for `/opt/package-frontend-ca.crt` inside the runner. Driver-injected. | +| `INFINITO_CACHE_SHARED_HOST_PATH` | Optional shared package and tool cache folder, bind-mounted read-write at the same path into the runner through `compose/shared-cache.override.yml` when non-empty. See [cache.md](../../environment/cache.md). | +| `INFINITO_CACHE_HOST_PACMAN_PATH` | Optional pacman cache of an Arch Linux host, bind-mounted read-only at the same path through `compose/pacman.override.yml` when it and the shared folder are non-empty. Empty on every host whose os-release `ID` is not `arch` and on every stack that does not run pacman. | ### Networking diff --git a/docs/contributing/environment/cache.md b/docs/contributing/environment/cache.md index cdb3a279e5..d8bf013bb2 100644 --- a/docs/contributing/environment/cache.md +++ b/docs/contributing/environment/cache.md @@ -1,6 +1,6 @@ # Local Caches 📦 -Three local cache services accelerate re-deploys, reduce upstream traffic, and harden the development workflow against transient upstream outages. They share one compose profile (`cache`) and one override file ([compose/cache.override.yml](../../../compose/cache.override.yml)) and are gated together by [profile.py](../../../cli/administration/deploy/development/profile.py): active on developer machines, inactive on CI runners. +Three local cache services and one shared cache folder accelerate re-deploys, reduce upstream traffic, and harden the development workflow against transient upstream outages. They share one compose profile (`cache`) and one override file ([compose/cache.override.yml](../../../compose/cache.override.yml)) and are gated together by [profile.py](../../../cli/administration/deploy/development/profile.py): active on developer machines, inactive on CI runners. ## Services 🧩 @@ -43,6 +43,34 @@ Two listener layers: Cert generation runs in a throw-away alpine container driven by [certs.sh](../../../roles/svc-cache-package/files/shell/certs.sh) before the frontend starts. It issues a leaf for each name in `INFINITO_CACHE_TLS_HOSTS` and holds no hostname of its own. +### Shared Package Cache Folder 🗂️ + +One host folder, `INFINITO_CACHE_SHARED_HOST_PATH` (default `/shared`, written by [paths.py](../../../utils/env/handlers/infinito/cache/paths.py)), holds the package and tool caches of every compose slot, swarm lab node and GUI lane on the host. Every consumer mounts it read-write at the same path it has on the host. Both cache variables are optional: an empty or missing value layers no mount, no cache environment and no pacman layering, so a checkout whose `.env` predates them renders and starts as before. + +| Subfolder | Tool | How the consumer points at it | +|---|---|---| +| `pacman/` | pacman | last `CacheDir` in `/etc/pacman.conf`, written by [pacman.sh](../../../scripts/docker/cache/pacman.sh); each node downloads into its own `pacman/node-XXXXXX/` | +| `pip/` | pip | `PIP_CACHE_DIR` | +| `npm/` | npm | `npm_config_cache` | +| `ms-playwright/` | Playwright browsers | `PLAYWRIGHT_BROWSERS_PATH` | + +The consumers in this repository are wired as follows: + +- Compose slots: `compose_file_args` layers [compose/shared-cache.override.yml](../../../compose/shared-cache.override.yml) when the cache stack is active and the variable is set. It mounts the folder and `pacman.sh` into the runner and sets `PIP_CACHE_DIR` and `npm_config_cache`. `make compose-up` then runs `pacman.sh` in the runner. +- Swarm lab nodes: `01_bootstrap.sh` layers [compose/swarm/shared-cache.override.yml](../../../compose/swarm/shared-cache.override.yml) next to the cache override when the variable is set. It mounts the folder and `pacman.sh` into the manager, both workers, the NFS server and the backup host, and [playbook.yml](../../../compose/swarm/playbook.yml) runs `pacman.sh` on each node that has it. +- Core Playwright runs take their browsers from the `mcr.microsoft.com/playwright` image, so they leave `ms-playwright/` empty. + +pacman on an arch stack reads up to three cache directories. `INFINITO_CACHE_HOST_PACMAN_PATH`, written by [host_pacman_path.py](../../../utils/env/handlers/infinito/cache/host_pacman_path.py), is the first `CacheDir` of the host's `pacman-conf`, and only on a host whose `/etc/os-release` `ID` is exactly `arch` running an arch stack. On Manjaro, EndeavourOS and every other host, and on every stack that does not run pacman, the variable is empty and pacman uses the shared layer alone. When it is set, [compose/pacman.override.yml](../../../compose/pacman.override.yml) and [compose/swarm/pacman.override.yml](../../../compose/swarm/pacman.override.yml) mount it read-only at the same path, layered only together with the shared folder. `pacman.sh` lists it as the first `CacheDir`, the node's own download directory `/pacman/node-XXXXXX/` as the second and `/pacman/` as the third. `pacman.sh` creates the node download directory with `mktemp` on its first run and reuses the one already listed in `pacman.conf` on every later run. pacman finds a package in any of the three directories and downloads a missing one, `.part` file included, into the node download directory, the first one it can write. The `PostTransaction` hook `pacman.sh` writes to `pacman.d/hooks/infinito-shared-cache.hook` beside `pacman.conf` then renames every finished package and signature from the node download directory into `/pacman/` and leaves `.part` files there for the next resume. `pacman.sh` fails when `pacman.conf` sets `HookDir`. The disc-space cleanup runs `pacman -Sc` per writable `CacheDir`, so it never touches the read-only layer. + +apt and dnf keep their per-container caches. Their downloads come through the `apt-*` and `yum-*` proxies of the package cache, which store each file once for the whole host. + +A GUI lane follows the same contract whenever `INFINITO_CACHE_SHARED_HOST_PATH` is non-empty, and mounts nothing when it is empty: + +- Mount `${INFINITO_CACHE_SHARED_HOST_PATH}` read-write at `${INFINITO_CACHE_SHARED_HOST_PATH}`. +- Set `PIP_CACHE_DIR=${INFINITO_CACHE_SHARED_HOST_PATH}/pip`, `npm_config_cache=${INFINITO_CACHE_SHARED_HOST_PATH}/npm` and `PLAYWRIGHT_BROWSERS_PATH=${INFINITO_CACHE_SHARED_HOST_PATH}/ms-playwright`. +- On an arch lane with a non-empty `INFINITO_CACHE_HOST_PACMAN_PATH`, mount that directory read-only at the same path and run `scripts/docker/cache/pacman.sh /etc/pacman.conf "${INFINITO_CACHE_SHARED_HOST_PATH}" "${INFINITO_CACHE_HOST_PACMAN_PATH}"` in the lane before its first `pacman` call. Pass an empty third argument without the read-only mount. +- Read both values from this checkout's `.env` after `make dotenv`. + ## Activation 🎚️ The `cache` decision is exposed via `Profile.cache_stack_enabled()` in [profile.py](../../../cli/administration/deploy/development/profile.py). When active: @@ -50,7 +78,7 @@ The `cache` decision is exposed via `Profile.cache_stack_enabled()` in [profile. - [compose/cache.override.yml](../../../compose/cache.override.yml) is layered on top of the base [compose.yml](../../../compose.yml) by [compose.py](../../../cli/administration/deploy/development/compose.py) and [down.py](../../../cli/administration/deploy/development/down.py) via [common.py](../../../cli/administration/deploy/development/common.py)`compose_file_args`. - The cache services are added. - The runner's `infinito` service receives: - - bind-mounts for the registry-cache CA, the package-cache client snippets (`pip.conf`, `npmrc`, `apt/${INFINITO_DISTRO}.list`), and the frontend CA file + - bind-mounts for the registry-cache CA, the package-cache client snippets (`pip.conf`, `npmrc`, `apt/${INFINITO_DISTRO}.list`), and the frontend CA file, plus the shared package cache folder when `INFINITO_CACHE_SHARED_HOST_PATH` is set - `extra_hosts` entries DNS-hijacking the HTTPS upstream hostnames to the frontend's static IP - `INFINITO_CACHE_PACKAGE_FRONTEND_IP` env var for the inner compose wrapper - Cert generation, Nexus repo bootstrap, and runner trust-store install run from [compose.py](../../../cli/administration/deploy/development/compose.py). @@ -68,6 +96,8 @@ CI signals (`GITHUB_ACTIONS=true`, `INFINITO_RUNNING_ON_GITHUB=true`, `CI=true`) | `pip install` (Ansible task in runner) | `pip.conf` URL-rewrite | ✓ | | `npm install` (Ansible task in runner) | `.npmrc` URL-rewrite | ✓ | | `gem`, `composer`, `go`, `curl https://…` (Ansible task in runner) | DNS-hijack + frontend HTTPS + runner CA-trust | ✓ | +| `pacman -S` (Ansible task in an arch runner or lab node) | read-only host cache + shared `pacman/` folder | ✓ | +| `pip`/`npm` local caches (runner) | shared `pip/` and `npm/` folders | ✓ | | `RUN apt-get install` (inner Dockerfile build, Debian/Ubuntu) | `build.extra_hosts` DNS-hijack + frontend HTTP | ✓ | | `RUN apk add` (inner Dockerfile build, Alpine ≤3.17 HTTP) | `build.extra_hosts` DNS-hijack + frontend HTTP | ✓ | | `RUN pip/npm/gem/composer/go install` (inner Dockerfile build) | none today | ✗ | diff --git a/docs/requirements/065-shared-stack-caches.md b/docs/requirements/065-shared-stack-caches.md new file mode 100644 index 0000000000..6891df150a --- /dev/null +++ b/docs/requirements/065-shared-stack-caches.md @@ -0,0 +1,28 @@ +# 065 - Shared stack caches and release after rescue + +## User Story + +As an operator who runs several compose slots, swarm lab clusters and GUI lanes on one host, I want repeated package and tool downloads to land in one shared host cache folder and every swarm or roundtrip run to release its cluster once the rescue bundle is saved, so that parallel stacks stop filling the root filesystem. + +## Acceptance Criteria + +- [ ] `make dotenv` writes `INFINITO_CACHE_SHARED_HOST_PATH=/shared` unless the environment or `custom.env` sets another path. +- [ ] With the cache stack enabled and `INFINITO_CACHE_SHARED_HOST_PATH` set, `docker compose config` of a compose slot shows that folder bind-mounted read-write at the same path into the `infinito` service, with `PIP_CACHE_DIR` and `npm_config_cache` pointing below it. +- [ ] With `INFINITO_CACHE_SHARED_HOST_PATH` set, rendering `compose/swarm/compose.yml` with `compose/swarm/cache.override.yml` and `compose/swarm/shared-cache.override.yml` shows the same read-write bind on the manager, both workers, the NFS server and the backup host. +- [ ] With `INFINITO_CACHE_SHARED_HOST_PATH` and `INFINITO_CACHE_HOST_PACMAN_PATH` empty or missing from `.env`, a compose slot renders and starts without the cache mounts and the cache environment, and `docker compose config` of its services is byte-identical to the render of the same `.env` before this change. +- [ ] `make dotenv` writes the host's pacman cache directory to `INFINITO_CACHE_HOST_PACMAN_PATH` only when the host's `/etc/os-release` `ID` is exactly `arch` and the stack runs arch; on Manjaro and every other host or stack the value is empty and pacman uses the shared folder alone. +- [ ] With `INFINITO_CACHE_HOST_PACMAN_PATH` and `INFINITO_CACHE_SHARED_HOST_PATH` set, the rendered compose slot and every rendered swarm lab node mount the host pacman cache read-only at the same path. +- [ ] `scripts/docker/cache/pacman.sh` rewrites `[options]` of a `pacman.conf` so that the read-only host cache is the first `CacheDir`, the node's own download directory `/pacman/node-XXXXXX/` the second and `/pacman/` the third, gives two nodes on one shared folder different download directories, removes every other `CacheDir` line, writes a `PostTransaction` hook that renames finished downloads from the node download directory into `/pacman/`, gives the same result when run twice, and leaves a system without `pacman.conf` untouched. +- [ ] `make compose-up` and the swarm node bootstrap run that script in every arch runner and arch lab node that mounts the shared folder. +- [ ] A package pacman downloads on an arch runner or lab node lands in `/pacman/` once its transaction completes, two nodes that download the same package at the same time never write the same file, and the host cache directory stays byte-identical. +- [ ] apt and dnf keep their per-container caches and reach the package cache proxies as before; `docs/contributing/environment/cache.md` states this next to the folder layout. +- [ ] `docs/contributing/environment/cache.md` documents the folder layout, both variables and the mount contract a GUI lane follows. +- [ ] Images keep reaching every inner daemon through `infinito-registry-cache`; no compose file shares a docker data root between daemons. +- [ ] `make swarm-zombie` without `keep=true` copies the act job's rescue diagnostics and Playwright artefacts to the same paths on the host and then removes the lab nodes, the lab network, the NFS export volume and the act outer container, on success and on failure. +- [ ] `make swarm-zombie keep=true` copies the same artefacts and leaves the cluster and the act outer container up. +- [ ] When the artefact copy fails, the lab nodes are still removed, the act outer container stays, and the run exits non-zero naming `make swarm-down`. +- [ ] `make roundtrip` hands `keep` to every swarm step, so a failed swarm step releases its cluster the same way a green one does. +- [ ] `make clean-cache` removes the shared cache folder. +- [ ] The disc-space cleanup cleans only pacman cache directories it can write, so a read-only host cache layer cannot fail it. +- [ ] `INFINITO_DISTRO` resolves exactly as before this change; deriving the default distro from the host ships as a separate follow-up. +- [ ] Every changed logic has one unit test, and each of these tests fails on the code before this change. diff --git a/roles/sys-ctl-cln-disc-space/files/shell/script.sh b/roles/sys-ctl-cln-disc-space/files/shell/script.sh index 26db5a4bf6..6e6186af47 100755 --- a/roles/sys-ctl-cln-disc-space/files/shell/script.sh +++ b/roles/sys-ctl-cln-disc-space/files/shell/script.sh @@ -89,8 +89,11 @@ if [ "$force_freeing" = true ]; then fi if command -v pacman >/dev/null 2>&1 ; then - echo "cleaning pacman cache" && - yes | pacman -Sc || exit 7 + echo "cleaning pacman cache" + for pacman_cache_dir in $(pacman-conf CacheDir); do + [ -w "${pacman_cache_dir}" ] || continue + yes | pacman -Sc --cachedir "${pacman_cache_dir}" || exit 7 + done fi echo "cleanup finished." diff --git a/scripts/docker/cache/pacman.sh b/scripts/docker/cache/pacman.sh new file mode 100755 index 0000000000..82fc892989 --- /dev/null +++ b/scripts/docker/cache/pacman.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Param: $1 pacman.conf to rewrite; a missing file means no pacman, nothing is layered; the publish hook goes to pacman.d/hooks/ beside it +# Param: $2 shared infinito cache root; pacman downloads into /pacman/node-XXXXXX/ and publishes into /pacman/ +# Param: $3 read-only host pacman cache, mounted at the same path; empty leaves that layer out +set -euo pipefail + +CONF="${1:?usage: pacman.sh }" +SHARED="${2:?usage: pacman.sh }" +HOST="${3?usage: pacman.sh }" + +if [ ! -f "${CONF}" ]; then + echo "[pacman-cache] ${CONF} absent; no pacman to layer" >&2 + exit 0 +fi + +grep -qx '\[options\]' "${CONF}" || { + echo "[pacman-cache] ${CONF} has no [options] section to carry CacheDir" >&2 + exit 1 +} + +if grep -qE '^[[:space:]]*HookDir[[:space:]]*=' "${CONF}"; then + echo "[pacman-cache] ${CONF} sets HookDir, so pacman would not read the publish hook in its default hook directory" >&2 + exit 1 +fi + +if [ -n "${HOST}" ] && [ ! -d "${HOST}" ]; then + echo "[pacman-cache] host cache ${HOST} is not mounted" >&2 + exit 1 +fi + +OWN="${SHARED%/}/pacman" +mkdir -p "${OWN}" + +STAGE="$(awk -v prefix="CacheDir = ${OWN}/node-" ' + index($0, prefix) == 1 { line = substr($0, 12); sub(/\/$/, "", line); print line; exit } +' "${CONF}")" +if [ -n "${STAGE}" ]; then + mkdir -p "${STAGE}" +else + STAGE="$(mktemp -d "${OWN}/node-XXXXXX")" +fi +chmod 0755 "${STAGE}" + +layered="$(mktemp)" +trap 'rm -f "${layered}"' EXIT +awk -v host="${HOST%/}" -v stage="${STAGE}" -v own="${OWN}" ' + /^[[:space:]]*CacheDir[[:space:]]*=/ { next } + { print } + $0 == "[options]" { + if (host != "") print "CacheDir = " host "/" + print "CacheDir = " stage "/" + print "CacheDir = " own "/" + } +' "${CONF}" >"${layered}" +cat "${layered}" >"${CONF}" + +HOOKS="$(dirname "${CONF}")/pacman.d/hooks" +mkdir -p "${HOOKS}" +printf '%s\n' \ + '[Trigger]' \ + 'Operation = Install' \ + 'Operation = Upgrade' \ + 'Type = Package' \ + 'Target = *' \ + '' \ + '[Action]' \ + 'Description = Publishing downloaded packages to the shared pacman cache...' \ + 'When = PostTransaction' \ + "Exec = /usr/bin/find \"${STAGE}\" -maxdepth 1 -type f ! -name *.part -exec /usr/bin/mv -f -t \"${OWN}\" {} +" \ + >"${HOOKS}/infinito-shared-cache.hook" + +echo "[pacman-cache] CacheDir: ${HOST:+${HOST%/}/ (read-only), }${STAGE}/ (downloads), ${OWN}/" >&2 diff --git a/scripts/system/cache/clean.sh b/scripts/system/cache/clean.sh index f19a47d8a8..f70a6f4bd8 100755 --- a/scripts/system/cache/clean.sh +++ b/scripts/system/cache/clean.sh @@ -36,6 +36,7 @@ PATHS=( "${INFINITO_CACHE_REGISTRY_HOST_PATH}" "${INFINITO_CACHE_REGISTRY_CA_HOST_PATH}" "${INFINITO_CACHE_PACKAGE_HOST_PATH}" + "${INFINITO_CACHE_SHARED_HOST_PATH}" ) # Remove each cache path individually (keeps the parent /var/cache/ @@ -62,5 +63,6 @@ remove_path() { remove_path "${PATHS[0]}" remove_path "${PATHS[1]}" remove_path "${PATHS[2]}" +remove_path "${PATHS[3]}" echo "[clean-cache] done. Re-run 'make compose-up' to recreate empty caches." diff --git a/scripts/tests/deploy/roundtrip.sh b/scripts/tests/deploy/roundtrip.sh index 99f281d247..2d0868b952 100755 --- a/scripts/tests/deploy/roundtrip.sh +++ b/scripts/tests/deploy/roundtrip.sh @@ -6,7 +6,8 @@ set -euo pipefail # apps - space-separated role ids; defaults to one role per dna cluster # (complexity clone == false), most complex first, when unset # modes - mode sequence (default "compose swarm"; append "k8s" here once it exists) -# keep - true keeps each validated swarm cluster instead of releasing it +# keep - true keeps each swarm cluster, green or failed; empty or false lets +# swarm-zombie release it once the rescue bundle is on the host # disable - comma-separated provider keys removed from the test inventory; # the Tor key pins the swarm step to clearnet _repo_root="$(cd "$(dirname "$0")/../../.." && pwd)" @@ -23,6 +24,7 @@ fi exit 2 } +: "${keep?keep is required: true keeps each swarm cluster, empty or false releases it}" modes="${modes:-compose swarm}" _log_dir="${TMPDIR:-/tmp}" # nocheck: posix TMPDIR convention @@ -58,13 +60,12 @@ for app in "${_apps[@]}"; do echo "==> roundtrip: ${app} [swarm] network=${network:-rotation}" ACT_PLATFORM_IMAGE=local/act-runner-fixed:latest \ make -C "$_repo_root" swarm-zombie app="$app" network="$network" \ - disable="${disable:-}" 2>&1 | tee "$log" + disable="${disable:-}" keep="${keep}" 2>&1 | tee "$log" marker="==> swarm drill complete: app=${app}" grep -qF "${marker}" "$log" || { echo "roundtrip: ${app} [swarm] FAILED: swarm-zombie exited 0 but the drill never completed for ${app} (marker '${marker}' absent from ${log})" >&2 exit 1 } - [ "${keep:-false}" = true ] || make -C "$_repo_root" swarm-down name="$app" ;; *) echo "roundtrip: unknown mode '${mode}' (expected: compose | swarm)" >&2 diff --git a/scripts/tests/deploy/swarm/README.md b/scripts/tests/deploy/swarm/README.md index 4d820742e5..a324e2f5ac 100644 --- a/scripts/tests/deploy/swarm/README.md +++ b/scripts/tests/deploy/swarm/README.md @@ -164,6 +164,7 @@ the cluster declaration (image, containers, network, DNS play) in | teardown | `utils/collect/playwright_reports.sh` | pull Playwright reports from the manager into `/tmp/playwright-artifacts//` | | teardown | `utils/collect/topology_summary.sh` | render the node list + service placement into `$GITHUB_STEP_SUMMARY` | | teardown | `utils/clean/teardown.sh` | kill the nodes + remove the lab network | +| teardown | `utils/release.sh` | local runs: copy the act job's rescue and Playwright artefacts to the host, then run `utils/clean/teardown.sh` and remove the act outer container unless `keep=true` | | helper | `utils/_context.sh` | per-app facts (entity, service, NFS volume, probes) | | helper | `utils/unmount/nfs_mounts.sh` | best-effort NFS unmount before node removal | | helper | `utils/unmount/host_state.sh` | release the host mount of the shared state so the next cycle cannot inherit it | @@ -175,5 +176,6 @@ The matrix orchestrator per variant round; `routine/00_one.sh` drives the surrounding stages for one distro, and the workflow `.github/workflows/call-test-deploy.yml` drives the distro loop. Run one app locally with -`make swarm-zombie app=` (keeps the cluster for inspection) or the -whole matrix via `make roundtrip`. +`make swarm-zombie app=` or the whole matrix via `make roundtrip`. Both +copy the rescue bundle to the host through `utils/release.sh` and then +release the cluster; `keep=true` keeps it for inspection. diff --git a/scripts/tests/deploy/swarm/routine/01_bootstrap.sh b/scripts/tests/deploy/swarm/routine/01_bootstrap.sh index cc0775bc30..4013c6c366 100755 --- a/scripts/tests/deploy/swarm/routine/01_bootstrap.sh +++ b/scripts/tests/deploy/swarm/routine/01_bootstrap.sh @@ -70,6 +70,14 @@ if docker inspect "${CACHE_FRONTEND}" >/dev/null 2>&1; then echo "==> registry-cache CA source: ${INFINITO_CACHE_REGISTRY_CA_HOST_PATH}" python3 -m cli.meta.cache COMPOSE_ARGS+=(-f "${SCRIPT_DIR}/../../../../../compose/swarm/cache.override.yml") + if [ -n "${INFINITO_CACHE_SHARED_HOST_PATH:-}" ]; then + echo "==> shared package cache ${INFINITO_CACHE_SHARED_HOST_PATH} mounted into every node" + COMPOSE_ARGS+=(-f "${SCRIPT_DIR}/../../../../../compose/swarm/shared-cache.override.yml") + if [ -n "${INFINITO_CACHE_HOST_PACMAN_PATH:-}" ]; then + echo "==> host pacman cache ${INFINITO_CACHE_HOST_PACMAN_PATH} mounted read-only into every node" + COMPOSE_ARGS+=(-f "${SCRIPT_DIR}/../../../../../compose/swarm/pacman.override.yml") + fi + fi fi build_attempts=3 diff --git a/scripts/tests/deploy/swarm/utils/release.sh b/scripts/tests/deploy/swarm/utils/release.sh new file mode 100755 index 0000000000..8ab632a70d --- /dev/null +++ b/scripts/tests/deploy/swarm/utils/release.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Param: SWARM_NAME cluster id the drill runs under +# Param: keep true keeps the cluster and the act outer container up; empty or false releases both +# Param: INFINITO_CONTAINER worktree label the act outer containers carry +# Param: INFINITO_RESCUE_DIAGNOSTICS_BASE rescue root, the same path inside the act job and on the host +# Param: $@ the drill command; its exit code is the script's exit code +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../../.." && pwd)" +PLAYWRIGHT_ARTEFACTS=/tmp/playwright-artifacts + +: "${SWARM_NAME:?SWARM_NAME is required (the cluster id)}" +: "${keep?keep is required: true keeps the cluster, empty or false releases it}" +: "${INFINITO_CONTAINER:?INFINITO_CONTAINER is not set; source scripts/meta/env/load.sh or invoke via make}" +: "${INFINITO_RESCUE_DIAGNOSTICS_BASE:?INFINITO_RESCUE_DIAGNOSTICS_BASE is not set; source scripts/meta/env/load.sh or invoke via make}" +case "${keep}" in +true | false | "") ;; +*) + echo "release: keep must be true, false or empty, got '${keep}'" >&2 + exit 2 + ;; +esac +[ "$#" -gt 0 ] || { + echo "usage: release.sh " >&2 + exit 2 +} + +save_artefacts() { + local id path out + local -a ids + mapfile -t ids < <(docker ps -aq --filter "name=^act-" --filter "label=infinito.worktree=${INFINITO_CONTAINER}") + for id in "${ids[@]}"; do + for path in "${INFINITO_RESCUE_DIAGNOSTICS_BASE}" "${PLAYWRIGHT_ARTEFACTS}"; do + mkdir -p "${path}" + # nocheck: container-cp - the act outer container runs on this host's own daemon, never on a swarm worker + if out="$(docker cp "${id}:${path}/." "${path}/" 2>&1)"; then + echo "==> release: saved ${path} from act container ${id}" + continue + fi + case "${out}" in + *"Could not find the file"* | *"No such container:path"*) + echo "==> release: act container ${id} holds no ${path}" + ;; + *) + echo "==> release: FAILED to save ${path} from act container ${id}: ${out}" >&2 + return 1 + ;; + esac + done + done +} + +release() { + local rc=$? saved=true + trap - EXIT + save_artefacts || saved=false + if [ "${keep}" = true ]; then + echo "==> release: keep=true, cluster ${SWARM_NAME} stays up; release it with make swarm-down name=${SWARM_NAME}" + exit "${rc}" + fi + SWARM_NAME="${SWARM_NAME}" INFINITO_KEEP_SWARM_NODES=false \ + bash "${SCRIPT_DIR}/clean/teardown.sh" + if [ "${saved}" = true ]; then + bash "${REPO_ROOT}/scripts/tests/deploy/act/down_act_outer.sh" + else + echo "==> release: the act outer container stays because its artefacts could not be copied; copy them, then run make swarm-down name=${SWARM_NAME}" >&2 + [ "${rc}" -ne 0 ] || rc=1 + fi + exit "${rc}" +} +trap release EXIT + +"$@" diff --git a/tests/unit/python/cli/administration/deploy/development/test_compose.py b/tests/unit/python/cli/administration/deploy/development/test_compose.py index 9cbc999255..1fe674f485 100644 --- a/tests/unit/python/cli/administration/deploy/development/test_compose.py +++ b/tests/unit/python/cli/administration/deploy/development/test_compose.py @@ -154,6 +154,8 @@ class TestComposeUpRetries(unittest.TestCase): "INFINITO_GIT_COMMON_DIR": "", "INFINITO_CACHE_NETWORK": "", "INFINITO_CACHE_STACK": "", + "INFINITO_CACHE_SHARED_HOST_PATH": "", + "INFINITO_CACHE_HOST_PACMAN_PATH": "", "INFINITO_GPU_COUNT": "0", "INFINITO_GHCR_MIRROR": "", }, @@ -273,6 +275,7 @@ class TestComposeUpRetries(unittest.TestCase): "INFINITO_GIT_COMMON_DIR": "", "INFINITO_CACHE_NETWORK": "", "INFINITO_CACHE_STACK": "", + "INFINITO_CACHE_SHARED_HOST_PATH": "/var/cache/infinito/core/cache/shared", "INFINITO_GHCR_MIRROR": "", }, clear=False, @@ -285,6 +288,7 @@ class TestComposeUpRetries(unittest.TestCase): compose._bootstrap_package_cache = MagicMock() compose._generate_package_frontend_certs = MagicMock() compose._install_package_frontend_ca_in_runner = MagicMock() + compose._layer_pacman_cache_in_runner = MagicMock() compose.up(run_entry_init=False) @@ -305,6 +309,7 @@ class TestComposeUpRetries(unittest.TestCase): compose._bootstrap_package_cache.assert_called_once() compose._generate_package_frontend_certs.assert_called_once() compose._install_package_frontend_ca_in_runner.assert_called_once() + compose._layer_pacman_cache_in_runner.assert_called_once() @patch.dict( os.environ, @@ -318,6 +323,7 @@ class TestComposeUpRetries(unittest.TestCase): "INFINITO_GIT_COMMON_DIR": "/primary/.git", "INFINITO_CACHE_NETWORK": "infinito-nexus-core_default", "INFINITO_CACHE_STACK": "", + "INFINITO_CACHE_SHARED_HOST_PATH": "/var/cache/infinito/core/cache/shared", }, clear=False, ) @@ -330,6 +336,7 @@ class TestComposeUpRetries(unittest.TestCase): compose._generate_package_frontend_certs = MagicMock() compose._apply_shared_cache = MagicMock() compose._install_package_frontend_ca_in_runner = MagicMock() + compose._layer_pacman_cache_in_runner = MagicMock() compose.up(run_entry_init=False) @@ -340,6 +347,63 @@ class TestComposeUpRetries(unittest.TestCase): compose._bootstrap_package_cache.assert_not_called() compose._apply_shared_cache.assert_called_once() compose._install_package_frontend_ca_in_runner.assert_called_once() + compose._layer_pacman_cache_in_runner.assert_called_once() + + @patch.dict( + os.environ, + { + "INFINITO_BUILD": "1", + "INFINITO_IMAGE": "infinito-debian", + "INFINITO_PULL_POLICY": "never", + "CI": "", + "GITHUB_ACTIONS": "", + "INFINITO_RUNNING_ON_GITHUB": "", + "INFINITO_GIT_COMMON_DIR": "", + "INFINITO_CACHE_NETWORK": "", + "INFINITO_CACHE_STACK": "", + "INFINITO_CACHE_SHARED_HOST_PATH": "", + "INFINITO_GHCR_MIRROR": "", + }, + clear=False, + ) + def test_up_skips_the_pacman_layering_without_a_shared_folder(self) -> None: + compose = self._compose() + compose._render_coredns_corefile = MagicMock() + compose._compose_up_with_retries = MagicMock() + compose.wait_for_healthy = MagicMock() + compose._bootstrap_package_cache = MagicMock() + compose._generate_package_frontend_certs = MagicMock() + compose._install_package_frontend_ca_in_runner = MagicMock() + compose._layer_pacman_cache_in_runner = MagicMock() + + compose.up(run_entry_init=False) + + compose._install_package_frontend_ca_in_runner.assert_called_once() + compose._layer_pacman_cache_in_runner.assert_not_called() + + @patch.dict( + os.environ, + { + "INFINITO_CACHE_SHARED_HOST_PATH": "/var/cache/infinito/core/cache/shared", + "INFINITO_CACHE_HOST_PACMAN_PATH": "/var/cache/pacman/pkg", + }, + clear=False, + ) + def test_the_runner_layers_pacman_over_both_caches(self) -> None: + compose = self._compose() + compose.exec = MagicMock() + + compose._layer_pacman_cache_in_runner() + + compose.exec.assert_called_once_with( + [ + "/usr/local/bin/pacman-cache.sh", + "/etc/pacman.conf", + "/var/cache/infinito/core/cache/shared", + "/var/cache/pacman/pkg", + ], + live=False, + ) if __name__ == "__main__": # pragma: no cover diff --git a/tests/unit/python/cli/administration/deploy/development/test_env_compose_file_args.py b/tests/unit/python/cli/administration/deploy/development/test_env_compose_file_args.py index e3ac2a5445..04e4bf7469 100644 --- a/tests/unit/python/cli/administration/deploy/development/test_env_compose_file_args.py +++ b/tests/unit/python/cli/administration/deploy/development/test_env_compose_file_args.py @@ -29,6 +29,8 @@ _LOCAL_PRIMARY = { "INFINITO_GIT_COMMON_DIR": "", "INFINITO_CACHE_NETWORK": "", "INFINITO_CACHE_STACK": "", + "INFINITO_CACHE_SHARED_HOST_PATH": "", + "INFINITO_CACHE_HOST_PACMAN_PATH": "", "INFINITO_GHCR_MIRROR": "", "INFINITO_PUBLISH_PORTS": "", "INFINITO_GPU_COUNT": "0", @@ -40,6 +42,9 @@ _LOCAL_PRIMARY = { "INFINITO_EXTRA_MOUNTS": "", } +_SHARED = "/var/cache/infinito/core/cache/shared" +_HOST_PACMAN = "/var/cache/pacman/pkg" + _WORKTREE = { **_LOCAL_PRIMARY, "INFINITO_INSTANCE": "2", @@ -132,6 +137,111 @@ class TestComposeFileArgs(unittest.TestCase): ["-f", "compose.yml", "-f", "compose/tools.override.yml"], ) + @patch.dict( + os.environ, + {**_LOCAL_PRIMARY, "INFINITO_CACHE_SHARED_HOST_PATH": _SHARED}, + clear=False, + ) + def test_a_shared_cache_folder_adds_its_mount(self) -> None: + self.assertEqual( + compose_file_args(), + [ + "-f", + "compose.yml", + "-f", + "compose/cache.override.yml", + "-f", + "compose/shared-cache.override.yml", + "-f", + "compose/tools.override.yml", + ], + ) + + @patch.dict( + os.environ, + { + **_LOCAL_PRIMARY, + "INFINITO_CACHE_SHARED_HOST_PATH": _SHARED, + "INFINITO_CACHE_HOST_PACMAN_PATH": _HOST_PACMAN, + }, + clear=False, + ) + def test_a_host_pacman_cache_adds_its_read_only_mount(self) -> None: + self.assertEqual( + compose_file_args(), + [ + "-f", + "compose.yml", + "-f", + "compose/cache.override.yml", + "-f", + "compose/shared-cache.override.yml", + "-f", + "compose/pacman.override.yml", + "-f", + "compose/tools.override.yml", + ], + ) + + @patch.dict( + os.environ, + {**_LOCAL_PRIMARY, "INFINITO_CACHE_HOST_PACMAN_PATH": _HOST_PACMAN}, + clear=False, + ) + def test_a_host_pacman_cache_needs_the_shared_folder(self) -> None: + self.assertEqual( + compose_file_args(), + [ + "-f", + "compose.yml", + "-f", + "compose/cache.override.yml", + "-f", + "compose/tools.override.yml", + ], + ) + + @patch.dict( + os.environ, + { + **_LOCAL_PRIMARY, + "CI": "true", + "INFINITO_CACHE_SHARED_HOST_PATH": _SHARED, + "INFINITO_CACHE_HOST_PACMAN_PATH": _HOST_PACMAN, + }, + clear=False, + ) + def test_the_cache_folders_stay_out_without_the_cache_stack(self) -> None: + self.assertEqual( + compose_file_args(), + ["-f", "compose.yml", "-f", "compose/tools.override.yml"], + ) + + def test_an_env_without_the_cache_keys_layers_neither_override(self) -> None: + stale = { + k: v + for k, v in _LOCAL_PRIMARY.items() + if k + not in ( + "INFINITO_CACHE_SHARED_HOST_PATH", + "INFINITO_CACHE_HOST_PACMAN_PATH", + ) + } + with patch.dict(os.environ, stale, clear=False): + os.environ.pop("INFINITO_CACHE_SHARED_HOST_PATH", None) + os.environ.pop("INFINITO_CACHE_HOST_PACMAN_PATH", None) + self.assertEqual( + compose_file_args(), + [ + "-f", + "compose.yml", + "-f", + "compose/cache.override.yml", + "-f", + "compose/tools.override.yml", + ], + ) + @patch.dict( os.environ, {**_LOCAL_PRIMARY, "INFINITO_PUBLISH_PORTS": "false"}, diff --git a/tests/unit/python/compose/__init__.py b/tests/unit/python/compose/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/compose/test_shared_cache_mounts.py b/tests/unit/python/compose/test_shared_cache_mounts.py new file mode 100644 index 0000000000..72c64ca945 --- /dev/null +++ b/tests/unit/python/compose/test_shared_cache_mounts.py @@ -0,0 +1,123 @@ +"""The cache overrides mount the shared folder at its host path; nothing always layered needs it.""" + +from __future__ import annotations + +import re +import unittest + +from utils.cache.files import PROJECT_ROOT, read_text +from utils.cache.render import cache_consumer_yml +from utils.cache.yaml import load_yaml_any + +SHARED = "INFINITO_CACHE_SHARED_HOST_PATH" +HOST_PACMAN = "INFINITO_CACHE_HOST_PACMAN_PATH" +PACMAN_SCRIPT = "/usr/local/bin/pacman-cache.sh" +LAB_NODES = ( + "nfs-server", + "swarm-mgr-01", + "swarm-wrk-01", + "swarm-wrk-02", + "backup-host", +) + + +def _var(text: str) -> str: + """Return the variable a compose interpolation names, empty for a literal.""" + found = re.fullmatch(r"\$\{([A-Z_][A-Z0-9_]*)(?::\?[^}]*)?\}", text.strip()) + return found.group(1) if found else "" + + +def _short(entry: str) -> list[str]: + """Split a short-syntax mount on the colons outside interpolations.""" + parts, depth, current = [], 0, "" + for char in entry: + if char == "{" and current.endswith("$"): + depth += 1 + elif char == "}" and depth: + depth -= 1 + if char == ":" and not depth: + parts.append(current) + current = "" + continue + current += char + return [*parts, current] + + +def _load(*parts: str) -> dict: + return load_yaml_any(str(PROJECT_ROOT.joinpath(*parts))) + + +class TestComposeSlot(unittest.TestCase): + def setUp(self) -> None: + self.runner = _load("compose", "shared-cache.override.yml")["services"][ + "infinito" + ] + + def test_the_shared_folder_is_bound_read_write_at_its_host_path(self) -> None: + binds = [ + v + for v in self.runner["volumes"] + if isinstance(v, dict) and _var(v["source"]) == SHARED + ] + self.assertEqual(len(binds), 1, self.runner["volumes"]) + self.assertEqual(_var(binds[0]["target"]), SHARED) + self.assertFalse(binds[0].get("read_only", False)) + + def test_pip_and_npm_cache_below_the_shared_folder(self) -> None: + env = self.runner["environment"] + self.assertRegex(env["PIP_CACHE_DIR"], rf"^\$\{{{SHARED}[^}}]*\}}/pip$") + self.assertRegex(env["npm_config_cache"], rf"^\$\{{{SHARED}[^}}]*\}}/npm$") + + def test_the_pacman_script_is_mounted(self) -> None: + targets = [_short(v)[1] for v in self.runner["volumes"] if isinstance(v, str)] + self.assertIn(PACMAN_SCRIPT, targets) + + +class TestAlwaysLayeredFiles(unittest.TestCase): + def test_the_always_layered_files_never_name_the_cache_variables(self) -> None: + always = ( + cache_consumer_yml(), + read_text(str(PROJECT_ROOT / "compose.yml")), + read_text(str(PROJECT_ROOT / "compose" / "cache.override.yml")), + read_text(str(PROJECT_ROOT / "compose" / "swarm" / "cache.override.yml")), + ) + for text in always: + self.assertNotIn(SHARED, text) + self.assertNotIn(HOST_PACMAN, text) + + +class TestHostPacmanOverride(unittest.TestCase): + def test_the_host_pacman_cache_is_mounted_read_only_at_its_host_path(self) -> None: + (bind,) = _load("compose", "pacman.override.yml")["services"]["infinito"][ + "volumes" + ] + self.assertEqual(_var(bind["source"]), HOST_PACMAN) + self.assertEqual(_var(bind["target"]), HOST_PACMAN) + self.assertTrue(bind["read_only"]) + + +class TestSwarmLabNodes(unittest.TestCase): + def test_every_node_binds_the_shared_folder_read_write(self) -> None: + services = _load("compose", "swarm", "shared-cache.override.yml")["services"] + for node in LAB_NODES: + with self.subTest(node=node): + mounts = [_short(v) for v in services[node]["volumes"]] + shared = [m for m in mounts if _var(m[0]) == SHARED] + self.assertEqual(len(shared), 1, mounts) + self.assertEqual(_var(shared[0][1]), SHARED) + self.assertEqual(len(shared[0]), 2, "read-write: no :ro suffix") + self.assertIn(PACMAN_SCRIPT, [m[1] for m in mounts]) + + def test_every_node_reads_the_host_pacman_cache_read_only(self) -> None: + services = _load("compose", "swarm", "pacman.override.yml")["services"] + for node in LAB_NODES: + with self.subTest(node=node): + (mount,) = [_short(v) for v in services[node]["volumes"]] + self.assertEqual( + [_var(mount[0]), _var(mount[1]), mount[2]], + [HOST_PACMAN, HOST_PACMAN, "ro"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/roles/sys-ctl-cln-disc-space/__init__.py b/tests/unit/python/roles/sys-ctl-cln-disc-space/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/roles/sys-ctl-cln-disc-space/files/__init__.py b/tests/unit/python/roles/sys-ctl-cln-disc-space/files/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/roles/sys-ctl-cln-disc-space/files/shell/__init__.py b/tests/unit/python/roles/sys-ctl-cln-disc-space/files/shell/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/roles/sys-ctl-cln-disc-space/files/shell/test_script.py b/tests/unit/python/roles/sys-ctl-cln-disc-space/files/shell/test_script.py new file mode 100644 index 0000000000..b11e35f9ae --- /dev/null +++ b/tests/unit/python/roles/sys-ctl-cln-disc-space/files/shell/test_script.py @@ -0,0 +1,57 @@ +"""The disc-space cleanup cleans only the pacman cache directories it can write.""" + +from __future__ import annotations + +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils.cache.files import PROJECT_ROOT + +SCRIPT = ( + PROJECT_ROOT / "roles" / "sys-ctl-cln-disc-space" / "files" / "shell" / "script.sh" +) +STUBS = { + "df": '#!/bin/sh\necho "Use%"\n', + "find": "#!/bin/sh\nexit 0\n", + "systemctl": "#!/bin/sh\nexit 1\n", + "pacman": '#!/bin/sh\nprintf "%s\\n" "$*" >>"${STUB_LOG}"\n', +} + + +class TestPacmanCleanup(unittest.TestCase): + def test_a_cache_dir_it_cannot_write_is_skipped(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + bin_dir = Path(tmp) / "bin" + bin_dir.mkdir() + writable = Path(tmp) / "shared" / "pacman" + writable.mkdir(parents=True) + unwritable = Path(tmp) / "host-pkg-not-writable" + stubs = { + **STUBS, + "pacman-conf": f'#!/bin/sh\nprintf "%s/\\n%s/\\n" "{unwritable}" "{writable}"\n', + } + for name, body in stubs.items(): + stub = bin_dir / name + stub.write_text(body) + stub.chmod(0o755) + for name in ("sed", "yes"): + (bin_dir / name).symlink_to(shutil.which(name)) + log = Path(tmp) / "pacman.log" + log.touch() + done = subprocess.run( + ["/bin/sh", str(SCRIPT), "90", "--force"], + env={"PATH": str(bin_dir), "STUB_LOG": str(log)}, + capture_output=True, + text=True, + check=False, + ) + calls = log.read_text().splitlines() # nocheck: cache-read stub call log + self.assertEqual(done.returncode, 0, done.stdout + done.stderr) + self.assertEqual(calls, [f"-Sc --cachedir {writable}/"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/scripts/docker/cache/test_pacman.py b/tests/unit/python/scripts/docker/cache/test_pacman.py new file mode 100644 index 0000000000..edf960b341 --- /dev/null +++ b/tests/unit/python/scripts/docker/cache/test_pacman.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +import shlex +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils.cache.files import PROJECT_ROOT + +SCRIPT = PROJECT_ROOT / "scripts" / "docker" / "cache" / "pacman.sh" +CONF = """\ +[options] +#CacheDir = /var/cache/pacman/pkg/ +CacheDir = /var/cache/pacman/pkg/ +HoldPkg = pacman glibc +Architecture = auto + +[core] +Include = /etc/pacman.d/mirrorlist +""" +HOOK = Path("pacman.d") / "hooks" / "infinito-shared-cache.hook" + + +def layer(conf: Path, shared: Path, host: str) -> subprocess.CompletedProcess: + """Run pacman.sh against ``conf``. + + Args: + conf: the pacman.conf to rewrite. + shared: the shared cache root. + host: the read-only host cache, empty for none. + """ + return subprocess.run( + ["bash", str(SCRIPT), str(conf), str(shared), host], + capture_output=True, + text=True, + check=False, + ) + + +class TestPacmanCacheLayering(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + self.conf = self.tmp / "pacman.conf" + self.conf.write_text(CONF) + self.shared = self.tmp / "shared" + self.host = self.tmp / "host-pkg" + self.host.mkdir() + + def tearDown(self) -> None: + self._tmp.cleanup() + + def _text(self) -> str: + return self.conf.read_text() # nocheck: cache-read the script rewrites it + + def _hook(self) -> str: + hook = self.tmp / HOOK + return hook.read_text() # nocheck: cache-read the script writes it + + def _options(self) -> list[str]: + lines = self._text().splitlines() + return lines[: lines.index("") if "" in lines else len(lines)] + + def _stage(self, conf: Path) -> Path: + prefix = f"CacheDir = {self.shared}/pacman/node-" + text = conf.read_text() # nocheck: cache-read the script rewrites it + lines = text.splitlines() + stages = [line for line in lines if line.startswith(prefix)] + self.assertEqual(len(stages), 1, lines) + self.assertTrue(stages[0].endswith("/"), stages[0]) + return Path(stages[0].removeprefix("CacheDir = ").rstrip("/")) + + def test_host_cache_comes_first_then_the_node_download_dir_then_the_shared_cache( + self, + ) -> None: + done = layer(self.conf, self.shared, str(self.host)) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual( + self._options()[:4], + [ + "[options]", + f"CacheDir = {self.host}/", + f"CacheDir = {self._stage(self.conf)}/", + f"CacheDir = {self.shared}/pacman/", + ], + ) + + def test_every_other_cache_dir_is_dropped_and_the_rest_kept(self) -> None: + layer(self.conf, self.shared, str(self.host)) + text = self._text() + self.assertNotIn("CacheDir = /var/cache/pacman/pkg/", text) + self.assertIn("#CacheDir = /var/cache/pacman/pkg/", text) + self.assertIn("[core]\nInclude = /etc/pacman.d/mirrorlist\n", text) + + def test_the_shared_pacman_dir_and_a_traversable_node_download_dir_are_created( + self, + ) -> None: + layer(self.conf, self.shared, str(self.host)) + stage = self._stage(self.conf) + self.assertTrue((self.shared / "pacman").is_dir()) + self.assertTrue(stage.is_dir()) + self.assertEqual(stage.parent, self.shared / "pacman") + self.assertEqual(stage.stat().st_mode & 0o777, 0o755) + + def test_a_second_run_changes_nothing(self) -> None: + layer(self.conf, self.shared, str(self.host)) + first, first_hook = self._text(), self._hook() + done = layer(self.conf, self.shared, str(self.host)) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(self._text(), first) + self.assertEqual(self._hook(), first_hook) + self.assertEqual(len(list((self.shared / "pacman").glob("node-*"))), 1) + + def test_a_second_run_recreates_a_removed_node_download_dir_under_its_name( + self, + ) -> None: + layer(self.conf, self.shared, str(self.host)) + stage = self._stage(self.conf) + stage.rmdir() + first = self._text() + layer(self.conf, self.shared, str(self.host)) + self.assertEqual(self._text(), first) + self.assertTrue(stage.is_dir()) + + def test_two_nodes_on_one_shared_folder_download_into_different_dirs(self) -> None: + other = self.tmp / "other" / "pacman.conf" + other.parent.mkdir() + other.write_text(CONF) + layer(self.conf, self.shared, str(self.host)) + layer(other, self.shared, str(self.host)) + self.assertNotEqual(self._stage(self.conf), self._stage(other)) + self.assertTrue(self._stage(other).is_dir()) + + def test_an_empty_host_cache_leaves_the_node_download_dir_and_the_shared_layer( + self, + ) -> None: + layer(self.conf, self.shared, "") + self.assertEqual( + self._options()[:3], + [ + "[options]", + f"CacheDir = {self._stage(self.conf)}/", + f"CacheDir = {self.shared}/pacman/", + ], + ) + self.assertEqual( + sum(line.startswith("CacheDir") for line in self._options()), 2 + ) + + def test_the_hook_runs_after_every_install_and_upgrade(self) -> None: + layer(self.conf, self.shared, str(self.host)) + hook = self._hook().splitlines() + for line in ( + "Operation = Install", + "Operation = Upgrade", + "Type = Package", + "Target = *", + "When = PostTransaction", + ): + self.assertIn(line, hook) + + def test_the_hook_publishes_finished_downloads_and_keeps_partials(self) -> None: + layer(self.conf, self.shared, str(self.host)) + stage = self._stage(self.conf) + own = self.shared / "pacman" + for name in ("a-1-1-any.pkg.tar.zst", "a-1-1-any.pkg.tar.zst.sig"): + (stage / name).write_text(name) + (stage / "b-1-1-any.pkg.tar.zst.part").write_text("partial") + (own / "a-1-1-any.pkg.tar.zst").write_text("stale") + (exec_line,) = [ + line for line in self._hook().splitlines() if line.startswith("Exec = ") + ] + done = subprocess.run( + shlex.split(exec_line.removeprefix("Exec = ")), + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual( + sorted(path.name for path in stage.iterdir()), + ["b-1-1-any.pkg.tar.zst.part"], + ) + published = own / "a-1-1-any.pkg.tar.zst" + self.assertEqual( + published.read_text(), # nocheck: cache-read the hook moves it + "a-1-1-any.pkg.tar.zst", + ) + self.assertTrue((own / "a-1-1-any.pkg.tar.zst.sig").is_file()) + + def test_a_system_without_pacman_conf_is_left_alone(self) -> None: + done = layer(self.tmp / "absent.conf", self.shared, str(self.host)) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertFalse(self.shared.exists()) + self.assertFalse((self.tmp / HOOK).exists()) + + def test_a_host_cache_that_is_not_mounted_fails(self) -> None: + done = layer(self.conf, self.shared, str(self.tmp / "missing")) + self.assertNotEqual(done.returncode, 0) + self.assertEqual(self._text(), CONF) + self.assertFalse((self.tmp / HOOK).exists()) + + def test_a_hook_dir_override_fails(self) -> None: + conf = CONF.replace("[options]\n", "[options]\nHookDir = /srv/hooks/\n") + self.conf.write_text(conf) + done = layer(self.conf, self.shared, str(self.host)) + self.assertNotEqual(done.returncode, 0) + self.assertEqual(self._text(), conf) + self.assertFalse(self.shared.exists()) + + def test_a_missing_shared_root_fails(self) -> None: + done = subprocess.run( + ["bash", str(SCRIPT), str(self.conf)], + capture_output=True, + text=True, + check=False, + ) + self.assertNotEqual(done.returncode, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/scripts/tests/deploy/swarm/utils/test_release.py b/tests/unit/python/scripts/tests/deploy/swarm/utils/test_release.py new file mode 100644 index 0000000000..a8402be8b5 --- /dev/null +++ b/tests/unit/python/scripts/tests/deploy/swarm/utils/test_release.py @@ -0,0 +1,171 @@ +"""release.sh saves the act job's artefacts, then releases the cluster in every outcome.""" + +from __future__ import annotations + +import os +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils.cache.files import PROJECT_ROOT, read_text + +RELEASE = ( + PROJECT_ROOT / "scripts" / "tests" / "deploy" / "swarm" / "utils" / "release.sh" +) +MAKEFILE = PROJECT_ROOT / "Makefile" +DOCKER_STUB = ( + "#!/usr/bin/env bash\n" + 'printf "docker %s\\n" "$*" >>"${STUB_LOG}"\n' + 'if [ "$1" = ps ]; then echo act-id; exit 0; fi\n' + 'if [ "$1" = cp ]; then\n' + ' if [ -n "${STUB_CP_ERROR}" ]; then echo "${STUB_CP_ERROR}" >&2; exit 1; fi\n' + " exit 0\n" + "fi\n" +) +TEARDOWN_STUB = ( + "#!/usr/bin/env bash\n" + 'printf "teardown SWARM_NAME=%s KEEP=%s\\n" "${SWARM_NAME}" ' + '"${INFINITO_KEEP_SWARM_NODES}" >>"${STUB_LOG}"\n' +) +DOWN_ACT_STUB = '#!/usr/bin/env bash\nprintf "down_act_outer\\n" >>"${STUB_LOG}"\n' + + +def release( + keep: str, drill_rc: int, cp_error: str = "" +) -> tuple[subprocess.CompletedProcess, list[str]]: + """Run release.sh around a drill that exits ``drill_rc``. + + Args: + keep: the ``keep`` value make hands over. + drill_rc: exit code of the stub drill. + cp_error: stderr the stub ``docker cp`` fails with, empty for success. + + Returns: + The finished process and every stub call in order. + """ + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "repo" + utils_dir = root / "scripts" / "tests" / "deploy" / "swarm" / "utils" + (utils_dir / "clean").mkdir(parents=True) + (utils_dir / "release.sh").symlink_to(RELEASE) + act_dir = root / "scripts" / "tests" / "deploy" / "act" + act_dir.mkdir(parents=True) + bin_dir = Path(tmp) / "bin" + bin_dir.mkdir() + for path, body in ( + (utils_dir / "clean" / "teardown.sh", TEARDOWN_STUB), + (act_dir / "down_act_outer.sh", DOWN_ACT_STUB), + (bin_dir / "docker", DOCKER_STUB), + ): + path.write_text(body) + path.chmod(0o755) + log = Path(tmp) / "calls.log" + log.touch() + env = {k: v for k, v in os.environ.items() if k != "BASH_ENV"} + env.update( + PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + STUB_LOG=str(log), + STUB_CP_ERROR=cp_error, + SWARM_NAME="web-app-demo", + keep=keep, + INFINITO_CONTAINER="own", + INFINITO_RESCUE_DIAGNOSTICS_BASE=str(Path(tmp) / "rescue"), + ) + done = subprocess.run( + [ + "bash", + str(utils_dir / "release.sh"), + "bash", + "-c", + f'echo drill >>"{log}"; exit {drill_rc}', + ], + env=env, + capture_output=True, + text=True, + check=False, + ) + return done, log.read_text().splitlines() # nocheck: cache-read stub call log + + +class TestReleaseAfterRescue(unittest.TestCase): + def test_a_green_drill_saves_then_releases(self) -> None: + done, calls = release("", 0) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(calls[0], "drill") + self.assertTrue(calls[2].startswith("docker cp act-id:"), calls) + self.assertEqual(calls[-2], "teardown SWARM_NAME=web-app-demo KEEP=false") + self.assertEqual(calls[-1], "down_act_outer") + + def test_a_failed_drill_releases_too_and_keeps_its_exit_code(self) -> None: + done, calls = release("false", 3) + self.assertEqual(done.returncode, 3) + self.assertIn("teardown SWARM_NAME=web-app-demo KEEP=false", calls) + self.assertEqual(calls[-1], "down_act_outer") + + def test_the_rescue_bundle_is_copied_before_anything_is_removed(self) -> None: + _, calls = release("", 1) + copies = [i for i, call in enumerate(calls) if call.startswith("docker cp ")] + self.assertEqual(len(copies), 2, calls) + self.assertLess( + max(copies), calls.index(next(c for c in calls if c.startswith("teardown"))) + ) + + def test_keep_true_saves_but_leaves_the_cluster_up(self) -> None: + done, calls = release("true", 1) + self.assertEqual(done.returncode, 1) + self.assertTrue(any(call.startswith("docker cp ") for call in calls)) + self.assertFalse([call for call in calls if call.startswith("teardown")]) + self.assertNotIn("down_act_outer", calls) + + def test_an_absent_bundle_does_not_stop_the_release(self) -> None: + done, calls = release( + "", 0, cp_error="Error: No such container:path: act-id:/tmp/x" + ) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(calls[-1], "down_act_outer") + + def test_a_failed_copy_keeps_the_act_container_and_fails(self) -> None: + done, calls = release("", 0, cp_error="no space left on device") + self.assertNotEqual(done.returncode, 0) + self.assertIn("teardown SWARM_NAME=web-app-demo KEEP=false", calls) + self.assertNotIn("down_act_outer", calls) + self.assertIn("make swarm-down name=web-app-demo", done.stderr) + + def test_an_unknown_keep_value_runs_nothing(self) -> None: + done, calls = release("yes", 0) + self.assertEqual(done.returncode, 2) + self.assertEqual(calls, []) + + +class TestSwarmZombieWiring(unittest.TestCase): + def setUp(self) -> None: + text = read_text(str(MAKEFILE)) + self.recipe = re.search( + r"^swarm-zombie:.*?(?=^\.PHONY)", text, re.DOTALL | re.MULTILINE + )[0] + + def test_the_drill_runs_inside_release(self) -> None: + self.assertIn( + "bash scripts/tests/deploy/swarm/utils/release.sh bash scripts/tests/deploy/act/workflow.sh", + self.recipe, + ) + + def test_an_env_without_the_cache_keys_still_starts_the_drill(self) -> None: + for key in ( + "INFINITO_CACHE_SHARED_HOST_PATH", + "INFINITO_CACHE_HOST_PACMAN_PATH", + ): + self.assertIn(f"{key}=$${{{key}:-}};", self.recipe) + + def test_keep_decides_whether_the_act_job_keeps_the_nodes(self) -> None: + self.assertIn("keep='$(keep)'", self.recipe) + self.assertIn( + "INFINITO_KEEP_SWARM_NODES=$(if $(filter true,$(keep)),true,false)", + self.recipe, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/scripts/tests/deploy/test_roundtrip.py b/tests/unit/python/scripts/tests/deploy/test_roundtrip.py index 5012055a33..106bf1b606 100644 --- a/tests/unit/python/scripts/tests/deploy/test_roundtrip.py +++ b/tests/unit/python/scripts/tests/deploy/test_roundtrip.py @@ -41,6 +41,23 @@ def swarm_network(disable: str, *, multi_accepted: bool) -> str: Returns: The ``network=`` value the roundtrip hands to ``swarm-zombie``. """ + calls = swarm_make_calls(disable, multi_accepted=multi_accepted, keep="true") + zombie = next(call for call in calls if " swarm-zombie " in call) + return re.search(r"(?:^| )network=(\S*)", zombie).group(1) + + +def swarm_make_calls(disable: str, *, multi_accepted: bool, keep: str) -> list[str]: + """Run the roundtrip's swarm step against stubs and return every make call. + + Args: + disable: the ``disable`` env the roundtrip runs with. + multi_accepted: whether the stubbed validator accepts the role's + ``multi`` row. + keep: the ``keep`` env the roundtrip runs with. + + Returns: + The argument lines the stubbed ``make`` received, in order. + """ with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) / "repo" script = root / "scripts" / "tests" / "deploy" / "roundtrip.sh" @@ -69,7 +86,7 @@ def swarm_network(disable: str, *, multi_accepted: bool) -> str: STUB_MULTI_RC="0" if multi_accepted else "1", apps=APP, modes="swarm", - keep="true", + keep=keep, disable=disable, ) done = subprocess.run( @@ -81,9 +98,7 @@ def swarm_network(disable: str, *, multi_accepted: bool) -> str: ) if done.returncode != 0: raise AssertionError(f"roundtrip failed: {done.stderr}") - calls = log.read_text().splitlines() # nocheck: cache-read stub call log - zombie = next(call for call in calls if " swarm-zombie " in call) - return re.search(r"(?:^| )network=(\S*)", zombie).group(1) + return log.read_text().splitlines() # nocheck: cache-read stub call log class TestRoundtripMarker(unittest.TestCase): @@ -140,5 +155,21 @@ class TestRoundtripSwarmNetwork(unittest.TestCase): self.assertEqual(swarm_network("tor-smtp", multi_accepted=True), "multi") +class TestRoundtripSwarmRelease(unittest.TestCase): + def test_an_empty_keep_reaches_swarm_zombie_which_releases(self) -> None: + calls = swarm_make_calls("", multi_accepted=False, keep="") + zombie = next(call for call in calls if " swarm-zombie " in call) + self.assertRegex(zombie, r"(?:^| )keep=(?: |$)") + + def test_keep_true_reaches_swarm_zombie(self) -> None: + calls = swarm_make_calls("", multi_accepted=False, keep="true") + zombie = next(call for call in calls if " swarm-zombie " in call) + self.assertRegex(zombie, r"(?:^| )keep=true(?: |$)") + + def test_the_roundtrip_runs_no_release_of_its_own(self) -> None: + calls = swarm_make_calls("", multi_accepted=False, keep="") + self.assertFalse([call for call in calls if " swarm-down " in call], calls) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/python/utils/env/handlers/infinito/test_cache_host_pacman_path.py b/tests/unit/python/utils/env/handlers/infinito/test_cache_host_pacman_path.py new file mode 100644 index 0000000000..912861d763 --- /dev/null +++ b/tests/unit/python/utils/env/handlers/infinito/test_cache_host_pacman_path.py @@ -0,0 +1,83 @@ +"""Unit tests for :mod:`utils.env.handlers.infinito.cache.host_pacman_path`.""" + +from __future__ import annotations + +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from utils.env.builder import BuildContext, EnvBuilder +from utils.env.handlers.infinito.cache import host_pacman_path as handler + +HELPER = "utils.env.handlers.infinito.cache.host_pacman_path.run_helper" +ARCH = 'NAME="Arch Linux"\nID=arch\n' +MANJARO = 'NAME="Manjaro Linux"\nID=manjaro\nID_LIKE=arch\n' +ENDEAVOUR = 'NAME="EndeavourOS"\nID="endeavouros"\nID_LIKE="arch"\n' + + +@patch.dict(os.environ, {handler.KEY: ""}, clear=False) +class TestHostPacmanPath(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + self.cache = self.tmp / "pkg" + self.cache.mkdir() + + def tearDown(self) -> None: + self._tmp.cleanup() + + def _apply(self, os_release: str | None, distro: str, conf_output: str) -> str: + path = self.tmp / "os-release" + if os_release is not None: + path.write_text(os_release) + eb = EnvBuilder() + eb.set("INFINITO_DISTRO", distro) + ctx = BuildContext( + static={}, + static_comments={}, + repo_root=Path("/repo"), + on_gha=False, + on_act=False, + ) + with ( + patch.object(handler, "OS_RELEASE", path), + patch(HELPER, return_value=conf_output), + ): + handler.apply(eb, ctx) + return eb.values[handler.KEY] + + def test_an_arch_host_offers_its_first_cache_dir_to_an_arch_stack(self) -> None: + self.assertEqual( + self._apply(ARCH, "arch", f"{self.cache}/\n/var/cache/other/\n"), + str(self.cache), + ) + + def test_a_manjaro_host_offers_nothing(self) -> None: + self.assertEqual(self._apply(MANJARO, "arch", f"{self.cache}/\n"), "") + + def test_an_arch_derivative_offers_nothing(self) -> None: + self.assertEqual(self._apply(ENDEAVOUR, "arch", f"{self.cache}/\n"), "") + + def test_a_host_without_os_release_offers_nothing(self) -> None: + self.assertEqual(self._apply(None, "arch", f"{self.cache}/\n"), "") + + def test_a_stack_without_pacman_gets_nothing(self) -> None: + self.assertEqual(self._apply(ARCH, "debian", f"{self.cache}/\n"), "") + + def test_an_arch_host_without_pacman_conf_offers_nothing(self) -> None: + self.assertEqual(self._apply(ARCH, "arch", ""), "") + + def test_a_cache_dir_that_does_not_exist_is_not_offered(self) -> None: + self.assertEqual(self._apply(ARCH, "arch", f"{self.cache}/missing/\n"), "") + + def test_an_exported_path_wins(self) -> None: + with patch.dict(os.environ, {handler.KEY: "/elsewhere"}, clear=False): + self.assertEqual( + self._apply(MANJARO, "arch", f"{self.cache}/\n"), "/elsewhere" + ) + + +if __name__ == "__main__": # pragma: no cover + unittest.main() diff --git a/utils/env/handlers/__init__.py b/utils/env/handlers/__init__.py index fff173968d..90797e43b2 100644 --- a/utils/env/handlers/__init__.py +++ b/utils/env/handlers/__init__.py @@ -37,6 +37,7 @@ from .infinito import ( from .infinito.ca import bundle_candidates as ca_bundle_candidates from .infinito.ca import cert_host as ca_cert_host from .infinito.cache import conf as cache_conf +from .infinito.cache import host_pacman_path as cache_host_pacman_path from .infinito.cache import hosts as cache_hosts from .infinito.cache import http_hosts as cache_http_hosts from .infinito.cache import paths as cache_paths @@ -103,6 +104,7 @@ ORDERED_HANDLERS = [ cache_registries, cache_stack, cache_paths, + cache_host_pacman_path, ghcr_mirror, tor_socks_port, is_wsl2, diff --git a/utils/env/handlers/infinito/cache/host_pacman_path.py b/utils/env/handlers/infinito/cache/host_pacman_path.py new file mode 100644 index 0000000000..dc4482cc8c --- /dev/null +++ b/utils/env/handlers/infinito/cache/host_pacman_path.py @@ -0,0 +1,58 @@ +"""INFINITO_CACHE_HOST_PACMAN_PATH: the pacman cache of an Arch Linux host, for an arch stack to read at the same path.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from utils.distros import distro_family +from utils.env.parser import parse_static_env +from utils.env.runtime import run_helper + +if TYPE_CHECKING: + from utils.env.builder import BuildContext, EnvBuilder + +KEY = "INFINITO_CACHE_HOST_PACMAN_PATH" +COMMENT = "Pacman cache of an Arch Linux host an arch stack mounts read-only at the same path; empty on every other host or stack." +PACMAN_FAMILY = "Archlinux" +ARCH_HOST_ID = "arch" +OS_RELEASE = Path("/etc/os-release") + + +def host_id(os_release: Path) -> str: + """Return the ``ID`` an os-release file declares. + + Args: + os_release: the host's os-release file. + + Returns: + The lowercased id, empty when the file is missing or declares none. + """ + if not os_release.is_file(): + return "" + return parse_static_env(os_release).get("ID", "").strip().lower() + + +def host_cache(repo_root: Path) -> str: + """Return the first ``CacheDir`` the host's ``pacman-conf`` reports. + + Args: + repo_root: working directory for the ``pacman-conf`` call. + + Returns: + The directory without its trailing slash, empty when the host has no + ``pacman-conf`` or the directory does not exist. + """ + dirs = run_helper(["pacman-conf", "CacheDir"], repo_root).splitlines() + first = dirs[0].strip().rstrip("/") if dirs else "" + return first if first and Path(first).is_dir() else "" + + +def apply(eb: EnvBuilder, ctx: BuildContext) -> None: + pacman_stack = distro_family().get(eb.get("INFINITO_DISTRO")) == PACMAN_FAMILY + arch_host = host_id(OS_RELEASE) == ARCH_HOST_ID + eb.setdefault( + KEY, + host_cache(ctx.repo_root) if pacman_stack and arch_host else "", + comment=COMMENT, + ) diff --git a/utils/env/handlers/infinito/cache/paths.py b/utils/env/handlers/infinito/cache/paths.py index acc5696a0f..3d3dee5c6d 100644 --- a/utils/env/handlers/infinito/cache/paths.py +++ b/utils/env/handlers/infinito/cache/paths.py @@ -3,10 +3,11 @@ Every one of them sits below the cache base the paths SPOT defines (``group_vars/all/05_paths.yml`` ``DIR_CACHE``), so the base is written once and each service names only the leg below it. ``make clean-cache`` removes -three of these entries, not the base. +four of these entries, not the base. Keys set here: the Nexus blobstore, the frontend's CA directory and file, its -TLS certificates, and the registry mirror's data and CA. +TLS certificates, the registry mirror's data and CA, and the shared package and +tool cache folder. """ from __future__ import annotations @@ -50,6 +51,11 @@ ENTRIES: tuple[tuple[str, tuple[str, ...], str], ...] = ( ("registry", "mirror"), "Host directory backing the registry-cache image mirror.", ), + ( + "INFINITO_CACHE_SHARED_HOST_PATH", + ("shared",), + "Host directory every compose slot, swarm lab node and GUI lane mounts at the same path for package and tool caches.", + ), ) diff --git a/utils/env/handlers/passthrough.py b/utils/env/handlers/passthrough.py index 80cd0b4c25..b0c4124c80 100644 --- a/utils/env/handlers/passthrough.py +++ b/utils/env/handlers/passthrough.py @@ -135,6 +135,8 @@ STATIC_KEYS = ( "INFINITO_CACHE_REGISTRY_HOST_PATH", "INFINITO_CACHE_REGISTRY_CA_HOST_PATH", "INFINITO_CACHE_REGISTRY_IP", + "INFINITO_CACHE_SHARED_HOST_PATH", + "INFINITO_CACHE_HOST_PACMAN_PATH", "INFINITO_CACHE_PACKAGE_CONTAINER", "INFINITO_CACHE_PACKAGE_EXEC_MODE", "INFINITO_CACHE_PACKAGE_HOST_PATH",