mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-07 19:32:17 +00:00
feat(cache): declare every cached upstream once and generate the rest
The package cache was configured in six places at once: the nginx upstream map, the consumer compose override, pip.conf, .npmrc, the apt sources and the twenty ensure_proxy_repo calls that bootstrap Nexus each spelled the same hostnames, repository names and suites by hand. A new upstream meant editing five files and hoping none was missed, and the docs told contributors to do exactly that. A role now declares what it needs under cache: in its meta/networks.yml, and everything else is derived from it. svc-cache-package holds the general upstreams; web-svc-libretranslate holds its own. All nine derived files are generated and gitignored, so a second spelling of a hostname cannot exist. utils.cache.hosts reads the declarations without PyYAML, because the .env generator reaches them before any dependency is installed; it raises rather than truncate when a comment sits at column 0 inside the block, which would otherwise drop every declaration below it in silence. utils.cache.records owns the Nexus record format and the Ubuntu mirror resolution, so the dev stack and the role provision from one rendering. A build may only resolve a cached host it can verify: the plain-HTTP hosts are hijacked for every image, and the TLS hosts only for an image whose Dockerfile installs the frontend CA. Hijacking all of them would fail every pip and npm build with CERTIFICATE_VERIFY_FAILED. svc-cache-package deploys the cache and its TLS frontend by calling the same shell scripts the dev stack calls, rather than reimplementing them as tasks. The role is invokable through the new svc-cache category and inert until a host carries it. Verified: the twenty Nexus payloads, the fifteen nginx server blocks and all seven package-manager configs are byte-identical to the hand-written files they replace, and the new category adds one role to the deploy order while preserving the relative order of the other 180. The role has not been deployed yet.
This commit is contained in:
kevinveenbirkenbach committed
1 parent
666312dfbd
commit
230d9a1e5e
66 files changed
+2066
-574
No files matched your search
@@ -39,6 +39,19 @@ inventories/production/
|
||||
compose/coredns/Corefile
|
||||
compose/Corefile
|
||||
|
||||
# Rendered by `make dotenv` from the cache: declarations in
|
||||
# roles/*/meta/networks.yml. Tracking any of these would put a second,
|
||||
# drifting spelling of a cached host, suite or upstream into the repository.
|
||||
compose/package-cache-frontend/upstreams.conf
|
||||
compose.cache-consumer.yml
|
||||
compose/package-cache/pip.conf
|
||||
compose/package-cache/npmrc
|
||||
compose/package-cache/apt/*.list
|
||||
|
||||
# The i18n lane pins load_only to the languages a run translates, so the image
|
||||
# bakes exactly those argos models instead of installing every one at runtime.
|
||||
inventories/development/i18n.rendered.yml
|
||||
|
||||
# Host shell / editor / tooling state accidentally surfaced inside this working copy
|
||||
.ansible
|
||||
.bash_profile
|
||||
|
||||
+13
@@ -39,6 +39,19 @@ inventories/production/
|
||||
compose/coredns/Corefile
|
||||
compose/Corefile
|
||||
|
||||
# Rendered by `make dotenv` from the cache: declarations in
|
||||
# roles/*/meta/networks.yml. Tracking any of these would put a second,
|
||||
# drifting spelling of a cached host, suite or upstream into the repository.
|
||||
compose/package-cache-frontend/upstreams.conf
|
||||
compose.cache-consumer.yml
|
||||
compose/package-cache/pip.conf
|
||||
compose/package-cache/npmrc
|
||||
compose/package-cache/apt/*.list
|
||||
|
||||
# The i18n lane pins load_only to the languages a run translates, so the image
|
||||
# bakes exactly those argos models instead of installing every one at runtime.
|
||||
inventories/development/i18n.rendered.yml
|
||||
|
||||
# Host shell / editor / tooling state accidentally surfaced inside this working copy
|
||||
.ansible
|
||||
.bash_profile
|
||||
|
||||
@@ -286,6 +286,7 @@ diagnose-network:
|
||||
# Note: runtime context covers distro, cache sizes, secrets, and the like.
|
||||
dotenv:
|
||||
@python3 -m cli.meta.env
|
||||
@python3 -m cli.meta.cache
|
||||
|
||||
.PHONY: dotenv-force
|
||||
# Force a clean .env regeneration in a stripped environment.
|
||||
@@ -295,6 +296,7 @@ dotenv-force:
|
||||
@env -i HOME="$${HOME}" PATH="$${PATH}" \
|
||||
INFINITO_CACHE_CONF_SOURCE="$${INFINITO_CACHE_CONF_SOURCE:-}" \
|
||||
python3 -m cli.meta.env
|
||||
@python3 -m cli.meta.cache
|
||||
|
||||
.PHONY: environment-bootstrap
|
||||
# Bootstrap the local development environment.
|
||||
|
||||
@@ -3,7 +3,12 @@ from __future__ import annotations
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
from pathlib import Path
|
||||
|
||||
from utils.cache.render import ROLE_DIR as CACHE_ROLE_DIR
|
||||
from utils.cache.render import render_artifacts
|
||||
from utils.env.handlers.infinito.cache.conf import KEY as CACHE_CONF_KEY
|
||||
from utils.env.handlers.infinito.cache.conf import RELATIVE as CACHE_CONF
|
||||
|
||||
from .coredns import CoreDNSCorefileRenderer
|
||||
from .env import compose_file_args
|
||||
@@ -11,8 +16,7 @@ from .network import detect_outer_network_mtu
|
||||
from .proc import run_streaming
|
||||
from .profile import Profile
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
CACHE_ROLE_SHELL_DIR = CACHE_ROLE_DIR / "files" / "shell"
|
||||
|
||||
|
||||
class Compose:
|
||||
@@ -110,7 +114,7 @@ class Compose:
|
||||
|
||||
def _bootstrap_package_cache(self, env: dict[str, str]) -> None:
|
||||
"""Run the host-side Nexus bootstrap helper. Idempotent."""
|
||||
helper = self.repo_root / "scripts" / "docker" / "cache" / "package.sh"
|
||||
helper = CACHE_ROLE_SHELL_DIR / "bootstrap.sh"
|
||||
print(">>> Bootstrapping package-cache proxy repos")
|
||||
r = subprocess.run(
|
||||
[str(helper)],
|
||||
@@ -127,14 +131,7 @@ class Compose:
|
||||
|
||||
def _generate_package_frontend_certs(self, env: dict[str, str]) -> None:
|
||||
"""Generate frontend CA + leaf certs before nginx starts."""
|
||||
helper = (
|
||||
self.repo_root
|
||||
/ "scripts"
|
||||
/ "docker"
|
||||
/ "cache"
|
||||
/ "package-frontend"
|
||||
/ "certs.sh"
|
||||
)
|
||||
helper = CACHE_ROLE_SHELL_DIR / "certs.sh"
|
||||
print(">>> Generating package-cache-frontend CA + per-hostname certs")
|
||||
subprocess.run(
|
||||
[str(helper)],
|
||||
@@ -169,7 +166,22 @@ class Compose:
|
||||
f"size={out.stat().st_size if out.exists() else 'n/a'}"
|
||||
)
|
||||
|
||||
def _render_cache_artifacts(self) -> None:
|
||||
"""Render the generated cache files the overrides mount.
|
||||
|
||||
`make dotenv` renders them too, but `scripts/meta/env/load.sh` only
|
||||
auto-generates `.env` — it runs on the bare bootstrap python, which
|
||||
has no Jinja. Without this, a checkout that never ran `make dotenv`
|
||||
reaches `extends: compose.cache-consumer.yml` with the file absent.
|
||||
"""
|
||||
conf = Path(os.environ.get(CACHE_CONF_KEY, "") or (self.repo_root / CACHE_CONF))
|
||||
for path in render_artifacts(self.repo_root, conf):
|
||||
print(f"[compose] cache file generated at: {path}")
|
||||
|
||||
def up(self, *, run_entry_init: bool = True) -> None:
|
||||
print(">>> Rendering the derived cache files from the cache: declarations")
|
||||
self._render_cache_artifacts()
|
||||
|
||||
print(">>> Rendering CoreDNS Corefile from template")
|
||||
self._render_coredns_corefile()
|
||||
|
||||
|
||||
Vendored
+14
@@ -0,0 +1,14 @@
|
||||
"""`infinito meta cache` subcommand -- render the derived cache files.
|
||||
|
||||
The nginx upstream map, the consumer compose override and the package
|
||||
managers' client configuration all follow from the ``cache:`` declarations
|
||||
in the roles' `meta/networks.yml`. None of them is tracked; this package
|
||||
regenerates them. Implementation lives in `utils.cache.render`.
|
||||
|
||||
It is deliberately separate from `cli.meta.env`, which has to import on the
|
||||
bare bootstrap python and therefore cannot reach Jinja or PyYAML.
|
||||
"""
|
||||
|
||||
from utils.cache import PROJECT_ROOT
|
||||
|
||||
__all__ = ["PROJECT_ROOT"]
|
||||
Vendored
+37
@@ -0,0 +1,37 @@
|
||||
"""CLI entry: `python -m cli.meta.cache` (run by `make dotenv`)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from utils.cache.render import render_artifacts
|
||||
from utils.env.handlers.infinito.cache.conf import KEY, RELATIVE, primary_checkout
|
||||
|
||||
from . import PROJECT_ROOT as REPO_ROOT
|
||||
|
||||
|
||||
def _conf_path() -> Path:
|
||||
"""Return where the frontend's upstream map belongs.
|
||||
|
||||
`.env` carries it once `make dotenv` has run; before that, fall back to
|
||||
the same checkout the handler would have chosen.
|
||||
"""
|
||||
configured = os.environ.get(KEY, "").strip()
|
||||
if configured:
|
||||
return Path(configured)
|
||||
return primary_checkout(REPO_ROOT) / RELATIVE
|
||||
|
||||
|
||||
def main() -> int:
|
||||
written = render_artifacts(REPO_ROOT, _conf_path())
|
||||
print(
|
||||
f"Rendered {len(written)} cache file(s) from the cache: declarations",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,39 +0,0 @@
|
||||
---
|
||||
# Never instantiated: every runner pulls this in through `extends`, which
|
||||
# resolves the relative sources below against THIS file's directory.
|
||||
|
||||
services:
|
||||
cache-consumer:
|
||||
environment:
|
||||
INFINITO_CACHE_PACKAGE_FRONTEND_IP: ${INFINITO_CACHE_PACKAGE_FRONTEND_IP:?Run 'make dotenv' to generate the .env single source of truth}
|
||||
extra_hosts:
|
||||
- "infinito-registry-cache:${INFINITO_CACHE_REGISTRY_IP:?Run 'make dotenv' to generate the .env single source of truth}"
|
||||
- "deb.debian.org:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
|
||||
- "archive.ubuntu.com:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
|
||||
- "security.ubuntu.com:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
|
||||
- "pypi.org:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
|
||||
- "files.pythonhosted.org:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
|
||||
- "registry.npmjs.org:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
|
||||
- "rubygems.org:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
|
||||
- "index.rubygems.org:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
|
||||
- "repo.packagist.org:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
|
||||
- "proxy.golang.org:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
|
||||
- "dl-cdn.alpinelinux.org:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
|
||||
- "raw.githubusercontent.com:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
|
||||
- "codeload.github.com:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
|
||||
- "argos-net.com:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ${INFINITO_CACHE_REGISTRY_CA_HOST_PATH:?Run 'make dotenv' to generate the .env single source of truth}
|
||||
target: /opt/registry-cache-ca
|
||||
read_only: true
|
||||
bind:
|
||||
create_host_path: true
|
||||
- ${INFINITO_CACHE_REGISTRY_PROXY_CONF:?Set by 'cli deploy development up'}:/etc/systemd/system/docker.service.d/registry-cache-proxy.conf:ro
|
||||
- ./scripts/docker/cache/registry-ca.sh:/usr/local/bin/registry-cache-ca.sh:ro
|
||||
- ${INFINITO_CACHE_PACKAGE_PIP_CONF:?Set by 'cli deploy development up'}:/etc/pip.conf:ro
|
||||
- ${INFINITO_CACHE_PACKAGE_NPMRC:?Set by 'cli deploy development up'}:/root/.npmrc:ro
|
||||
- ./compose/package-cache/apt/${INFINITO_DISTRO:?Run 'make dotenv' to generate the .env single source of
|
||||
truth}.list:/etc/apt/sources.list.d/package-cache.list:ro
|
||||
- ${INFINITO_CACHE_PACKAGE_FRONTEND_CA_FILE:?Set by 'cli deploy development up'}:/opt/package-frontend-ca.crt:ro
|
||||
- ./scripts/docker/cache/package-frontend/ca.sh:/usr/local/bin/package-frontend-ca.sh:ro
|
||||
@@ -37,7 +37,7 @@ services:
|
||||
|
||||
package-cache:
|
||||
image: sonatype/nexus3:3.91.1-ubi
|
||||
container_name: infinito-package-cache
|
||||
container_name: ${INFINITO_CACHE_PACKAGE_CONTAINER:?Run 'make dotenv' to generate the .env single source of truth}
|
||||
# Exception: fresh bind-mount is root:root and Nexus aborts unless run as root.
|
||||
user: "0:0"
|
||||
environment:
|
||||
|
||||
@@ -1,213 +0,0 @@
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Accept-Encoding "";
|
||||
proxy_buffering on;
|
||||
proxy_request_buffering off;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_connect_timeout 30s;
|
||||
|
||||
# Docker embedded DNS, pinned to bypass image-specific resolv.conf order.
|
||||
resolver 127.0.0.11 ipv6=off valid=30s;
|
||||
|
||||
# Nexus answers 502 when a proxy's remote is unreachable (measured) and has no
|
||||
# group repository type for apt, so the mirror failover sits here rather than in
|
||||
# the repo layout. 5xx only: a 404 means the file genuinely is not in the suite,
|
||||
# and apt probes for missing files often enough that retrying those would double
|
||||
# the request rate.
|
||||
#
|
||||
# Nexus answers that 502 only after trying every address of the remote in turn,
|
||||
# one connect timeout each; for a remote with several addresses that outlasts a
|
||||
# client's own timeout before the mirror is ever asked.
|
||||
#
|
||||
# These stay prefix locations. A regex location forfeits nginx's implicit
|
||||
# `/debian` -> `/debian/` redirect, and the URL-reachability test probes exactly
|
||||
# that slash-less form through the runner's DNS hijack, so it turns into a 404.
|
||||
server {
|
||||
listen 80;
|
||||
server_name deb.debian.org;
|
||||
location /debian/ {
|
||||
rewrite ^/debian/(.*)$ /repository/apt-debian/$1 break;
|
||||
proxy_intercept_errors on;
|
||||
proxy_read_timeout 8s;
|
||||
error_page 502 503 504 = @debian_mirror;
|
||||
proxy_pass http://package-cache:8081;
|
||||
}
|
||||
location @debian_mirror {
|
||||
rewrite ^/repository/apt-debian/(.*)$ /repository/apt-debian-mirror/$1 break;
|
||||
proxy_pass http://package-cache:8081;
|
||||
}
|
||||
location /debian-security/ {
|
||||
rewrite ^/debian-security/(.*)$ /repository/apt-debian-security/$1 break;
|
||||
proxy_intercept_errors on;
|
||||
proxy_read_timeout 8s;
|
||||
error_page 502 503 504 = @debian_security_mirror;
|
||||
proxy_pass http://package-cache:8081;
|
||||
}
|
||||
location @debian_security_mirror {
|
||||
rewrite ^/repository/apt-debian-security/(.*)$ /repository/apt-debian-security-mirror/$1 break;
|
||||
proxy_pass http://package-cache:8081;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name archive.ubuntu.com;
|
||||
location /ubuntu/ {
|
||||
rewrite ^/ubuntu/(.*)$ /repository/apt-ubuntu/$1 break;
|
||||
proxy_intercept_errors on;
|
||||
proxy_read_timeout 8s;
|
||||
error_page 502 503 504 = @ubuntu_mirror;
|
||||
proxy_pass http://package-cache:8081;
|
||||
}
|
||||
location @ubuntu_mirror {
|
||||
rewrite ^/repository/apt-ubuntu/(.*)$ /repository/apt-ubuntu-mirror/$1 break;
|
||||
proxy_pass http://package-cache:8081;
|
||||
}
|
||||
}
|
||||
server {
|
||||
listen 80;
|
||||
server_name security.ubuntu.com;
|
||||
location /ubuntu/ {
|
||||
rewrite ^/ubuntu/(.*)$ /repository/apt-ubuntu-security/$1 break;
|
||||
proxy_intercept_errors on;
|
||||
proxy_read_timeout 8s;
|
||||
error_page 502 503 504 = @ubuntu_security_mirror;
|
||||
proxy_pass http://package-cache:8081;
|
||||
}
|
||||
location @ubuntu_security_mirror {
|
||||
rewrite ^/repository/apt-ubuntu-security/(.*)$ /repository/apt-ubuntu-security-mirror/$1 break;
|
||||
proxy_pass http://package-cache:8081;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name dl-cdn.alpinelinux.org;
|
||||
location /alpine/ {
|
||||
rewrite ^/alpine/(.*)$ /repository/raw-alpine/$1 break;
|
||||
proxy_pass http://package-cache:8081;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name pypi.org;
|
||||
ssl_certificate /etc/nginx/certs/pypi.org.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/pypi.org.key;
|
||||
location / {
|
||||
proxy_pass http://package-cache:8081/repository/pypi-proxy/;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name files.pythonhosted.org;
|
||||
ssl_certificate /etc/nginx/certs/files.pythonhosted.org.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/files.pythonhosted.org.key;
|
||||
location / {
|
||||
proxy_pass http://package-cache:8081/repository/pypi-proxy/;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name registry.npmjs.org;
|
||||
ssl_certificate /etc/nginx/certs/registry.npmjs.org.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/registry.npmjs.org.key;
|
||||
# Tarball URLs Nexus emits already carry /repository/npm-proxy/; pass them
|
||||
# through as-is (no path on proxy_pass) so they aren't double-prefixed → 404.
|
||||
location /repository/npm-proxy/ {
|
||||
proxy_pass http://package-cache:8081;
|
||||
}
|
||||
location / {
|
||||
proxy_pass http://package-cache:8081/repository/npm-proxy/;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name rubygems.org;
|
||||
ssl_certificate /etc/nginx/certs/rubygems.org.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/rubygems.org.key;
|
||||
location / {
|
||||
proxy_pass http://package-cache:8081/repository/gem-proxy/;
|
||||
}
|
||||
}
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name index.rubygems.org;
|
||||
ssl_certificate /etc/nginx/certs/index.rubygems.org.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/index.rubygems.org.key;
|
||||
location / {
|
||||
proxy_pass http://package-cache:8081/repository/gem-proxy/;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name repo.packagist.org;
|
||||
ssl_certificate /etc/nginx/certs/repo.packagist.org.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/repo.packagist.org.key;
|
||||
location / {
|
||||
proxy_pass http://package-cache:8081/repository/raw-packagist/;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name proxy.golang.org;
|
||||
ssl_certificate /etc/nginx/certs/proxy.golang.org.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/proxy.golang.org.key;
|
||||
location / {
|
||||
proxy_pass http://package-cache:8081/repository/go-proxy/;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name dl-cdn.alpinelinux.org;
|
||||
ssl_certificate /etc/nginx/certs/dl-cdn.alpinelinux.org.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/dl-cdn.alpinelinux.org.key;
|
||||
location / {
|
||||
proxy_pass http://package-cache:8081/repository/raw-alpine/;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name raw.githubusercontent.com;
|
||||
ssl_certificate /etc/nginx/certs/raw.githubusercontent.com.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/raw.githubusercontent.com.key;
|
||||
location / {
|
||||
proxy_pass http://package-cache:8081/repository/raw-githubusercontent/;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name codeload.github.com;
|
||||
ssl_certificate /etc/nginx/certs/codeload.github.com.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/codeload.github.com.key;
|
||||
location / {
|
||||
proxy_pass http://package-cache:8081/repository/raw-codeload-github/;
|
||||
}
|
||||
}
|
||||
|
||||
# Nexus emits no byte of an Argos model until it has fetched and stored the
|
||||
# whole 70-150 MB object, so the wait for the response header is the wait for
|
||||
# the entire cold fetch. Measured: 46-88s uncontended, and `504 ups=504
|
||||
# upstime=300.101` once eight lanes compete. The global 300s stays where the
|
||||
# apt mirror failover needs it.
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name argos-net.com;
|
||||
ssl_certificate /etc/nginx/certs/argos-net.com.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/argos-net.com.key;
|
||||
location / {
|
||||
proxy_read_timeout 1800s;
|
||||
proxy_pass http://package-cache:8081/repository/raw-argos-net/;
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
# Empty on purpose: Arch uses pacman, which the package-cache does not
|
||||
# proxy. The bind mount is unconditional in the compose overrides, and
|
||||
# docker would create a host directory here if the source were missing.
|
||||
@@ -1,4 +0,0 @@
|
||||
# Empty on purpose: CentOS uses dnf, which reaches the package-cache
|
||||
# through `yum-rocky`, not through apt. The bind mount is unconditional
|
||||
# in the compose overrides, and docker would create a host directory
|
||||
# here if the source were missing.
|
||||
@@ -1,11 +0,0 @@
|
||||
# apt sources for the Debian runner: routes apt-get through the
|
||||
# package-cache (Sonatype Nexus 3 OSS) pull-through proxy.
|
||||
#
|
||||
# Real mirror hostname, not the `package-cache` service name, which
|
||||
# coredns — the container's resolver — cannot resolve; the compose
|
||||
# overrides map the hostname to the cache frontend via extra_hosts.
|
||||
#
|
||||
# One file per INFINITO_DISTRO: apt fails the whole `apt-get update`
|
||||
# when a single source is unreachable, so a Debian runner must never
|
||||
# carry the Ubuntu entry.
|
||||
deb [trusted=yes] http://deb.debian.org/debian bookworm main
|
||||
@@ -1,4 +0,0 @@
|
||||
# Empty on purpose: Fedora uses dnf, which reaches the package-cache
|
||||
# through `yum-fedora`, not through apt. The bind mount is unconditional
|
||||
# in the compose overrides, and docker would create a host directory
|
||||
# here if the source were missing.
|
||||
@@ -1,11 +0,0 @@
|
||||
# apt sources for the Ubuntu runner: routes apt-get through the
|
||||
# package-cache (Sonatype Nexus 3 OSS) pull-through proxy.
|
||||
#
|
||||
# Real mirror hostname, not the `package-cache` service name, which
|
||||
# coredns — the container's resolver — cannot resolve; the compose
|
||||
# overrides map the hostname to the cache frontend via extra_hosts.
|
||||
#
|
||||
# One file per INFINITO_DISTRO: apt fails the whole `apt-get update`
|
||||
# when a single source is unreachable, so an Ubuntu runner must never
|
||||
# carry the Debian entry.
|
||||
deb [trusted=yes] http://archive.ubuntu.com/ubuntu jammy main
|
||||
@@ -1,5 +0,0 @@
|
||||
# npm → cache frontend via registry.npmjs.org (mapped by extra_hosts);
|
||||
# not `package-cache`, which coredns can't resolve. strict-ssl=false
|
||||
# accepts the frontend's self-signed cert.
|
||||
registry=https://registry.npmjs.org/
|
||||
strict-ssl=false
|
||||
@@ -1,8 +0,0 @@
|
||||
# pip → cache frontend via pypi.org/files.pythonhosted.org (mapped by
|
||||
# extra_hosts); not `package-cache`, which coredns can't resolve.
|
||||
# trusted-host skips the frontend's self-signed cert.
|
||||
[global]
|
||||
index-url = https://pypi.org/simple/
|
||||
trusted-host =
|
||||
pypi.org
|
||||
files.pythonhosted.org
|
||||
@@ -0,0 +1,16 @@
|
||||
{#
|
||||
Rendered from the cache: declarations, one file per distribution. apt fails
|
||||
the whole `apt-get update` when a single source is unreachable, so a runner
|
||||
must never carry another distribution's entry.
|
||||
|
||||
The source names the real mirror, not the service name, which coredns, the
|
||||
container's resolver, cannot resolve; the compose overrides map the
|
||||
hostname to the frontend via extra_hosts.
|
||||
|
||||
A distribution that does not install through apt renders an empty file. The
|
||||
bind mount is unconditional in the compose overrides, and docker would
|
||||
create a host directory here if the source were missing.
|
||||
#}
|
||||
{% if CACHE_PACKAGE_APT_SOURCE %}
|
||||
deb [trusted=yes] {{ CACHE_PACKAGE_APT_SOURCE }} {{ CACHE_PACKAGE_APT_SUITE }} main
|
||||
{% endif %}
|
||||
@@ -0,0 +1,33 @@
|
||||
{#
|
||||
Rendered from the cache: declarations, which are the single point of truth
|
||||
for every cached host. Nothing here is written by hand; edit the
|
||||
declaration instead.
|
||||
|
||||
The rendered file is never instantiated: every runner pulls it in through
|
||||
`extends`, which resolves the relative sources below against the rendered
|
||||
file's own directory, so it must stay at the repository root.
|
||||
#}
|
||||
---
|
||||
services:
|
||||
cache-consumer:
|
||||
environment:
|
||||
INFINITO_CACHE_PACKAGE_FRONTEND_IP: ${INFINITO_CACHE_PACKAGE_FRONTEND_IP:?Run 'make dotenv' to generate the .env single source of truth}
|
||||
extra_hosts:
|
||||
- "infinito-registry-cache:${INFINITO_CACHE_REGISTRY_IP:?Run 'make dotenv' to generate the .env single source of truth}"
|
||||
{% for hostname in CACHE_PACKAGE_HOSTNAMES %}
|
||||
- "{{ hostname }}:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
|
||||
{% endfor %}
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ${INFINITO_CACHE_REGISTRY_CA_HOST_PATH:?Run 'make dotenv' to generate the .env single source of truth}
|
||||
target: /opt/registry-cache-ca
|
||||
read_only: true
|
||||
bind:
|
||||
create_host_path: true
|
||||
- ${INFINITO_CACHE_REGISTRY_PROXY_CONF:?Set by 'cli deploy development up'}:/etc/systemd/system/docker.service.d/registry-cache-proxy.conf:ro
|
||||
- ./scripts/docker/cache/registry-ca.sh:/usr/local/bin/registry-cache-ca.sh:ro
|
||||
- ${INFINITO_CACHE_PACKAGE_PIP_CONF:?Set by 'cli deploy development up'}:/etc/pip.conf:ro
|
||||
- ${INFINITO_CACHE_PACKAGE_NPMRC:?Set by 'cli deploy development up'}:/root/.npmrc:ro
|
||||
- ./compose/package-cache/apt/${INFINITO_DISTRO:?Run 'make dotenv' to generate the .env single source of truth}.list:/etc/apt/sources.list.d/package-cache.list:ro
|
||||
- ${INFINITO_CACHE_PACKAGE_FRONTEND_CA_FILE:?Set by 'cli deploy development up'}:/opt/package-frontend-ca.crt:ro
|
||||
- ./roles/svc-cache-package/files/shell/ca.sh:/usr/local/bin/package-frontend-ca.sh:ro
|
||||
@@ -0,0 +1,8 @@
|
||||
{#
|
||||
Rendered from the cache: declarations. npm reaches the frontend under the
|
||||
real registry hostname, which extra_hosts maps to it; the service name
|
||||
would not resolve, because coredns does not answer for it. strict-ssl=false
|
||||
accepts the frontend's self-signed certificate.
|
||||
#}
|
||||
registry={{ CACHE_PACKAGE_NPM_REGISTRY }}
|
||||
strict-ssl=false
|
||||
@@ -0,0 +1,12 @@
|
||||
{#
|
||||
Rendered from the cache: declarations. pip reaches the frontend under the
|
||||
real PyPI hostnames, which extra_hosts maps to it; the service name would
|
||||
not resolve, because coredns does not answer for it. trusted-host skips the
|
||||
frontend's self-signed certificate.
|
||||
#}
|
||||
[global]
|
||||
index-url = {{ CACHE_PACKAGE_PYPI_INDEX }}
|
||||
trusted-host =
|
||||
{% for host in CACHE_PACKAGE_PYPI_HOSTS %}
|
||||
{{ host }}
|
||||
{% endfor %}
|
||||
@@ -61,6 +61,9 @@ INFINITO_CACHE_CONF_SOURCE=primary
|
||||
# External docker network carrying the shared cache stack; empty on the primary instance, which owns the stack itself.
|
||||
INFINITO_CACHE_NETWORK=
|
||||
|
||||
# Name of the Nexus container; the bootstrap reaches it by this name.
|
||||
INFINITO_CACHE_PACKAGE_CONTAINER=infinito-package-cache
|
||||
|
||||
# Host directory holding the package-cache frontend CA materials.
|
||||
INFINITO_CACHE_PACKAGE_FRONTEND_CA_DIR=/var/cache/infinito/core/cache/package/frontend/ca
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ A proxy repo that holds a cached copy serves it when its remote is unreachable,
|
||||
|
||||
Every apt suite exists twice, once per upstream mirror. Nexus has no group repository type for `apt` (only maven, raw, docker, yum, npm, pypi, rubygems, go and friends), so the failover lives in the frontend: a request that the primary repo answers with `502`/`503`/`504` is re-run against the `-mirror` repo, which proxies a different host. `404` deliberately does not retry, because a missing file is missing on both mirrors and apt probes for missing files often. The frontend waits at most 8 s for a primary before it re-runs the request against the mirror; the `-mirror` locations keep the global 300 s.
|
||||
|
||||
Bootstrap is idempotent and runs from [package.sh](../../../scripts/docker/cache/package.sh) once the stack is healthy.
|
||||
Bootstrap is idempotent and runs from [bootstrap.sh](../../../roles/svc-cache-package/files/shell/bootstrap.sh) once the stack is healthy. It loops over `INFINITO_CACHE_UPSTREAMS` and holds no repository of its own.
|
||||
|
||||
### Package Cache Frontend 🔐
|
||||
|
||||
@@ -38,10 +38,10 @@ Bootstrap is idempotent and runs from [package.sh](../../../scripts/docker/cache
|
||||
|
||||
Two listener layers:
|
||||
|
||||
- HTTPS (port 443): per-hostname server certs signed by a dedicated CA. Used by the `infinito` runner (Ansible-driven `pip install`, `gem install`, `composer install`, `curl https://…`). The runner trusts the CA via [package-frontend-ca.sh](../../../scripts/docker/cache/package-frontend/ca.sh).
|
||||
- HTTPS (port 443): per-hostname server certs signed by a dedicated CA. Used by the `infinito` runner (Ansible-driven `pip install`, `gem install`, `composer install`, `curl https://…`). The runner trusts the CA via [ca.sh](../../../roles/svc-cache-package/files/shell/ca.sh).
|
||||
- HTTP (port 80): plain mirrors for `deb.debian.org`, `archive.ubuntu.com`, `security.ubuntu.com`, `dl-cdn.alpinelinux.org`. Used by inner-`dockerd` Dockerfile builds via `build.extra_hosts` DNS-hijack. No CA-trust required in the build container.
|
||||
|
||||
Cert generation runs in a throw-away alpine container driven by [package-frontend-certs.sh](../../../scripts/docker/cache/package-frontend/certs.sh) before the frontend starts.
|
||||
Cert generation runs in a throw-away alpine container driven by [certs.sh](../../../roles/svc-cache-package/files/shell/certs.sh) before the frontend starts. It issues a leaf for each name in `INFINITO_CACHE_TLS_HOSTS` and holds no hostname of its own.
|
||||
|
||||
## Activation 🎚️
|
||||
|
||||
@@ -104,8 +104,9 @@ Per-variable defaults and purposes are in [compose.yml.md](../artefact/files/com
|
||||
| Start the stack with caches | `make compose-up` |
|
||||
| Stop the stack | `make compose-down` |
|
||||
| Wipe local cache state | `make clean-cache` |
|
||||
| Manually re-bootstrap Nexus repos | `bash scripts/docker/cache/package.sh` (after `make dotenv` or sourcing `scripts/meta/env/load.sh`) |
|
||||
| Manually regenerate frontend certs | `bash scripts/docker/cache/package-frontend/certs.sh` |
|
||||
| Manually re-bootstrap Nexus repos | `bash roles/svc-cache-package/files/shell/bootstrap.sh` (after `make dotenv` or sourcing `scripts/meta/env/load.sh`) |
|
||||
| Manually regenerate frontend certs | `bash roles/svc-cache-package/files/shell/certs.sh` |
|
||||
| Regenerate every derived cache file | `make dotenv` |
|
||||
| Reload nginx in the frontend | `docker exec infinito-package-cache-frontend nginx -s reload` |
|
||||
| Inspect cache hits | `docker logs -f infinito-package-cache` and `docker logs -f infinito-package-cache-frontend` |
|
||||
|
||||
@@ -115,11 +116,24 @@ Cache state persists under `/var/cache/infinito/core/cache/`. Paths are configur
|
||||
|
||||
When a new package manager or upstream needs caching:
|
||||
|
||||
1. Register a Nexus proxy repo in [package.sh](../../../scripts/docker/cache/package.sh).
|
||||
2. If the upstream uses HTTPS, add it to `HOSTNAMES` in [package-frontend-certs.sh](../../../scripts/docker/cache/package-frontend/certs.sh) so a leaf cert is issued.
|
||||
3. Add a server-block in [upstreams.conf](../../../compose/package-cache-frontend/upstreams.conf) that reverse-proxies onto the new Nexus repo path (rewrite if upstream URL prefix differs from the Nexus repo path).
|
||||
4. Add an `extra_hosts` entry on the `infinito` service in [compose/cache.override.yml](../../../compose/cache.override.yml) for runner-side traffic.
|
||||
5. If the upstream is HTTP-only and inner-`dockerd` builds need it, also add it to `_CACHE_HTTP_HOSTNAMES` in [compose.py](../../../roles/sys-svc-compose/files/python/compose.py) so the per-app `compose.cache.override.yml` includes it in `build.extra_hosts`.
|
||||
Declare it under `cache:` in a `meta/networks.yml`, then run `make dotenv`. That file is the single point of truth; the Nexus repo, the leaf cert, the nginx server block, the `extra_hosts` entries, `pip.conf`, `.npmrc` and the apt sources are all derived from it.
|
||||
|
||||
Put the declaration in the role that needs the upstream. An upstream no single role owns goes in [svc-cache-package](../../../roles/svc-cache-package/meta/networks.yml).
|
||||
|
||||
```yaml
|
||||
cache:
|
||||
repos:
|
||||
raw-example:
|
||||
flavor: raw
|
||||
upstream: https://example.org/
|
||||
hosts:
|
||||
example.org:
|
||||
repo: raw-example
|
||||
```
|
||||
|
||||
A repository is proxied by Nexus. A host is additionally answered for by the frontend, so a client keeps addressing its real upstream URL. `listen: [80]` serves it over plain HTTP and issues no certificate; `paths:` maps several repositories under one host; `mirror:` on a repository names the sibling the frontend retries against on `502`, `503` or `504`; `passthrough: true` suits an upstream that hands out URLs already carrying the repository prefix.
|
||||
|
||||
A deployed host may correct or extend any of this from its inventory under `applications.svc-cache-package.repos` and `.upstreams`; the dev stack has no inventory and uses the declarations unchanged.
|
||||
|
||||
## Background 📚
|
||||
|
||||
|
||||
@@ -260,6 +260,17 @@ roles:
|
||||
run_after: [svc-ai]
|
||||
invokable: true
|
||||
|
||||
cache:
|
||||
title: "Package Caches"
|
||||
hanzi: "缓存" # Means "cache" or "buffer".
|
||||
description: "Pull-through caches for package upstreams - proxy repositories and the TLS frontend that answers for them."
|
||||
icon: "fas fa-warehouse"
|
||||
bootstrap: true
|
||||
modes:
|
||||
compose: true
|
||||
swarm: false
|
||||
invokable: true
|
||||
|
||||
registry:
|
||||
title: "Container Registries"
|
||||
hanzi: "仓库" # Means "warehouse" or "repository".
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Lookup ``cache_records``: the repositories, as the bootstrap script reads them.
|
||||
|
||||
The same string `.env` carries as ``INFINITO_CACHE_UPSTREAMS``, so the role's
|
||||
bootstrap task and the dev stack provision Nexus from one rendering.
|
||||
`utils.cache.records` owns the format.
|
||||
|
||||
{{ lookup('cache_records') }}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from ansible.errors import AnsibleError
|
||||
from ansible.plugins.lookup import LookupBase
|
||||
|
||||
from utils.cache.records import MIRRORS_KEY, records
|
||||
|
||||
|
||||
class LookupModule(LookupBase):
|
||||
def run(
|
||||
self,
|
||||
terms: list[Any] | None,
|
||||
variables: dict[str, Any] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[Any]:
|
||||
rendered = records()
|
||||
if not rendered:
|
||||
raise AnsibleError(
|
||||
f"{MIRRORS_KEY} needs a primary and a fallback mirror before the "
|
||||
"apt-ubuntu repositories can be proxied; Nexus would otherwise be "
|
||||
"bootstrapped against an unresolved placeholder"
|
||||
)
|
||||
return [rendered]
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Lookup ``cache_repos``: the repositories the package cache proxies.
|
||||
|
||||
Keyed by repository name, so a template resolving a host's route reaches its
|
||||
flavor, upstream and mirror without a second declaration. `utils.cache.hosts`
|
||||
is the single point of truth; the entries come from ``cache: repos:`` in the
|
||||
roles' ``meta/networks.yml``, and deployed, the inventory may correct or
|
||||
extend them; see `utils.cache.inventory`.
|
||||
|
||||
{{ lookup('cache_repos')['apt-debian'].mirror }}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from ansible.plugins.lookup import LookupBase
|
||||
|
||||
from utils.cache.hosts import repo_records
|
||||
from utils.cache.inventory import configured, merge
|
||||
|
||||
SECTION = "repos"
|
||||
|
||||
|
||||
class LookupModule(LookupBase):
|
||||
def run(
|
||||
self,
|
||||
terms: list[Any] | None,
|
||||
variables: dict[str, Any] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[Any]:
|
||||
overrides = configured(
|
||||
self._loader, getattr(self, "_templar", None), variables, SECTION
|
||||
)
|
||||
return [merge(repo_records(), overrides)]
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Lookup ``cache_upstreams``: the hosts the package cache answers for.
|
||||
|
||||
The declarations live under ``cache: hosts:`` in every role's
|
||||
``meta/networks.yml`` and `utils.cache.hosts` collects them. Deployed, the
|
||||
inventory may correct or extend them; see `utils.cache.inventory`.
|
||||
|
||||
Returns one dict per host, sorted by host, carrying ``host``, ``repo``,
|
||||
``routes``, ``listen``, ``read_timeout`` and ``passthrough``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from ansible.plugins.lookup import LookupBase
|
||||
|
||||
from utils.cache.hosts import host_records
|
||||
from utils.cache.inventory import configured, merge
|
||||
|
||||
SECTION = "upstreams"
|
||||
|
||||
|
||||
class LookupModule(LookupBase):
|
||||
def run(
|
||||
self,
|
||||
terms: list[Any] | None,
|
||||
variables: dict[str, Any] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[Any]:
|
||||
overrides = configured(
|
||||
self._loader, getattr(self, "_templar", None), variables, SECTION
|
||||
)
|
||||
return [list(merge(host_records(), overrides).values())]
|
||||
@@ -0,0 +1,20 @@
|
||||
# Package cache
|
||||
|
||||
## Description
|
||||
|
||||
[Nexus Repository](https://www.sonatype.com/products/sonatype-nexus-repository) is a repository manager that proxies package registries and serves an artefact it has already fetched from local disk. [nginx](https://nginx.org/) sits in front of it and answers for the upstream hostnames themselves, so a package manager keeps the URLs it was built with and still reads through the proxy.
|
||||
|
||||
## Overview
|
||||
|
||||
The role deploys two containers on its inventory group's host: the repository manager and its nginx frontend. Both run in compose mode on the manager, like the sibling [svc-registry-cache](../svc-registry-cache/README.md).
|
||||
|
||||
[upstreams.conf.j2](./templates/upstreams.conf.j2) is rendered to the host and mounted into the frontend as `conf.d/upstreams.conf`. Its content comes from the `cache:` sections of the roles' `meta/networks.yml`, so no hostname, repository or upstream URL is written in this role.
|
||||
|
||||
`compose.cache-consumer.yml.j2`, `apt.list.j2`, `npmrc.j2` and `pip.conf.j2` under `templates/`, and the shell scripts under `files/`, belong to the development compose stack, which drives them through [render.py](../../utils/cache/render.py) and [compose.py](../../cli/administration/deploy/development/compose.py). The Ansible deployment reads none of them.
|
||||
|
||||
## Features
|
||||
|
||||
- **Declaration driven:** Every proxied repository and every answered hostname comes from the `cache:` declarations, collected by the `cache_upstreams` and `cache_repos` lookups. A new upstream is added by declaring it there.
|
||||
- **Mirror failover:** A repository that declares a mirror is served through a guarded location: a 5xx from its primary remote re-runs the same path against the sibling repository, while a 404 is passed straight back.
|
||||
- **Persistent store:** Proxied artefacts live in a named volume, so a redeploy reuses what earlier runs already pulled.
|
||||
- **Backup free:** The volume is backup-disabled and its backup consumer stays off; every artefact in it is re-fetchable from its upstream.
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck shell=bash
|
||||
# nocheck: raw-docker this runs on the developer's host to bring the dev cache up, where the `container` wrapper is not on PATH (verified: `which container` -> not found)
|
||||
# Idempotent Nexus 3 OSS proxy bootstrap.
|
||||
# See docs/contributing/environment/cache.md.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD:?Source scripts/meta/env/load.sh first}"
|
||||
: "${INFINITO_CACHE_PACKAGE_BLOBSTORE_MAX:?Source scripts/meta/env/load.sh first}"
|
||||
: "${INFINITO_CACHE_PACKAGE_MAX_AGE_MIN:?Source scripts/meta/env/load.sh first}"
|
||||
: "${INFINITO_CACHE_PACKAGE_METADATA_MAX_AGE_MIN:?Source scripts/meta/env/load.sh first}"
|
||||
: "${INFINITO_CACHE_UPSTREAMS:?Source scripts/meta/env/load.sh first}"
|
||||
: "${INFINITO_CACHE_PACKAGE_CONTAINER:?Source scripts/meta/env/load.sh first}"
|
||||
: "${INFINITO_CACHE_PACKAGE_PORT:?Source scripts/meta/env/load.sh first}"
|
||||
|
||||
CACHE_CONTENT_MAX_AGE_MIN="${INFINITO_CACHE_PACKAGE_MAX_AGE_MIN}"
|
||||
CACHE_METADATA_MAX_AGE_MIN="${INFINITO_CACHE_PACKAGE_METADATA_MAX_AGE_MIN}"
|
||||
|
||||
PKGCACHE_CONTAINER="${INFINITO_CACHE_PACKAGE_CONTAINER}"
|
||||
NEXUS_REST="http://127.0.0.1:${INFINITO_CACHE_PACKAGE_PORT}/service/rest"
|
||||
|
||||
BOOTSTRAP_DONE_FILE="/nexus-data/.infinito-bootstrap-done"
|
||||
NEXUS_ADMIN_PW_FILE="/nexus-data/.infinito-admin-password"
|
||||
NEXUS_DEFAULT_PASSWORD="admin123"
|
||||
|
||||
log() { printf '[package-cache-bootstrap] %s\n' "$*" >&2; }
|
||||
|
||||
nexus_curl() { docker exec "${PKGCACHE_CONTAINER}" curl "$@"; }
|
||||
|
||||
wait_for_nexus() {
|
||||
local _attempt
|
||||
for _attempt in $(seq 1 120); do
|
||||
if nexus_curl -fsS "${NEXUS_REST}/v1/status" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
log "Nexus REST not reachable at ${NEXUS_REST}/v1/status after 4 minutes"
|
||||
return 1
|
||||
}
|
||||
|
||||
store_admin_password() {
|
||||
printf %s "${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD}" |
|
||||
docker exec -i "${PKGCACHE_CONTAINER}" sh -c "umask 077; cat > '${NEXUS_ADMIN_PW_FILE}'"
|
||||
}
|
||||
|
||||
rotate_admin_password() {
|
||||
ADMIN_USER="admin"
|
||||
if docker exec "${PKGCACHE_CONTAINER}" test -f "${BOOTSTRAP_DONE_FILE}"; then
|
||||
log "Already bootstrapped; using stored admin password"
|
||||
ADMIN_PASS="$(docker exec "${PKGCACHE_CONTAINER}" cat "${NEXUS_ADMIN_PW_FILE}" 2>/dev/null || true)"
|
||||
if [ -z "${ADMIN_PASS}" ]; then
|
||||
ADMIN_PASS="${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD}"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Rotating admin password from default '${NEXUS_DEFAULT_PASSWORD}'"
|
||||
if ! nexus_curl -fsS -u "admin:${NEXUS_DEFAULT_PASSWORD}" \
|
||||
-H "Content-Type: text/plain" \
|
||||
-X PUT "${NEXUS_REST}/v1/security/users/admin/change-password" \
|
||||
--data-binary "${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD}"; then
|
||||
if nexus_curl -fsS -o /dev/null -u "admin:${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD}" \
|
||||
"${NEXUS_REST}/v1/repositories" >/dev/null 2>&1; then
|
||||
log "Default rotation rejected but configured password authenticates; assuming previous bootstrap, marking done"
|
||||
else
|
||||
log "Cannot authenticate with default '${NEXUS_DEFAULT_PASSWORD}' nor with configured INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD."
|
||||
log "Wipe the Nexus data and retry; the dev stack keeps it under ${INFINITO_CACHE_PACKAGE_HOST_PATH:-} and the role in the package_cache_data volume."
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
store_admin_password
|
||||
docker exec "${PKGCACHE_CONTAINER}" touch "${BOOTSTRAP_DONE_FILE}"
|
||||
ADMIN_PASS="${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD}"
|
||||
}
|
||||
|
||||
ensure_blobstore() {
|
||||
local quota_gb="${INFINITO_CACHE_PACKAGE_BLOBSTORE_MAX%g}"
|
||||
local quota_mb=$((quota_gb * 1024))
|
||||
local payload
|
||||
payload="$(printf '{"name":"default","path":"default","softQuota":{"type":"spaceUsedQuota","limit":%d}}' "${quota_mb}")"
|
||||
local code
|
||||
code="$(nexus_curl -sS -o /dev/null -w '%{http_code}' \
|
||||
-u "${ADMIN_USER}:${ADMIN_PASS}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "${NEXUS_REST}/v1/blobstores/file" \
|
||||
--data "${payload}" || true)"
|
||||
case "${code}" in
|
||||
201 | 204) log "blobstore default created (HTTP ${code})" ;;
|
||||
400 | 409) log "blobstore default already exists (HTTP ${code})" ;;
|
||||
*)
|
||||
log "unexpected HTTP ${code} creating blobstore"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
ensure_repo() {
|
||||
local format="$1" kind="$2" name="$3" payload="$4"
|
||||
local code
|
||||
code="$(nexus_curl -sS -o /dev/null -w '%{http_code}' \
|
||||
-u "${ADMIN_USER}:${ADMIN_PASS}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "${NEXUS_REST}/v1/repositories/${format}/${kind}" \
|
||||
--data "${payload}" || true)"
|
||||
case "${code}" in
|
||||
201 | 204) log "${format} ${kind} ${name} created (HTTP ${code})" ;;
|
||||
400 | 409)
|
||||
code="$(nexus_curl -sS -o /dev/null -w '%{http_code}' \
|
||||
-u "${ADMIN_USER}:${ADMIN_PASS}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X PUT "${NEXUS_REST}/v1/repositories/${format}/${kind}/${name}" \
|
||||
--data "${payload}" || true)"
|
||||
case "${code}" in
|
||||
200 | 204) log "${format} ${kind} ${name} updated (HTTP ${code})" ;;
|
||||
*)
|
||||
log "unexpected HTTP ${code} updating ${format} ${kind} ${name}"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
log "unexpected HTTP ${code} creating ${format} ${kind} ${name}"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
ensure_proxy_repo() { ensure_repo "$1" proxy "$2" "$3"; }
|
||||
|
||||
# autoBlock stays off: once Nexus blocks a remote it answers every client
|
||||
# with `404 Remote Auto Blocked` for the whole block window, so a single
|
||||
# upstream hiccup fails every concurrent job instead of just the request
|
||||
# that hit it.
|
||||
ensure_all_proxies() {
|
||||
local storage='"storage":{"blobStoreName":"default","strictContentTypeValidation":true},"proxy":{"contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"negativeCache":{"enabled":true,"timeToLive":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"httpClient":{"blocked":false,"autoBlock":false}'
|
||||
|
||||
local record name flavor url distribution depth content_max_age content extra
|
||||
|
||||
for record in ${INFINITO_CACHE_UPSTREAMS//,/ }; do
|
||||
IFS='|' read -r name flavor url distribution depth content_max_age <<<"${record}"
|
||||
content="${content_max_age:-${CACHE_CONTENT_MAX_AGE_MIN}}"
|
||||
case "${flavor}" in
|
||||
apt) extra=',"apt":{"distribution":"'"${distribution}"'","flat":false}' ;;
|
||||
yum) extra=',"yum":{"repodataDepth":'"${depth}"'}' ;;
|
||||
raw) extra=',"raw":{"contentDisposition":"ATTACHMENT"}' ;;
|
||||
*) extra='' ;;
|
||||
esac
|
||||
ensure_proxy_repo "${flavor}" "${name}" \
|
||||
'{"name":"'"${name}"'","online":true,'"${storage}"',"proxy":{"remoteUrl":"'"${url}"'","contentMaxAge":'"${content}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'}'"${extra}"'}'
|
||||
done
|
||||
}
|
||||
|
||||
ensure_eula_accepted() {
|
||||
local code
|
||||
# shellcheck disable=SC1112
|
||||
code="$(nexus_curl -sS -o /dev/null -w '%{http_code}' \
|
||||
-u "${ADMIN_USER}:${ADMIN_PASS}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "${NEXUS_REST}/v1/system/eula" \
|
||||
--data '{"accepted":true,"disclaimer":"Use of Sonatype Nexus Repository - Community Edition is governed by the End User License Agreement at https://links.sonatype.com/products/nxrm/ce-eula. By returning the value from ‘accepted:false’ to ‘accepted:true’, you acknowledge that you have read and agree to the End User License Agreement at https://links.sonatype.com/products/nxrm/ce-eula."}' || true)"
|
||||
case "${code}" in
|
||||
200 | 204) log "EULA accepted (HTTP ${code})" ;;
|
||||
*)
|
||||
log "unexpected HTTP ${code} accepting EULA"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
ensure_anonymous_access() {
|
||||
local code
|
||||
code="$(nexus_curl -sS -o /dev/null -w '%{http_code}' \
|
||||
-u "${ADMIN_USER}:${ADMIN_PASS}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X PUT "${NEXUS_REST}/v1/security/anonymous" \
|
||||
--data '{"enabled":true,"userId":"anonymous","realmName":"NexusAuthorizingRealm"}' || true)"
|
||||
case "${code}" in
|
||||
200 | 204) log "anonymous access enabled (HTTP ${code})" ;;
|
||||
*)
|
||||
log "unexpected HTTP ${code} enabling anonymous access"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main() {
|
||||
wait_for_nexus
|
||||
rotate_admin_password
|
||||
ensure_eula_accepted
|
||||
ensure_anonymous_access
|
||||
ensure_blobstore
|
||||
ensure_all_proxies
|
||||
log "bootstrap done"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
File renamed without changes.
+2
-1
@@ -10,7 +10,8 @@ CA_KEY=/ca/ca.key
|
||||
CA_DAYS=3650
|
||||
LEAF_DAYS=730
|
||||
|
||||
HOSTNAMES="pypi.org files.pythonhosted.org registry.npmjs.org rubygems.org index.rubygems.org repo.packagist.org proxy.golang.org dl-cdn.alpinelinux.org raw.githubusercontent.com codeload.github.com argos-net.com"
|
||||
: "${INFINITO_CACHE_TLS_HOSTS:?Pass the hostnames from the cache: declarations}"
|
||||
HOSTNAMES="${INFINITO_CACHE_TLS_HOSTS}"
|
||||
|
||||
log() { printf '[package-frontend-certs] %s\n' "$*" >&2; }
|
||||
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck shell=bash
|
||||
# nocheck: raw-docker this runs on the developer's host to bring the dev cache up, where the `container` wrapper is not on PATH (verified: `which container` -> not found)
|
||||
# Host wrapper for the in-container cert generator.
|
||||
# See docs/contributing/environment/cache.md.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${INFINITO_CACHE_PACKAGE_FRONTEND_CA_DIR:?Source scripts/meta/env/load.sh first}"
|
||||
: "${INFINITO_CACHE_PACKAGE_FRONTEND_CERTS_DIR:?Source scripts/meta/env/load.sh first}"
|
||||
: "${INFINITO_CACHE_TLS_HOSTS:?Source scripts/meta/env/load.sh first}"
|
||||
|
||||
ROLE_FILES="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${ROLE_FILES}/../../../.." && pwd)"
|
||||
INNER_SCRIPT="${ROLE_FILES}/cert-gen.sh"
|
||||
RETRY="${REPO_ROOT}/scripts/github/runner/pull_with_retry.sh"
|
||||
|
||||
if [[ ! -r "${INNER_SCRIPT}" ]]; then
|
||||
echo "[package-frontend-certs] missing inner script: ${INNER_SCRIPT}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
: "${INFINITO_CACHE_PACKAGE_FRONTEND_INIT_IMAGE:?Source scripts/meta/env/load.sh first}"
|
||||
ALPINE_IMAGE="${INFINITO_CACHE_PACKAGE_FRONTEND_INIT_IMAGE}"
|
||||
|
||||
# Exception: deployed by the role, only files/ ships, so the repo-only retry
|
||||
# helper is absent and a plain pull is the whole of what can be attempted.
|
||||
if [[ -x "${RETRY}" ]]; then
|
||||
MAX_ATTEMPTS=3 RETRY_DELAY_SECONDS=10 "${RETRY}" "${ALPINE_IMAGE}" ||
|
||||
echo "[package-frontend-certs] could not pull ${ALPINE_IMAGE}; falling back to a cached copy" >&2
|
||||
else
|
||||
docker pull "${ALPINE_IMAGE}" >/dev/null 2>&1 ||
|
||||
echo "[package-frontend-certs] could not pull ${ALPINE_IMAGE}; falling back to a cached copy" >&2
|
||||
fi
|
||||
|
||||
exec docker run --rm \
|
||||
-e "INFINITO_CACHE_TLS_HOSTS=${INFINITO_CACHE_TLS_HOSTS}" \
|
||||
-v "${INFINITO_CACHE_PACKAGE_FRONTEND_CA_DIR}:/ca" \
|
||||
-v "${INFINITO_CACHE_PACKAGE_FRONTEND_CERTS_DIR}:/certs" \
|
||||
-v "${INNER_SCRIPT}:/work/cert-gen.sh:ro" \
|
||||
--entrypoint /bin/sh \
|
||||
"${ALPINE_IMAGE}" \
|
||||
/work/cert-gen.sh
|
||||
@@ -0,0 +1 @@
|
||||
# nocheck: role-files-layout - Role test harness with test.sh as its entry point; test-e2e-cli stages files/test/ by that fixed path.
|
||||
Executable
+133
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Interrogate the deployed package cache through its own containers.
|
||||
|
||||
Every assertion is made against the running stack: Nexus is asked for each
|
||||
declared repository, the frontend is asked for the certificates it serves,
|
||||
and each cached hostname is requested through the frontend exactly as a
|
||||
package manager would. A declaration the deploy did not realise therefore
|
||||
fails here rather than passing on the strength of the file that declared it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
NEXUS = os.environ["CACHE_PACKAGE_TEST_CONTAINER"]
|
||||
FRONTEND = os.environ["CACHE_PACKAGE_TEST_FRONTEND"]
|
||||
PORT = os.environ["CACHE_PACKAGE_TEST_PORT"]
|
||||
CERTS_DIR = os.environ["CACHE_PACKAGE_TEST_CERTS_DIR"]
|
||||
REPOS = json.loads(os.environ["CACHE_PACKAGE_TEST_REPOS"])
|
||||
HOSTS = json.loads(os.environ["CACHE_PACKAGE_TEST_HOSTS"])
|
||||
|
||||
MISSING = "404"
|
||||
UNREACHABLE = {"502", "503", "504", "000", ""}
|
||||
TIMEOUT = 60
|
||||
|
||||
failures: list[str] = []
|
||||
|
||||
|
||||
def run(container: str, argv: list[str]) -> tuple[int, str]:
|
||||
"""Return the exit status and stdout of *argv* inside *container*.
|
||||
|
||||
Args:
|
||||
container: the container to run in.
|
||||
argv: the command and its arguments.
|
||||
"""
|
||||
done = subprocess.run(
|
||||
["container", "exec", container, *argv],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=TIMEOUT,
|
||||
)
|
||||
return done.returncode, done.stdout.strip()
|
||||
|
||||
|
||||
def status(container: str, url: str, host: str = "") -> str:
|
||||
"""Return the HTTP status *url* answers with, as seen from *container*."""
|
||||
argv = [
|
||||
"curl",
|
||||
"-sS",
|
||||
"-o",
|
||||
"/dev/null",
|
||||
"-w",
|
||||
"%{http_code}",
|
||||
"-k",
|
||||
"--max-time",
|
||||
"30",
|
||||
]
|
||||
if host:
|
||||
argv += ["-H", f"Host: {host}"]
|
||||
code, out = run(container, [*argv, url])
|
||||
return out if code == 0 else ""
|
||||
|
||||
|
||||
def check_nexus_is_up() -> None:
|
||||
code = status(NEXUS, f"http://127.0.0.1:{PORT}/service/rest/v1/status")
|
||||
if code != "200":
|
||||
failures.append(f"Nexus status endpoint answered {code!r}, want 200")
|
||||
|
||||
|
||||
def check_every_declared_repository_exists() -> None:
|
||||
"""A repository Nexus does not hold answers 404 on its own root."""
|
||||
for name in sorted(REPOS):
|
||||
code = status(NEXUS, f"http://127.0.0.1:{PORT}/repository/{name}/")
|
||||
if code == MISSING:
|
||||
failures.append(
|
||||
f"repository {name} is declared but Nexus answers 404 for it; "
|
||||
"the bootstrap did not create it"
|
||||
)
|
||||
elif code in UNREACHABLE:
|
||||
failures.append(f"repository {name} answered {code!r} on its root")
|
||||
|
||||
|
||||
def check_frontend_serves_a_cert_per_tls_host() -> None:
|
||||
for entry in HOSTS:
|
||||
if not any(listener.get("tls") for listener in entry.get("listen", [])):
|
||||
continue
|
||||
host = entry["host"]
|
||||
for suffix in ("crt", "key"):
|
||||
code, _ = run(FRONTEND, ["test", "-s", f"{CERTS_DIR}/{host}.{suffix}"])
|
||||
if code != 0:
|
||||
failures.append(
|
||||
f"{host} is served over TLS but {CERTS_DIR}/{host}.{suffix} "
|
||||
"is missing or empty, so nginx cannot complete a handshake"
|
||||
)
|
||||
|
||||
|
||||
def check_every_host_is_proxied() -> None:
|
||||
"""Each cached hostname must reach its repository through the frontend."""
|
||||
for entry in HOSTS:
|
||||
host = entry["host"]
|
||||
for listener in entry.get("listen", []):
|
||||
scheme = "https" if listener.get("tls") else "http"
|
||||
port = listener.get("port")
|
||||
for route in entry.get("routes", []):
|
||||
url = f"{scheme}://127.0.0.1:{port}{route['path']}"
|
||||
code = status(FRONTEND, url, host=host)
|
||||
if code in UNREACHABLE:
|
||||
failures.append(
|
||||
f"{host}{route['path']} on port {port} answered {code!r}; "
|
||||
f"the frontend did not reach repository {route['repo']}"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
check_nexus_is_up()
|
||||
check_every_declared_repository_exists()
|
||||
check_frontend_serves_a_cert_per_tls_host()
|
||||
check_every_host_is_proxied()
|
||||
|
||||
if failures:
|
||||
for line in failures:
|
||||
print(f"[FAIL] {line}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"OK: {len(REPOS)} repositories and {len(HOSTS)} hosts served by the cache")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
# E2E orchestrator for svc-cache-package.
|
||||
# Drives the deployed cache: Nexus answers, every declared repository exists
|
||||
# in it, the frontend holds a leaf cert for every host it terminates TLS for,
|
||||
# and a request carrying a real upstream Host header comes back proxied onto
|
||||
# the matching repository. Every check interrogates the running stack.
|
||||
# Variables sourced from test.env.j2 by test-e2e-cli.
|
||||
set -euo pipefail
|
||||
|
||||
: "${CACHE_PACKAGE_TEST_CONTAINER:?}"
|
||||
: "${CACHE_PACKAGE_TEST_FRONTEND:?}"
|
||||
: "${CACHE_PACKAGE_TEST_PORT:?}"
|
||||
: "${CACHE_PACKAGE_TEST_CERTS_DIR:?}"
|
||||
: "${CACHE_PACKAGE_TEST_REPOS:?}"
|
||||
: "${CACHE_PACKAGE_TEST_HOSTS:?}"
|
||||
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
exec python3 "${DIR}/probe.py"
|
||||
@@ -0,0 +1,2 @@
|
||||
---
|
||||
# nocheck: info-media Self-written infrastructure glue role deploying a Nexus proxy cache and its TLS frontend; no consumer-facing upstream product website.
|
||||
@@ -0,0 +1,31 @@
|
||||
---
|
||||
galaxy_info:
|
||||
author: Kevin Veen-Birkenbach
|
||||
description: Cluster-local proxy cache for package upstreams, serving apt, pip, npm and raw artefacts every role declares
|
||||
license: Infinito.Nexus Community License (Non-Commercial)
|
||||
company: |
|
||||
Kevin Veen-Birkenbach
|
||||
https://www.veen.world
|
||||
galaxy_tags:
|
||||
- registry
|
||||
- docker
|
||||
- cache
|
||||
- swarm
|
||||
issue_tracker_url: https://s.infinito.nexus/issues
|
||||
min_ansible_version: "2.9"
|
||||
platforms:
|
||||
- name: ArchLinux
|
||||
versions:
|
||||
- all
|
||||
- name: Debian
|
||||
versions:
|
||||
- all
|
||||
- name: EL
|
||||
versions:
|
||||
- all
|
||||
- name: Fedora
|
||||
versions:
|
||||
- all
|
||||
- name: Ubuntu
|
||||
versions:
|
||||
- all
|
||||
@@ -0,0 +1,121 @@
|
||||
---
|
||||
local:
|
||||
subnet: 192.168.227.0/24
|
||||
cache:
|
||||
repos:
|
||||
apt-debian:
|
||||
flavor: apt
|
||||
upstream: http://deb.debian.org/debian
|
||||
distribution: bookworm
|
||||
mirror: apt-debian-mirror
|
||||
apt-debian-mirror:
|
||||
flavor: apt
|
||||
upstream: http://ftp.debian.org/debian
|
||||
distribution: bookworm
|
||||
apt-debian-security:
|
||||
flavor: apt
|
||||
upstream: http://deb.debian.org/debian-security # nocheck: url the bare GET on the redirector returns 404; the apt paths under dists/ resolve
|
||||
distribution: bookworm-security
|
||||
mirror: apt-debian-security-mirror
|
||||
apt-debian-security-mirror:
|
||||
flavor: apt
|
||||
upstream: http://security.debian.org/debian-security
|
||||
distribution: bookworm-security
|
||||
apt-ubuntu:
|
||||
flavor: apt
|
||||
upstream: "${UBUNTU_PRIMARY}"
|
||||
distribution: jammy
|
||||
mirror: apt-ubuntu-mirror
|
||||
apt-ubuntu-mirror:
|
||||
flavor: apt
|
||||
upstream: "${UBUNTU_FALLBACK}"
|
||||
distribution: jammy
|
||||
apt-ubuntu-security:
|
||||
flavor: apt
|
||||
upstream: "${UBUNTU_PRIMARY}"
|
||||
distribution: jammy-security
|
||||
mirror: apt-ubuntu-security-mirror
|
||||
apt-ubuntu-security-mirror:
|
||||
flavor: apt
|
||||
upstream: "${UBUNTU_FALLBACK}"
|
||||
distribution: jammy-security
|
||||
gem-proxy:
|
||||
flavor: rubygems
|
||||
upstream: https://rubygems.org/
|
||||
go-proxy:
|
||||
flavor: go
|
||||
upstream: https://proxy.golang.org/
|
||||
helm-bitnami:
|
||||
flavor: helm
|
||||
upstream: https://charts.bitnami.com/bitnami
|
||||
npm-proxy:
|
||||
flavor: npm
|
||||
upstream: https://registry.npmjs.org/
|
||||
pypi-proxy:
|
||||
flavor: pypi
|
||||
upstream: https://pypi.org/
|
||||
raw-alpine:
|
||||
flavor: raw
|
||||
upstream: https://dl-cdn.alpinelinux.org/alpine/ # nocheck: url answers 200 direct (measured); the 404 is the proxy serving this repository's own root
|
||||
content_max_age: "0"
|
||||
raw-codeload-github:
|
||||
flavor: raw
|
||||
upstream: https://codeload.github.com/
|
||||
raw-githubusercontent:
|
||||
flavor: raw
|
||||
upstream: https://raw.githubusercontent.com/
|
||||
raw-packagist:
|
||||
flavor: raw
|
||||
upstream: https://repo.packagist.org/ # nocheck: url the metadata paths under p2/ resolve; the bare root has no index
|
||||
content_max_age: "0"
|
||||
yum-fedora:
|
||||
flavor: yum
|
||||
upstream: https://download.fedoraproject.org/pub/fedora/linux/
|
||||
repodata_depth: "5"
|
||||
yum-rocky:
|
||||
flavor: yum
|
||||
upstream: https://download.rockylinux.org/pub/rocky/
|
||||
repodata_depth: "5"
|
||||
hosts:
|
||||
archive.ubuntu.com:
|
||||
ports:
|
||||
- "80"
|
||||
paths:
|
||||
/ubuntu/: apt-ubuntu
|
||||
codeload.github.com:
|
||||
repo: raw-codeload-github
|
||||
deb.debian.org:
|
||||
ports:
|
||||
- "80"
|
||||
paths:
|
||||
/debian/: apt-debian
|
||||
/debian-security/: apt-debian-security
|
||||
dl-cdn.alpinelinux.org:
|
||||
repo: raw-alpine
|
||||
ports:
|
||||
- "80"
|
||||
- "443"
|
||||
paths:
|
||||
/alpine/: raw-alpine
|
||||
files.pythonhosted.org:
|
||||
repo: pypi-proxy
|
||||
index.rubygems.org:
|
||||
repo: gem-proxy
|
||||
proxy.golang.org:
|
||||
repo: go-proxy
|
||||
pypi.org:
|
||||
repo: pypi-proxy
|
||||
raw.githubusercontent.com:
|
||||
repo: raw-githubusercontent
|
||||
registry.npmjs.org:
|
||||
repo: npm-proxy
|
||||
passthrough: "true"
|
||||
repo.packagist.org:
|
||||
repo: raw-packagist
|
||||
rubygems.org:
|
||||
repo: gem-proxy
|
||||
security.ubuntu.com:
|
||||
ports:
|
||||
- "80"
|
||||
paths:
|
||||
/ubuntu/: apt-ubuntu-security
|
||||
@@ -0,0 +1,74 @@
|
||||
---
|
||||
# nocheck: dynamic-flag
|
||||
node:
|
||||
enabled: true
|
||||
shared: true
|
||||
bond: 1
|
||||
package:
|
||||
healthcheck:
|
||||
flavor: http_status
|
||||
port_key: http
|
||||
path: service/rest/v1/status
|
||||
modes:
|
||||
compose:
|
||||
enabled: true
|
||||
swarm:
|
||||
enabled: false
|
||||
# nocheck: default-placement-manager the proxy repositories back every build on this host; scheduling them on a worker that still needs them is a chicken-and-egg
|
||||
placement: manager
|
||||
enabled: true
|
||||
shared: true
|
||||
bond: 1
|
||||
image: sonatype/nexus3
|
||||
version: 3.91.1-ubi
|
||||
name: package-cache
|
||||
# nocheck: min-storage-headroom Reason: the image transfers 0.373 GiB, but the service is the blobstore itself and fills up to blobstore_max (40g); sizing the host to the image alone would let the deploy prune a cache it has no room to refill
|
||||
min_storage: 4GB
|
||||
backup:
|
||||
disabled: true
|
||||
cpus: "2"
|
||||
mem_reservation: 1g
|
||||
mem_limit: 4g
|
||||
pids_limit: 512
|
||||
ports:
|
||||
local:
|
||||
http: 8095
|
||||
internal:
|
||||
http: 8081
|
||||
blobstore_max: 40g
|
||||
content_max_age_min: 8640
|
||||
metadata_max_age_min: 0
|
||||
lifecycle: beta
|
||||
# nocheck: dynamic-flag the frontend has no group of its own; it is up exactly when the cache it fronts is
|
||||
frontend:
|
||||
healthcheck:
|
||||
flavor: http
|
||||
port_key: http
|
||||
modes:
|
||||
compose:
|
||||
enabled: true
|
||||
swarm:
|
||||
enabled: false
|
||||
placement: manager # nocheck: default-placement-manager it fronts the cache beside it
|
||||
enabled: true
|
||||
shared: true
|
||||
bond: 1
|
||||
image: nginx
|
||||
version: 1.31-alpine
|
||||
name: package-cache-frontend
|
||||
# nocheck: dynamic-flag the throwaway container that issues the CA and leaf certs before nginx starts, not a deployed service
|
||||
init_image: alpine:latest
|
||||
min_storage: 64MB
|
||||
cpus: "0.5"
|
||||
mem_reservation: 64m
|
||||
mem_limit: 256m
|
||||
pids_limit: 256
|
||||
ports:
|
||||
internal:
|
||||
http: 80
|
||||
https: 443
|
||||
lifecycle: beta
|
||||
# nocheck: dynamic-flag - proxied package content is re-fetchable, the backup consumer stays permanently off
|
||||
container_backup:
|
||||
enabled: false
|
||||
shared: true
|
||||
@@ -0,0 +1,32 @@
|
||||
# nocheck: backup-service-flag proxied package content is re-fetchable from its upstream; the volume is backup-disabled and the consumer permanently off
|
||||
---
|
||||
data:
|
||||
type: volume
|
||||
name: package_cache_data
|
||||
backup: false
|
||||
# nocheck: nfs-false-data-loss Reason: every blob is a proxy copy of an upstream artefact, so a drain costs a re-fetch and loses nothing; the service is pinned to the manager and disabled in swarm
|
||||
nfs: false
|
||||
mounts:
|
||||
- service: package
|
||||
target: /nexus-data
|
||||
frontend_ca:
|
||||
type: bind
|
||||
source: "{{ [lookup('container', 'svc-cache-package', 'directories.config'), 'ca'] | path_join }}"
|
||||
read_only: true
|
||||
mounts:
|
||||
- service: frontend
|
||||
target: /etc/nginx/ca
|
||||
frontend_certs:
|
||||
type: bind
|
||||
source: "{{ [lookup('container', 'svc-cache-package', 'directories.config'), 'certs'] | path_join }}"
|
||||
read_only: true
|
||||
mounts:
|
||||
- service: frontend
|
||||
target: /etc/nginx/certs
|
||||
frontend_conf:
|
||||
type: config
|
||||
source: "{{ [lookup('container', 'svc-cache-package', 'directories.config'), 'upstreams.conf'] | path_join }}"
|
||||
mode: "0444"
|
||||
mounts:
|
||||
- service: frontend
|
||||
target: /etc/nginx/conf.d/upstreams.conf
|
||||
@@ -0,0 +1,44 @@
|
||||
---
|
||||
- include_tasks: utils/once/flag.yml
|
||||
|
||||
- include_tasks: utils/load/handlers.yml
|
||||
vars:
|
||||
handler_role_name: "sys-svc-compose"
|
||||
|
||||
- name: "📄 Render the frontend map for '{{ application_id }}'"
|
||||
ansible.builtin.include_tasks:
|
||||
file: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/render_replicated_templates.yml') }}"
|
||||
vars:
|
||||
role_templates:
|
||||
- src: "upstreams.conf.j2"
|
||||
dest: "{{ CACHE_PACKAGE_FRONTEND_CONF_HOST }}"
|
||||
mode: "0644"
|
||||
|
||||
- name: "📁 Create the frontend's CA and certificate directories"
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
loop:
|
||||
- "{{ CACHE_PACKAGE_CA_DIR_HOST }}"
|
||||
- "{{ CACHE_PACKAGE_CERTS_DIR_HOST }}"
|
||||
|
||||
- name: "🔐 Issue the frontend CA and one leaf per TLS host"
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ [CACHE_PACKAGE_SHELL_DIR, 'certs.sh'] | path_join }}"
|
||||
environment: "{{ lookup('template', 'env/certs.j2') | from_yaml }}"
|
||||
register: cache_package_certs
|
||||
changed_when: "'Generating' in cache_package_certs.stderr"
|
||||
|
||||
- include_role:
|
||||
name: sys-svc-compose
|
||||
vars:
|
||||
compose_handlers_flush: true
|
||||
|
||||
- name: "📦 Create the proxy repositories the frontend maps onto"
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ [CACHE_PACKAGE_SHELL_DIR, 'bootstrap.sh'] | path_join }}"
|
||||
environment: "{{ lookup('template', 'env/bootstrap.j2') | from_yaml }}"
|
||||
no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
|
||||
register: cache_package_bootstrap
|
||||
changed_when: "'created' in cache_package_bootstrap.stderr"
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
- include_tasks: 00_core.yml
|
||||
when: run_once_svc_cache_package is not defined
|
||||
vars:
|
||||
application_id: svc-cache-package
|
||||
@@ -0,0 +1,23 @@
|
||||
{% include 'roles/sys-svc-compose/templates/base.yml.j2' %}
|
||||
{% set service_name = entity_name %}
|
||||
{{ service_name }}:
|
||||
{{ lookup('container_image', application_id, service_name) }}
|
||||
{{ lookup('compose_only', 'container_name', CACHE_PACKAGE_CONTAINER) }}
|
||||
{% include 'roles/sys-svc-container/templates/base.yml.j2' %}
|
||||
{{ lookup('container_healthcheck', service_name) | indent(4) }}
|
||||
{{ lookup('container_ports', [service_name, 'http']) | indent(4) }}
|
||||
{{ lookup('container_volumes', service_name) | indent(4) }}
|
||||
{{ lookup('container_networks') | indent(4) }}
|
||||
|
||||
{% set service_name = 'frontend' %}
|
||||
{{ service_name }}:
|
||||
{{ lookup('container_image', application_id, 'frontend') }}
|
||||
{{ lookup('compose_only', 'container_name', CACHE_PACKAGE_FRONTEND_CONTAINER) }}
|
||||
{% include 'roles/sys-svc-container/templates/base.yml.j2' %}
|
||||
{{ lookup('container_healthcheck', service_name) | indent(4) }}
|
||||
{{ lookup('container_volumes', service_name) | indent(4) }}
|
||||
{{ lookup('container_networks', provider_self_alias=False) | indent(4) }}
|
||||
{{ lookup('depends_on', {entity_name: 'service_healthy'}) }}
|
||||
|
||||
{{ lookup('compose_volumes') }}
|
||||
{{ lookup('compose_networks') }}
|
||||
@@ -0,0 +1,12 @@
|
||||
{#
|
||||
Environment for the Nexus bootstrap. The script is shared with the dev
|
||||
stack, so it reads the same INFINITO_* names there; only the values differ.
|
||||
The admin password is why the task carries no_log.
|
||||
#}
|
||||
INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD: {{ CACHE_PACKAGE_ADMIN_PASSWORD | to_json }}
|
||||
INFINITO_CACHE_PACKAGE_BLOBSTORE_MAX: {{ CACHE_PACKAGE_BLOBSTORE_MAX | to_json }}
|
||||
INFINITO_CACHE_PACKAGE_CONTAINER: {{ CACHE_PACKAGE_CONTAINER | to_json }}
|
||||
INFINITO_CACHE_PACKAGE_MAX_AGE_MIN: {{ CACHE_PACKAGE_MAX_AGE_MIN | to_json }}
|
||||
INFINITO_CACHE_PACKAGE_METADATA_MAX_AGE_MIN: {{ CACHE_PACKAGE_METADATA_MAX_AGE_MIN | to_json }}
|
||||
INFINITO_CACHE_PACKAGE_PORT: {{ CACHE_PACKAGE_PORT | to_json }}
|
||||
INFINITO_CACHE_UPSTREAMS: {{ CACHE_PACKAGE_UPSTREAM_RECORDS | to_json }}
|
||||
@@ -0,0 +1,8 @@
|
||||
{#
|
||||
Environment for the certificate issuer. The script is shared with the dev
|
||||
stack, so it reads the same INFINITO_* names there; only the values differ.
|
||||
#}
|
||||
INFINITO_CACHE_PACKAGE_FRONTEND_CA_DIR: {{ CACHE_PACKAGE_CA_DIR_HOST | to_json }}
|
||||
INFINITO_CACHE_PACKAGE_FRONTEND_CERTS_DIR: {{ CACHE_PACKAGE_CERTS_DIR_HOST | to_json }}
|
||||
INFINITO_CACHE_PACKAGE_FRONTEND_INIT_IMAGE: {{ CACHE_PACKAGE_INIT_IMAGE | to_json }}
|
||||
INFINITO_CACHE_TLS_HOSTS: {{ CACHE_PACKAGE_TLS_HOSTS | join(' ') | to_json }}
|
||||
@@ -0,0 +1,6 @@
|
||||
CACHE_PACKAGE_TEST_CERTS_DIR={{ CACHE_PACKAGE_CERTS_DIR }}
|
||||
CACHE_PACKAGE_TEST_CONTAINER={{ CLI_LOCAL_CID }}
|
||||
CACHE_PACKAGE_TEST_FRONTEND={{ CACHE_PACKAGE_FRONTEND_CONTAINER }}
|
||||
CACHE_PACKAGE_TEST_HOSTS='{{ CACHE_PACKAGE_UPSTREAMS | to_json }}'
|
||||
CACHE_PACKAGE_TEST_PORT={{ CACHE_PACKAGE_PORT }}
|
||||
CACHE_PACKAGE_TEST_REPOS='{{ CACHE_PACKAGE_REPOS | to_json }}'
|
||||
@@ -0,0 +1,69 @@
|
||||
{#
|
||||
Rendered from the cache: declarations in roles/*/meta/networks.yml, which
|
||||
are the single point of truth for every cached host, repository and
|
||||
upstream. Nothing here is written by hand; edit the declaration instead.
|
||||
|
||||
A repository that declares a mirror is proxied through a guarded location:
|
||||
Nexus answers 5xx while a remote is unreachable, and the named location
|
||||
retries the same path against the sibling. 404 stays unguarded, because a
|
||||
missing file is missing on both remotes and apt probes for them constantly.
|
||||
#}
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Accept-Encoding "";
|
||||
proxy_buffering on;
|
||||
proxy_request_buffering off;
|
||||
proxy_read_timeout {{ CACHE_PACKAGE_READ_TIMEOUT }};
|
||||
proxy_connect_timeout {{ CACHE_PACKAGE_CONNECT_TIMEOUT }};
|
||||
|
||||
resolver {{ CACHE_PACKAGE_RESOLVER }} ipv6=off valid=30s;
|
||||
{% for host in CACHE_PACKAGE_UPSTREAMS %}
|
||||
{% for listener in host.listen %}
|
||||
|
||||
server {
|
||||
listen {{ listener.port }}{{ ' ssl' if listener.tls else '' }};
|
||||
server_name {{ host.host }};
|
||||
{% if listener.tls %}
|
||||
ssl_certificate {{ CACHE_PACKAGE_CERTS_DIR }}/{{ host.host }}.crt;
|
||||
ssl_certificate_key {{ CACHE_PACKAGE_CERTS_DIR }}/{{ host.host }}.key;
|
||||
{% set repo = CACHE_PACKAGE_REPOS[host.repo] %}
|
||||
{% if host.passthrough %}
|
||||
location /repository/{{ repo.name }}/ {
|
||||
proxy_pass {{ CACHE_PACKAGE_BACKEND }};
|
||||
}
|
||||
{% endif %}
|
||||
location / {
|
||||
{% if host.read_timeout %}
|
||||
proxy_read_timeout {{ host.read_timeout }};
|
||||
{% endif %}
|
||||
proxy_pass {{ CACHE_PACKAGE_BACKEND }}/repository/{{ repo.name }}/;
|
||||
}
|
||||
{% else %}
|
||||
{% for route in host.routes %}
|
||||
{% set repo = CACHE_PACKAGE_REPOS[route.repo] %}
|
||||
{% if repo.mirror %}
|
||||
location {{ route.path }} {
|
||||
rewrite ^{{ route.path }}(.*)$ /repository/{{ repo.name }}/$1 break;
|
||||
proxy_intercept_errors on;
|
||||
proxy_read_timeout {{ CACHE_PACKAGE_MIRROR_TIMEOUT }};
|
||||
error_page 502 503 504 = @{{ repo.name | replace('-', '_') }}_mirror;
|
||||
proxy_pass {{ CACHE_PACKAGE_BACKEND }};
|
||||
}
|
||||
location @{{ repo.name | replace('-', '_') }}_mirror {
|
||||
rewrite ^/repository/{{ repo.name }}/(.*)$ /repository/{{ repo.mirror }}/$1 break;
|
||||
proxy_pass {{ CACHE_PACKAGE_BACKEND }};
|
||||
}
|
||||
{% else %}
|
||||
location {{ route.path }} {
|
||||
rewrite ^{{ route.path }}(.*)$ /repository/{{ repo.name }}/$1 break;
|
||||
proxy_pass {{ CACHE_PACKAGE_BACKEND }};
|
||||
}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
@@ -0,0 +1,30 @@
|
||||
---
|
||||
application_id: svc-cache-package
|
||||
entity_name: "{{ application_id | get_entity_name }}"
|
||||
|
||||
CACHE_PACKAGE_CONTAINER: "{{ lookup('config', application_id, 'services.package.name') }}"
|
||||
CACHE_PACKAGE_FRONTEND_CONTAINER: "{{ lookup('config', application_id, 'services.frontend.name') }}"
|
||||
CACHE_PACKAGE_PORT: "{{ lookup('config', application_id, 'services.package.ports.internal.http') }}"
|
||||
|
||||
CACHE_PACKAGE_FRONTEND_CONF_HOST: "{{ [lookup('container', application_id, 'directories.config'), 'upstreams.conf'] | path_join }}"
|
||||
CACHE_PACKAGE_CA_DIR_HOST: "{{ [lookup('container', application_id, 'directories.config'), 'ca'] | path_join }}"
|
||||
CACHE_PACKAGE_CERTS_DIR_HOST: "{{ [lookup('container', application_id, 'directories.config'), 'certs'] | path_join }}"
|
||||
CACHE_PACKAGE_SHELL_DIR: "{{ [role_path, 'files', 'shell'] | path_join }}"
|
||||
CACHE_PACKAGE_TLS_HOSTS: "{{ CACHE_PACKAGE_UPSTREAMS | selectattr('tls') | map(attribute='host') | list }}"
|
||||
|
||||
CACHE_PACKAGE_UPSTREAMS: "{{ lookup('cache_upstreams') }}"
|
||||
CACHE_PACKAGE_REPOS: "{{ lookup('cache_repos') }}"
|
||||
CACHE_PACKAGE_UPSTREAM_RECORDS: "{{ lookup('cache_records') }}"
|
||||
|
||||
CACHE_PACKAGE_INIT_IMAGE: "{{ lookup('config', application_id, 'services.frontend.init_image') }}"
|
||||
CACHE_PACKAGE_BLOBSTORE_MAX: "{{ lookup('config', application_id, 'services.package.blobstore_max') }}"
|
||||
CACHE_PACKAGE_MAX_AGE_MIN: "{{ lookup('config', application_id, 'services.package.content_max_age_min') }}"
|
||||
CACHE_PACKAGE_METADATA_MAX_AGE_MIN: "{{ lookup('config', application_id, 'services.package.metadata_max_age_min') }}"
|
||||
CACHE_PACKAGE_ADMIN_PASSWORD: "{{ lookup('credentials', application_id, 'admin_password') }}"
|
||||
|
||||
CACHE_PACKAGE_BACKEND: "http://{{ CACHE_PACKAGE_CONTAINER }}:{{ CACHE_PACKAGE_PORT }}"
|
||||
CACHE_PACKAGE_CERTS_DIR: "{{ lookup('volume', application_id, 'frontend_certs').target }}"
|
||||
CACHE_PACKAGE_RESOLVER: 127.0.0.11
|
||||
CACHE_PACKAGE_READ_TIMEOUT: 300s
|
||||
CACHE_PACKAGE_CONNECT_TIMEOUT: 30s
|
||||
CACHE_PACKAGE_MIRROR_TIMEOUT: 8s
|
||||
@@ -76,12 +76,55 @@ def detect_compose_files(project_dir: Path) -> list[Path]:
|
||||
return files
|
||||
|
||||
|
||||
_CACHE_HTTP_HOSTNAMES = (
|
||||
"archive.ubuntu.com",
|
||||
"argos-net.com",
|
||||
"deb.debian.org",
|
||||
"security.ubuntu.com",
|
||||
)
|
||||
_CACHE_HOSTS_KEY = "INFINITO_CACHE_HOSTS"
|
||||
_CACHE_HTTP_HOSTS_KEY = "INFINITO_CACHE_HTTP_HOSTS"
|
||||
_CA_INSTALLER = "package-frontend-ca.sh"
|
||||
|
||||
|
||||
def _cache_http_hostnames() -> list[str]:
|
||||
"""Return the hosts the frontend serves, as `make dotenv` derived them.
|
||||
|
||||
The list lives in the ``cache:`` entries of every role's meta/networks.yml
|
||||
and reaches this script only through the environment, because the script
|
||||
is installed as a standalone binary and cannot import the repository.
|
||||
"""
|
||||
raw = os.environ.get(_CACHE_HOSTS_KEY) or ""
|
||||
return [host for host in (part.strip() for part in raw.split(",")) if host]
|
||||
|
||||
|
||||
def _cache_http_only_hostnames() -> list[str]:
|
||||
"""Return the hosts the frontend serves without TLS."""
|
||||
raw = os.environ.get(_CACHE_HTTP_HOSTS_KEY) or ""
|
||||
return [host for host in (part.strip() for part in raw.split(",")) if host]
|
||||
|
||||
|
||||
def _installs_frontend_ca(service: dict) -> bool:
|
||||
"""Whether this service's image build trusts the frontend's CA.
|
||||
|
||||
Hijacking a TLS host for a build that does not install the CA turns every
|
||||
`pip install` into CERTIFICATE_VERIFY_FAILED, so the plain-HTTP hosts are
|
||||
the only ones safe by default. A Dockerfile that runs the CA installer
|
||||
earns the rest.
|
||||
"""
|
||||
build = service.get("build")
|
||||
context = build.get("context", ".") if isinstance(build, dict) else str(build)
|
||||
name = (
|
||||
build.get("dockerfile", "Dockerfile")
|
||||
if isinstance(build, dict)
|
||||
else "Dockerfile"
|
||||
)
|
||||
path = Path(context) / name
|
||||
try:
|
||||
return _CA_INSTALLER in path.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _buildable_hostnames(service: dict) -> list[str]:
|
||||
"""Return the hosts this service's build may safely resolve to the cache."""
|
||||
if _installs_frontend_ca(service):
|
||||
return _cache_http_hostnames()
|
||||
return _cache_http_only_hostnames()
|
||||
|
||||
|
||||
def _cache_frontend_ca_present() -> bool:
|
||||
@@ -113,10 +156,24 @@ def generate_cache_override(project_dir: Path, base_compose: Path) -> Path | Non
|
||||
if not services_with_build:
|
||||
return None
|
||||
|
||||
extra_hosts = [f"{host}:{cache_ip}" for host in _CACHE_HTTP_HOSTNAMES]
|
||||
hostnames = _cache_http_hostnames()
|
||||
if not hostnames:
|
||||
raise SystemExit(
|
||||
f"the package cache is active but {_CACHE_HOSTS_KEY} is empty, so "
|
||||
"every build would silently fetch from upstream; run `make dotenv` "
|
||||
"to derive it from the roles' cache: declarations"
|
||||
)
|
||||
|
||||
override_doc = {
|
||||
"services": {
|
||||
name: {"build": {"extra_hosts": list(extra_hosts)}}
|
||||
name: {
|
||||
"build": {
|
||||
"extra_hosts": [
|
||||
f"{host}:{cache_ip}"
|
||||
for host in _buildable_hostnames(services[name])
|
||||
]
|
||||
}
|
||||
}
|
||||
for name in services_with_build
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
---
|
||||
cache:
|
||||
repos:
|
||||
raw-argos-net:
|
||||
flavor: raw
|
||||
upstream: https://argos-net.com/ # nocheck: url the bare GET returns 404; the model paths Nexus prefixes under /v1/ answer 206
|
||||
hosts:
|
||||
argos-net.com:
|
||||
repo: raw-argos-net
|
||||
read_timeout: "1800s"
|
||||
local:
|
||||
subnet: 192.168.69.0/24
|
||||
overlay:
|
||||
|
||||
@@ -32,4 +32,4 @@ LIBRETRANSLATE_MCP_CONTRACT:
|
||||
LIBRETRANSLATE_MCP_UPSTREAM_KEY: ""
|
||||
|
||||
LIBRETRANSLATE_CACHE_CA_FILE: /opt/package-frontend-ca.crt
|
||||
LIBRETRANSLATE_CACHE_CA_INSTALLER: scripts/docker/cache/package-frontend/ca.sh
|
||||
LIBRETRANSLATE_CACHE_CA_INSTALLER: roles/svc-cache-package/files/shell/ca.sh
|
||||
-33
@@ -1,33 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck shell=bash
|
||||
# Host wrapper for the in-container cert generator.
|
||||
# See docs/contributing/environment/cache.md.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${INFINITO_CACHE_PACKAGE_FRONTEND_CA_DIR:?Source scripts/meta/env/load.sh first}"
|
||||
: "${INFINITO_CACHE_PACKAGE_FRONTEND_CERTS_DIR:?Source scripts/meta/env/load.sh first}"
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
INNER_SCRIPT="${REPO_ROOT}/compose/package-cache-frontend/cert-gen.sh"
|
||||
|
||||
if [[ ! -r "${INNER_SCRIPT}" ]]; then
|
||||
echo "[package-frontend-certs] missing inner script: ${INNER_SCRIPT}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
: "${INFINITO_CACHE_PACKAGE_FRONTEND_INIT_IMAGE:?Source scripts/meta/env/load.sh first}"
|
||||
ALPINE_IMAGE="${INFINITO_CACHE_PACKAGE_FRONTEND_INIT_IMAGE}"
|
||||
|
||||
if ! MAX_ATTEMPTS=3 RETRY_DELAY_SECONDS=10 \
|
||||
"${REPO_ROOT}/scripts/github/runner/pull_with_retry.sh" "${ALPINE_IMAGE}"; then
|
||||
echo "[package-frontend-certs] could not pull ${ALPINE_IMAGE}; falling back to a cached copy" >&2
|
||||
fi
|
||||
|
||||
exec docker run --rm \
|
||||
-v "${INFINITO_CACHE_PACKAGE_FRONTEND_CA_DIR}:/ca" \
|
||||
-v "${INFINITO_CACHE_PACKAGE_FRONTEND_CERTS_DIR}:/certs" \
|
||||
-v "${INNER_SCRIPT}:/work/cert-gen.sh:ro" \
|
||||
--entrypoint /bin/sh \
|
||||
"${ALPINE_IMAGE}" \
|
||||
/work/cert-gen.sh
|
||||
Vendored
-207
@@ -1,207 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck shell=bash
|
||||
# Idempotent Nexus 3 OSS proxy bootstrap.
|
||||
# See docs/contributing/environment/cache.md.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${INFINITO_CACHE_PACKAGE_HOST_PATH:?Source scripts/meta/env/load.sh first}"
|
||||
: "${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD:?Source scripts/meta/env/load.sh first}"
|
||||
: "${INFINITO_CACHE_PACKAGE_BLOBSTORE_MAX:?Source scripts/meta/env/load.sh first}"
|
||||
: "${INFINITO_CACHE_PACKAGE_MAX_AGE_MIN:?Source scripts/meta/env/load.sh first}"
|
||||
: "${INFINITO_CACHE_PACKAGE_METADATA_MAX_AGE_MIN:?Source scripts/meta/env/load.sh first}"
|
||||
: "${INFINITO_APT_UBUNTU_MIRRORS:?Source scripts/meta/env/load.sh first}"
|
||||
|
||||
CACHE_CONTENT_MAX_AGE_MIN="${INFINITO_CACHE_PACKAGE_MAX_AGE_MIN}"
|
||||
CACHE_METADATA_MAX_AGE_MIN="${INFINITO_CACHE_PACKAGE_METADATA_MAX_AGE_MIN}"
|
||||
|
||||
read -r -a UBUNTU_MIRRORS <<<"${INFINITO_APT_UBUNTU_MIRRORS}"
|
||||
UBUNTU_PRIMARY="${UBUNTU_MIRRORS[0]%/}"
|
||||
UBUNTU_FALLBACK="${UBUNTU_MIRRORS[1]:?INFINITO_APT_UBUNTU_MIRRORS needs a primary and a fallback mirror}"
|
||||
UBUNTU_FALLBACK="${UBUNTU_FALLBACK%/}"
|
||||
|
||||
PKGCACHE_CONTAINER="infinito-package-cache"
|
||||
NEXUS_REST="http://127.0.0.1:8081/service/rest"
|
||||
|
||||
BOOTSTRAP_DONE_FILE="/nexus-data/.infinito-bootstrap-done"
|
||||
NEXUS_ADMIN_PW_FILE="/nexus-data/.infinito-admin-password"
|
||||
NEXUS_DEFAULT_PASSWORD="admin123"
|
||||
|
||||
log() { printf '[package-cache-bootstrap] %s\n' "$*" >&2; }
|
||||
|
||||
nexus_curl() { docker exec "${PKGCACHE_CONTAINER}" curl "$@"; }
|
||||
|
||||
wait_for_nexus() {
|
||||
local _attempt
|
||||
for _attempt in $(seq 1 120); do
|
||||
if nexus_curl -fsS "${NEXUS_REST}/v1/status" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
log "Nexus REST not reachable at ${NEXUS_REST}/v1/status after 4 minutes"
|
||||
return 1
|
||||
}
|
||||
|
||||
store_admin_password() {
|
||||
printf %s "${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD}" |
|
||||
docker exec -i "${PKGCACHE_CONTAINER}" sh -c "umask 077; cat > '${NEXUS_ADMIN_PW_FILE}'"
|
||||
}
|
||||
|
||||
rotate_admin_password() {
|
||||
ADMIN_USER="admin"
|
||||
if docker exec "${PKGCACHE_CONTAINER}" test -f "${BOOTSTRAP_DONE_FILE}"; then
|
||||
log "Already bootstrapped; using stored admin password"
|
||||
ADMIN_PASS="$(docker exec "${PKGCACHE_CONTAINER}" cat "${NEXUS_ADMIN_PW_FILE}" 2>/dev/null || true)"
|
||||
if [ -z "${ADMIN_PASS}" ]; then
|
||||
ADMIN_PASS="${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD}"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Rotating admin password from default '${NEXUS_DEFAULT_PASSWORD}'"
|
||||
if ! nexus_curl -fsS -u "admin:${NEXUS_DEFAULT_PASSWORD}" \
|
||||
-H "Content-Type: text/plain" \
|
||||
-X PUT "${NEXUS_REST}/v1/security/users/admin/change-password" \
|
||||
--data-binary "${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD}"; then
|
||||
if nexus_curl -fsS -o /dev/null -u "admin:${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD}" \
|
||||
"${NEXUS_REST}/v1/repositories" >/dev/null 2>&1; then
|
||||
log "Default rotation rejected but configured password authenticates; assuming previous bootstrap, marking done"
|
||||
else
|
||||
log "Cannot authenticate with default '${NEXUS_DEFAULT_PASSWORD}' nor with configured INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD."
|
||||
log "Wipe ${INFINITO_CACHE_PACKAGE_HOST_PATH} (e.g. via 'make clean-cache') and retry."
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
store_admin_password
|
||||
docker exec "${PKGCACHE_CONTAINER}" touch "${BOOTSTRAP_DONE_FILE}"
|
||||
ADMIN_PASS="${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD}"
|
||||
}
|
||||
|
||||
ensure_blobstore() {
|
||||
local quota_gb="${INFINITO_CACHE_PACKAGE_BLOBSTORE_MAX%g}"
|
||||
local quota_mb=$((quota_gb * 1024))
|
||||
local payload
|
||||
payload="$(printf '{"name":"default","path":"default","softQuota":{"type":"spaceUsedQuota","limit":%d}}' "${quota_mb}")"
|
||||
local code
|
||||
code="$(nexus_curl -sS -o /dev/null -w '%{http_code}' \
|
||||
-u "${ADMIN_USER}:${ADMIN_PASS}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "${NEXUS_REST}/v1/blobstores/file" \
|
||||
--data "${payload}" || true)"
|
||||
case "${code}" in
|
||||
201 | 204) log "blobstore default created (HTTP ${code})" ;;
|
||||
400 | 409) log "blobstore default already exists (HTTP ${code})" ;;
|
||||
*)
|
||||
log "unexpected HTTP ${code} creating blobstore"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
ensure_repo() {
|
||||
local format="$1" kind="$2" name="$3" payload="$4"
|
||||
local code
|
||||
code="$(nexus_curl -sS -o /dev/null -w '%{http_code}' \
|
||||
-u "${ADMIN_USER}:${ADMIN_PASS}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "${NEXUS_REST}/v1/repositories/${format}/${kind}" \
|
||||
--data "${payload}" || true)"
|
||||
case "${code}" in
|
||||
201 | 204) log "${format} ${kind} ${name} created (HTTP ${code})" ;;
|
||||
400 | 409)
|
||||
code="$(nexus_curl -sS -o /dev/null -w '%{http_code}' \
|
||||
-u "${ADMIN_USER}:${ADMIN_PASS}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X PUT "${NEXUS_REST}/v1/repositories/${format}/${kind}/${name}" \
|
||||
--data "${payload}" || true)"
|
||||
case "${code}" in
|
||||
200 | 204) log "${format} ${kind} ${name} updated (HTTP ${code})" ;;
|
||||
*)
|
||||
log "unexpected HTTP ${code} updating ${format} ${kind} ${name}"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
log "unexpected HTTP ${code} creating ${format} ${kind} ${name}"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
ensure_proxy_repo() { ensure_repo "$1" proxy "$2" "$3"; }
|
||||
|
||||
# autoBlock stays off: once Nexus blocks a remote it answers every client
|
||||
# with `404 Remote Auto Blocked` for the whole block window, so a single
|
||||
# upstream hiccup fails every concurrent job instead of just the request
|
||||
# that hit it.
|
||||
ensure_all_proxies() {
|
||||
local storage='"storage":{"blobStoreName":"default","strictContentTypeValidation":true},"proxy":{"contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"negativeCache":{"enabled":true,"timeToLive":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"httpClient":{"blocked":false,"autoBlock":false}'
|
||||
|
||||
ensure_proxy_repo apt apt-debian "$(printf '{"name":"apt-debian","online":true,%s,"proxy":{"remoteUrl":"http://deb.debian.org/debian","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"bookworm","flat":false}}' "${storage}")"
|
||||
ensure_proxy_repo apt apt-ubuntu "$(printf '{"name":"apt-ubuntu","online":true,%s,"proxy":{"remoteUrl":"'"${UBUNTU_PRIMARY}"'","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"jammy","flat":false}}' "${storage}")"
|
||||
ensure_proxy_repo apt apt-debian-security "$(printf '{"name":"apt-debian-security","online":true,%s,"proxy":{"remoteUrl":"http://deb.debian.org/debian-security","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"bookworm-security","flat":false}}' "${storage}")" # nocheck: url (bare GET on the deb.debian.org redirector returns 404; apt-client paths underneath dists/bookworm-security/ resolve correctly)
|
||||
ensure_proxy_repo apt apt-ubuntu-security "$(printf '{"name":"apt-ubuntu-security","online":true,%s,"proxy":{"remoteUrl":"'"${UBUNTU_PRIMARY}"'","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"jammy-security","flat":false}}' "${storage}")"
|
||||
ensure_proxy_repo apt apt-debian-mirror "$(printf '{"name":"apt-debian-mirror","online":true,%s,"proxy":{"remoteUrl":"http://ftp.debian.org/debian","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"bookworm","flat":false}}' "${storage}")"
|
||||
ensure_proxy_repo apt apt-ubuntu-mirror "$(printf '{"name":"apt-ubuntu-mirror","online":true,%s,"proxy":{"remoteUrl":"'"${UBUNTU_FALLBACK}"'","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"jammy","flat":false}}' "${storage}")"
|
||||
ensure_proxy_repo apt apt-debian-security-mirror "$(printf '{"name":"apt-debian-security-mirror","online":true,%s,"proxy":{"remoteUrl":"http://security.debian.org/debian-security","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"bookworm-security","flat":false}}' "${storage}")"
|
||||
ensure_proxy_repo apt apt-ubuntu-security-mirror "$(printf '{"name":"apt-ubuntu-security-mirror","online":true,%s,"proxy":{"remoteUrl":"'"${UBUNTU_FALLBACK}"'","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"jammy-security","flat":false}}' "${storage}")"
|
||||
ensure_proxy_repo pypi pypi-proxy "$(printf '{"name":"pypi-proxy","online":true,%s,"proxy":{"remoteUrl":"https://pypi.org/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'}}' "${storage}")"
|
||||
ensure_proxy_repo npm npm-proxy "$(printf '{"name":"npm-proxy","online":true,%s,"proxy":{"remoteUrl":"https://registry.npmjs.org/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'}}' "${storage}")"
|
||||
ensure_proxy_repo helm helm-bitnami "$(printf '{"name":"helm-bitnami","online":true,%s,"proxy":{"remoteUrl":"https://charts.bitnami.com/bitnami","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'}}' "${storage}")"
|
||||
ensure_proxy_repo raw raw-githubusercontent "$(printf '{"name":"raw-githubusercontent","online":true,%s,"proxy":{"remoteUrl":"https://raw.githubusercontent.com/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"raw":{"contentDisposition":"ATTACHMENT"}}' "${storage}")"
|
||||
ensure_proxy_repo raw raw-codeload-github "$(printf '{"name":"raw-codeload-github","online":true,%s,"proxy":{"remoteUrl":"https://codeload.github.com/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"raw":{"contentDisposition":"ATTACHMENT"}}' "${storage}")"
|
||||
ensure_proxy_repo raw raw-argos-net "$(printf '{"name":"raw-argos-net","online":true,%s,"proxy":{"remoteUrl":"https://argos-net.com/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"raw":{"contentDisposition":"ATTACHMENT"}}' "${storage}")" # nocheck: url (bare GET on argos-net.com returns 404, measured; the model paths Nexus prefixes underneath /v1/ answer 206)
|
||||
ensure_proxy_repo rubygems gem-proxy "$(printf '{"name":"gem-proxy","online":true,%s,"proxy":{"remoteUrl":"https://rubygems.org/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'}}' "${storage}")"
|
||||
ensure_proxy_repo go go-proxy "$(printf '{"name":"go-proxy","online":true,%s,"proxy":{"remoteUrl":"https://proxy.golang.org/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'}}' "${storage}")"
|
||||
ensure_proxy_repo yum yum-rocky "$(printf '{"name":"yum-rocky","online":true,%s,"proxy":{"remoteUrl":"https://download.rockylinux.org/pub/rocky/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"yum":{"repodataDepth":5}}' "${storage}")"
|
||||
ensure_proxy_repo yum yum-fedora "$(printf '{"name":"yum-fedora","online":true,%s,"proxy":{"remoteUrl":"https://download.fedoraproject.org/pub/fedora/linux/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"yum":{"repodataDepth":5}}' "${storage}")"
|
||||
ensure_proxy_repo raw raw-packagist "$(printf '{"name":"raw-packagist","online":true,%s,"proxy":{"remoteUrl":"https://repo.packagist.org/","contentMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"raw":{"contentDisposition":"ATTACHMENT"}}' "${storage}")" # nocheck: url
|
||||
ensure_proxy_repo raw raw-alpine "$(printf '{"name":"raw-alpine","online":true,%s,"proxy":{"remoteUrl":"https://dl-cdn.alpinelinux.org/alpine/","contentMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"raw":{"contentDisposition":"ATTACHMENT"}}' "${storage}")"
|
||||
}
|
||||
|
||||
ensure_eula_accepted() {
|
||||
local code
|
||||
# shellcheck disable=SC1112
|
||||
code="$(nexus_curl -sS -o /dev/null -w '%{http_code}' \
|
||||
-u "${ADMIN_USER}:${ADMIN_PASS}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "${NEXUS_REST}/v1/system/eula" \
|
||||
--data '{"accepted":true,"disclaimer":"Use of Sonatype Nexus Repository - Community Edition is governed by the End User License Agreement at https://links.sonatype.com/products/nxrm/ce-eula. By returning the value from ‘accepted:false’ to ‘accepted:true’, you acknowledge that you have read and agree to the End User License Agreement at https://links.sonatype.com/products/nxrm/ce-eula."}' || true)"
|
||||
case "${code}" in
|
||||
200 | 204) log "EULA accepted (HTTP ${code})" ;;
|
||||
*)
|
||||
log "unexpected HTTP ${code} accepting EULA"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
ensure_anonymous_access() {
|
||||
local code
|
||||
code="$(nexus_curl -sS -o /dev/null -w '%{http_code}' \
|
||||
-u "${ADMIN_USER}:${ADMIN_PASS}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X PUT "${NEXUS_REST}/v1/security/anonymous" \
|
||||
--data '{"enabled":true,"userId":"anonymous","realmName":"NexusAuthorizingRealm"}' || true)"
|
||||
case "${code}" in
|
||||
200 | 204) log "anonymous access enabled (HTTP ${code})" ;;
|
||||
*)
|
||||
log "unexpected HTTP ${code} enabling anonymous access"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main() {
|
||||
wait_for_nexus
|
||||
rotate_admin_password
|
||||
ensure_eula_accepted
|
||||
ensure_anonymous_access
|
||||
ensure_blobstore
|
||||
ensure_all_proxies
|
||||
log "bootstrap done"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -68,6 +68,7 @@ if docker inspect "${CACHE_FRONTEND}" >/dev/null 2>&1; then
|
||||
fi
|
||||
export INFINITO_CACHE_REGISTRY_CA_HOST_PATH="${registry_ca_src}"
|
||||
echo "==> registry-cache CA source: ${INFINITO_CACHE_REGISTRY_CA_HOST_PATH}"
|
||||
python3 -m cli.meta.cache
|
||||
COMPOSE_ARGS+=(-f "${SCRIPT_DIR}/../../../../../compose/swarm/cache.override.yml")
|
||||
fi
|
||||
|
||||
|
||||
@@ -172,10 +172,11 @@
|
||||
service_loader_preloading: true
|
||||
changed_when: false
|
||||
|
||||
- name: "🥾 Bootstrap NFS clients + registry + sandbox runtimes (MUST precede Preload)"
|
||||
- name: "🥾 Bootstrap storage + cache + registry + runtimes (MUST precede Preload)"
|
||||
ansible.builtin.include_tasks: ./tasks/utils/setup/group.yml
|
||||
loop:
|
||||
- svc-storage
|
||||
- svc-cache
|
||||
- svc-registry
|
||||
- svc-virt
|
||||
loop_control:
|
||||
|
||||
@@ -111,15 +111,55 @@ class TestInfinitoComposeWrapper(unittest.TestCase):
|
||||
env = {
|
||||
"INFINITO_CACHE_PACKAGE_FRONTEND_IP": "172.30.0.4",
|
||||
"INFINITO_CACHE_PACKAGE_FRONTEND_CA_FILE": str(ca),
|
||||
"INFINITO_CACHE_HOSTS": "first.example.org,plain.example.net",
|
||||
"INFINITO_CACHE_HTTP_HOSTS": "plain.example.net",
|
||||
}
|
||||
with patch.dict(os.environ, env, clear=False):
|
||||
out = s.generate_cache_override(proj, base)
|
||||
self.assertIsNotNone(out)
|
||||
self.assertIn("deb.debian.org:172.30.0.4", read_text(str(out)))
|
||||
rendered = read_text(str(out))
|
||||
self.assertIn("plain.example.net:172.30.0.4", rendered)
|
||||
self.assertNotIn(
|
||||
"first.example.org",
|
||||
rendered,
|
||||
"a build that does not install the frontend CA must not have a "
|
||||
"TLS host hijacked onto the self-signed frontend",
|
||||
)
|
||||
with (
|
||||
patch.dict(os.environ, {**env, "INFINITO_CACHE_HOSTS": ""}),
|
||||
self.assertRaises(SystemExit),
|
||||
):
|
||||
s.generate_cache_override(proj, base)
|
||||
ca.unlink()
|
||||
with patch.dict(os.environ, env, clear=False):
|
||||
self.assertIsNone(s.generate_cache_override(proj, base))
|
||||
|
||||
def test_a_build_that_trusts_the_ca_also_gets_the_tls_hosts(self):
|
||||
s = self.script
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
proj = Path(td)
|
||||
(proj / "Dockerfile").write_text(
|
||||
"FROM alpine\nRUN /usr/local/bin/package-frontend-ca.sh\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
base = proj / "compose.yml"
|
||||
base.write_text(
|
||||
f"services:\n app:\n build:\n context: {proj}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
ca = proj / "ca.crt"
|
||||
ca.write_text("CERT", encoding="utf-8")
|
||||
env = {
|
||||
"INFINITO_CACHE_PACKAGE_FRONTEND_IP": "172.30.0.4",
|
||||
"INFINITO_CACHE_PACKAGE_FRONTEND_CA_FILE": str(ca),
|
||||
"INFINITO_CACHE_HOSTS": "first.example.org,plain.example.net",
|
||||
"INFINITO_CACHE_HTTP_HOSTS": "plain.example.net",
|
||||
}
|
||||
with patch.dict(os.environ, env, clear=False):
|
||||
rendered = read_text(str(s.generate_cache_override(proj, base)))
|
||||
self.assertIn("first.example.org:172.30.0.4", rendered)
|
||||
self.assertIn("plain.example.net:172.30.0.4", rendered)
|
||||
|
||||
def test_build_cmd_contains_env_and_files(self):
|
||||
s = self.script
|
||||
base = Path("/proj")
|
||||
|
||||
Vendored
+306
@@ -0,0 +1,306 @@
|
||||
"""Collect what the package cache proxies, from ``cache:`` in meta/networks.yml.
|
||||
|
||||
The single point of truth for the cache. A role declares the repositories it
|
||||
needs Nexus to proxy and, where a container must resolve an upstream by name,
|
||||
the hostname the frontend answers for::
|
||||
|
||||
# roles/<role>/meta/networks.yml
|
||||
cache:
|
||||
repos:
|
||||
raw-example:
|
||||
flavor: raw
|
||||
upstream: https://example.org/
|
||||
hosts:
|
||||
example.org:
|
||||
repo: raw-example
|
||||
read_timeout: 1800s
|
||||
|
||||
A repository may serve several hosts and a host may be served by one
|
||||
repository only; a repository without a host is proxied but never hijacked,
|
||||
which is how a client that already addresses Nexus directly is served.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from utils.cache.files import read_text
|
||||
from utils.roles.mapping import ROLE_FILE_META_NETWORKS
|
||||
from utils.yaml_bootstrap import BootstrapYamlError, load_block
|
||||
|
||||
from . import PROJECT_ROOT
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
ROLES_DIR = PROJECT_ROOT / "roles"
|
||||
HTTPS_PORT = 443
|
||||
HTTP_PORT = 80
|
||||
CACHE_KEY = "cache"
|
||||
INDENT = " "
|
||||
TRUE = "true"
|
||||
COMMENT = "#"
|
||||
|
||||
|
||||
def _opens_cache(line: str) -> bool:
|
||||
"""Whether *line* is the ``cache:`` key itself, comment and all.
|
||||
|
||||
A trailing comment on the key would otherwise make the whole block
|
||||
invisible, which reads exactly like a role that declares nothing.
|
||||
"""
|
||||
head = line.split(COMMENT, 1)[0].rstrip()
|
||||
return head == f"{CACHE_KEY}:"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CachedRepo:
|
||||
"""One repository Nexus proxies.
|
||||
|
||||
Args:
|
||||
name: the Nexus repository name.
|
||||
flavor: its repository type, such as ``apt`` or ``raw``.
|
||||
upstream: the absolute URL it proxies.
|
||||
distribution: the suite an ``apt`` repository serves.
|
||||
repodata_depth: the depth a ``yum`` repository indexes.
|
||||
mirror: a sibling repository proxying a different host, which the
|
||||
frontend falls back to when this one answers 5xx.
|
||||
content_max_age: minutes Nexus serves a cached artefact for, where the
|
||||
repository needs something other than the stack-wide default.
|
||||
role: the role that declared it.
|
||||
"""
|
||||
|
||||
name: str
|
||||
flavor: str
|
||||
upstream: str
|
||||
distribution: str = ""
|
||||
repodata_depth: int = 0
|
||||
mirror: str = ""
|
||||
content_max_age: str = ""
|
||||
role: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CachedHost:
|
||||
"""One hostname the frontend answers for, in place of its upstream.
|
||||
|
||||
Args:
|
||||
host: the hostname a container resolves to the frontend.
|
||||
repo: the repository serving its root, when it serves only one.
|
||||
paths: the repository behind each path, for a host whose suites live
|
||||
side by side, as Debian's do under /debian/ and /debian-security/.
|
||||
ports: the frontend ports serving it; apt suites are plain HTTP.
|
||||
read_timeout: nginx ``proxy_read_timeout`` where the default is too
|
||||
short, as for a multi-gigabyte model download.
|
||||
passthrough: the upstream hands out URLs that already carry the
|
||||
repository prefix, as npm does for tarballs, so those reach the
|
||||
backend unprefixed instead of being prefixed twice into a 404.
|
||||
role: the role that declared it.
|
||||
"""
|
||||
|
||||
host: str
|
||||
repo: str = ""
|
||||
paths: tuple[tuple[str, str], ...] = ()
|
||||
listen: tuple[int, ...] = (HTTPS_PORT,)
|
||||
read_timeout: str = ""
|
||||
passthrough: bool = False
|
||||
role: str = ""
|
||||
|
||||
def routes(self) -> tuple[tuple[str, str], ...]:
|
||||
"""Return the (path, repository) pairs this host serves."""
|
||||
return self.paths or ((("/"), self.repo),)
|
||||
|
||||
def listeners(self) -> tuple[tuple[int, bool], ...]:
|
||||
"""Return the (port, terminates TLS) pairs this host is served on."""
|
||||
return tuple((port, port == HTTPS_PORT) for port in self.listen)
|
||||
|
||||
def scheme(self) -> str:
|
||||
"""Return the scheme a client addresses this host with."""
|
||||
return "https" if HTTPS_PORT in self.listen else "http"
|
||||
|
||||
def path_of(self, repo: str) -> str:
|
||||
"""Return the path this host serves *repo* under, or the empty string."""
|
||||
return next((path for path, name in self.routes() if name == repo), "")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Declarations:
|
||||
"""Everything the roles declare, collected.
|
||||
|
||||
Args:
|
||||
repos: the proxied repositories, sorted by name.
|
||||
hosts: the hijacked hostnames, sorted by host.
|
||||
"""
|
||||
|
||||
repos: list[CachedRepo] = field(default_factory=list)
|
||||
hosts: list[CachedHost] = field(default_factory=list)
|
||||
|
||||
|
||||
def _cache_block(role_dir: Path) -> dict:
|
||||
"""Return the ``cache:`` mapping of one role's networks file.
|
||||
|
||||
Only that block is parsed, and only through the bootstrap reader: the env
|
||||
generator reaches this module before PyYAML exists, and the rest of the
|
||||
file carries constructs (a subnet, for one) the reader rejects.
|
||||
"""
|
||||
try:
|
||||
text = read_text(str(role_dir / ROLE_FILE_META_NETWORKS))
|
||||
except OSError:
|
||||
return {}
|
||||
captured: list[str] = []
|
||||
inside = False
|
||||
for number, line in enumerate(text.splitlines(), start=1):
|
||||
if not inside:
|
||||
inside = _opens_cache(line)
|
||||
continue
|
||||
if not line.strip():
|
||||
captured.append(line)
|
||||
continue
|
||||
if not line.startswith(INDENT):
|
||||
if line.lstrip().startswith(COMMENT):
|
||||
raise BootstrapYamlError(
|
||||
f"{role_dir.name}: line {number} sits at column 0 inside the "
|
||||
f"cache: block. The reader would stop there and silently drop "
|
||||
f"every declaration below it; indent the comment instead."
|
||||
)
|
||||
break
|
||||
captured.append(line[len(INDENT) :])
|
||||
if not any(line.strip() for line in captured):
|
||||
return {}
|
||||
block = load_block("\n".join(captured))
|
||||
return block if isinstance(block, dict) else {}
|
||||
|
||||
|
||||
def _section(role_dir: Path, name: str) -> Iterator[tuple[str, dict]]:
|
||||
entries = _cache_block(role_dir).get(name)
|
||||
if not isinstance(entries, dict):
|
||||
return
|
||||
for key, entry in entries.items():
|
||||
if isinstance(entry, dict):
|
||||
yield str(key), entry
|
||||
|
||||
|
||||
def _repos(role_dir: Path) -> Iterator[CachedRepo]:
|
||||
for name, entry in _section(role_dir, "repos"):
|
||||
yield CachedRepo(
|
||||
name=name,
|
||||
flavor=str(entry.get("flavor", "raw")),
|
||||
upstream=str(entry.get("upstream", "")),
|
||||
distribution=str(entry.get("distribution", "")),
|
||||
repodata_depth=int(entry.get("repodata_depth", 0)),
|
||||
mirror=str(entry.get("mirror", "")),
|
||||
content_max_age=str(entry.get("content_max_age", "")),
|
||||
role=role_dir.name,
|
||||
)
|
||||
|
||||
|
||||
def _hosts(role_dir: Path) -> Iterator[CachedHost]:
|
||||
for host, entry in _section(role_dir, "hosts"):
|
||||
paths = entry.get("paths") or {}
|
||||
yield CachedHost(
|
||||
host=host,
|
||||
repo=str(entry.get("repo", "")),
|
||||
paths=tuple((str(k), str(v)) for k, v in paths.items()),
|
||||
listen=tuple(int(port) for port in entry.get("ports") or [HTTPS_PORT]),
|
||||
read_timeout=str(entry.get("read_timeout", "")),
|
||||
passthrough=str(entry.get("passthrough", "")).lower() == TRUE,
|
||||
role=role_dir.name,
|
||||
)
|
||||
|
||||
|
||||
def declarations() -> Declarations:
|
||||
"""Return every declared repository and host.
|
||||
|
||||
Raises:
|
||||
ValueError: two roles declare the same repository or host differently,
|
||||
or a host names a repository nobody declares.
|
||||
"""
|
||||
repos: dict[str, CachedRepo] = {}
|
||||
hosts: dict[str, CachedHost] = {}
|
||||
if not ROLES_DIR.is_dir():
|
||||
return Declarations()
|
||||
for role_dir in sorted(p for p in ROLES_DIR.iterdir() if p.is_dir()):
|
||||
for repo in _repos(role_dir):
|
||||
_claim(repos, repo.name, repo, "repository")
|
||||
for host in _hosts(role_dir):
|
||||
_claim(hosts, host.host, host, "host")
|
||||
missing = sorted(
|
||||
f"{host.host} (declared by {host.role}) -> {name or '<none>'}"
|
||||
for host in hosts.values()
|
||||
for _, name in host.routes()
|
||||
if name not in repos
|
||||
)
|
||||
if missing:
|
||||
raise ValueError("hosts naming an undeclared repository: " + ", ".join(missing))
|
||||
return Declarations(
|
||||
repos=[repos[name] for name in sorted(repos)],
|
||||
hosts=[hosts[host] for host in sorted(hosts)],
|
||||
)
|
||||
|
||||
|
||||
def hosts_serving(repo: str) -> list[CachedHost]:
|
||||
"""Return every host the frontend answers for on behalf of *repo*."""
|
||||
return [
|
||||
entry
|
||||
for entry in declarations().hosts
|
||||
if any(name == repo for _, name in entry.routes())
|
||||
]
|
||||
|
||||
|
||||
def repo_named(name: str) -> CachedRepo | None:
|
||||
"""Return the declared repository *name*, or None where none declares it."""
|
||||
return next((repo for repo in declarations().repos if repo.name == name), None)
|
||||
|
||||
|
||||
def origin_of(repo: CachedRepo) -> str:
|
||||
"""Return the hostname *repo* proxies, taken from its upstream URL."""
|
||||
return urlsplit(repo.upstream).hostname or ""
|
||||
|
||||
|
||||
def host_records() -> dict[str, dict]:
|
||||
"""Return every declared host as a template-ready record, keyed by host.
|
||||
|
||||
The shape every consumer renders from: the ansible lookups, which then
|
||||
apply the inventory, and the dev stack, which does not.
|
||||
"""
|
||||
return {
|
||||
entry.host: {
|
||||
"host": entry.host,
|
||||
"repo": entry.repo or entry.routes()[0][1],
|
||||
"routes": [{"path": path, "repo": repo} for path, repo in entry.routes()],
|
||||
"listen": [{"port": port, "tls": tls} for port, tls in entry.listeners()],
|
||||
"tls": HTTPS_PORT in entry.listen,
|
||||
"read_timeout": entry.read_timeout,
|
||||
"passthrough": entry.passthrough,
|
||||
"role": entry.role,
|
||||
}
|
||||
for entry in declarations().hosts
|
||||
}
|
||||
|
||||
|
||||
def repo_records() -> dict[str, dict]:
|
||||
"""Return every declared repository as a record, keyed by name."""
|
||||
return {
|
||||
repo.name: {
|
||||
"name": repo.name,
|
||||
"flavor": repo.flavor,
|
||||
"upstream": repo.upstream,
|
||||
"distribution": repo.distribution,
|
||||
"repodata_depth": repo.repodata_depth,
|
||||
"mirror": repo.mirror,
|
||||
"content_max_age": repo.content_max_age,
|
||||
"role": repo.role,
|
||||
}
|
||||
for repo in declarations().repos
|
||||
}
|
||||
|
||||
|
||||
def _claim(seen: dict, key: str, found, kind: str) -> None:
|
||||
first = seen.setdefault(key, found)
|
||||
if first != found:
|
||||
raise ValueError(
|
||||
f"{kind} {key} is declared by {first.role} and again, differently, "
|
||||
f"by {found.role}"
|
||||
)
|
||||
Vendored
+65
@@ -0,0 +1,65 @@
|
||||
"""Apply a host's inventory to the cache declarations the roles make.
|
||||
|
||||
The dev compose stack has no inventory and caches exactly what the roles
|
||||
declare. A deployed host caches what its configuration says, which starts
|
||||
from the same declarations and lets the inventory correct or extend them::
|
||||
|
||||
applications:
|
||||
svc-cache-package:
|
||||
repos:
|
||||
raw-example:
|
||||
upstream: https://mirror.example.net/
|
||||
upstreams:
|
||||
example.org:
|
||||
read_timeout: 1800s
|
||||
|
||||
Both sections are merged per entry, so an inventory that names one field
|
||||
keeps the rest of the declaration, and a key nobody declared is added.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
ROLE = "svc-cache-package"
|
||||
|
||||
|
||||
def configured(loader: Any, templar: Any, variables: Any, section: str) -> dict:
|
||||
"""Return the inventory's entries for one ``cache:`` section.
|
||||
|
||||
Args:
|
||||
loader: the lookup plugin's loader.
|
||||
templar: the lookup plugin's templar, where it has one.
|
||||
variables: the task variables the lookup was called with.
|
||||
section: ``repos`` or ``upstreams``.
|
||||
|
||||
Returns:
|
||||
The configured entries, or an empty mapping where no inventory
|
||||
answers, as in the dev stack.
|
||||
"""
|
||||
from ansible.errors import AnsibleError
|
||||
from ansible.plugins.loader import lookup_loader
|
||||
|
||||
try:
|
||||
applications = lookup_loader.get(
|
||||
"applications", loader=loader, templar=templar
|
||||
).run([], variables=variables or {})[0]
|
||||
except (AnsibleError, KeyError, TypeError):
|
||||
return {}
|
||||
role = applications.get(ROLE) if isinstance(applications, dict) else None
|
||||
found = (role or {}).get(section) if isinstance(role, dict) else None
|
||||
return found if isinstance(found, dict) else {}
|
||||
|
||||
|
||||
def merge(declared: dict[str, dict], overrides: dict) -> dict[str, dict]:
|
||||
"""Return *declared* with *overrides* applied, sorted by key.
|
||||
|
||||
Args:
|
||||
declared: the entries the roles declare, keyed by name.
|
||||
overrides: the inventory's entries for the same section.
|
||||
"""
|
||||
merged = {key: dict(value) for key, value in declared.items()}
|
||||
for key, entry in (overrides or {}).items():
|
||||
if isinstance(entry, dict):
|
||||
merged.setdefault(str(key), {}).update(entry)
|
||||
return {key: merged[key] for key in sorted(merged)}
|
||||
Vendored
+82
@@ -0,0 +1,82 @@
|
||||
"""Render the declared repositories as the records Nexus is bootstrapped from.
|
||||
|
||||
One format, two readers: the ``.env`` generator writes it to
|
||||
``INFINITO_CACHE_UPSTREAMS`` for the dev stack, and the ``cache_records``
|
||||
lookup hands the same string to the role's bootstrap task. Both resolve the
|
||||
Ubuntu mirror placeholders from ``INFINITO_APT_UBUNTU_MIRRORS``, which the
|
||||
image build reads too, so the mirror order has a single home.
|
||||
|
||||
Stdlib only: the ``.env`` generator reaches this before PyYAML exists.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from utils.cache.hosts import declarations
|
||||
|
||||
MIRRORS_KEY = "INFINITO_APT_UBUNTU_MIRRORS"
|
||||
PRIMARY = "${UBUNTU_PRIMARY}"
|
||||
FALLBACK = "${UBUNTU_FALLBACK}"
|
||||
PLACEHOLDER = "${"
|
||||
FIELD = "|"
|
||||
RECORD = ","
|
||||
|
||||
|
||||
def mirrors(raw: str = "") -> tuple[str, str] | None:
|
||||
"""Return the primary and fallback Ubuntu archive, without trailing slash.
|
||||
|
||||
Args:
|
||||
raw: the mirror list, or empty to read it from the environment.
|
||||
|
||||
Returns:
|
||||
None where the list does not carry both, in which case the
|
||||
placeholders cannot be resolved.
|
||||
"""
|
||||
found = [
|
||||
entry.rstrip("/")
|
||||
for entry in (raw or os.environ.get(MIRRORS_KEY, "")).strip('"').split()
|
||||
if entry
|
||||
]
|
||||
if len(found) < 2:
|
||||
return None
|
||||
return found[0], found[1]
|
||||
|
||||
|
||||
def upstream(raw: str, resolved: tuple[str, str] | None) -> str:
|
||||
"""Return *raw* with the Ubuntu mirror placeholders substituted."""
|
||||
if resolved is None:
|
||||
return raw
|
||||
return raw.replace(PRIMARY, resolved[0]).replace(FALLBACK, resolved[1])
|
||||
|
||||
|
||||
def records(raw_mirrors: str = "") -> str:
|
||||
"""Return every repository as ``name|flavor|url|suite|depth|max_age``.
|
||||
|
||||
Args:
|
||||
raw_mirrors: the mirror list, or empty to read it from the environment.
|
||||
|
||||
Returns:
|
||||
The comma-separated records, or the empty string where a placeholder
|
||||
stays unresolved, so a consumer's own guard reports it rather than
|
||||
Nexus proxying a host that does not exist.
|
||||
"""
|
||||
resolved = mirrors(raw_mirrors)
|
||||
rendered = []
|
||||
for repo in declarations().repos:
|
||||
url = upstream(repo.upstream, resolved)
|
||||
if PLACEHOLDER in url:
|
||||
return ""
|
||||
rendered.append(
|
||||
FIELD.join(
|
||||
(
|
||||
repo.name,
|
||||
repo.flavor,
|
||||
url,
|
||||
repo.distribution,
|
||||
str(repo.repodata_depth or ""),
|
||||
repo.content_max_age,
|
||||
)
|
||||
)
|
||||
)
|
||||
return RECORD.join(rendered)
|
||||
Vendored
+193
@@ -0,0 +1,193 @@
|
||||
"""Render what the package cache needs beside its role, from the declarations.
|
||||
|
||||
The role deploys the frontend and owns the templates and their tuning; the dev
|
||||
compose stack has no inventory to render them with and comes through here
|
||||
instead. Every value still has one home: the hosts and repositories come from
|
||||
the ``cache:`` declarations, the nginx tuning from the role's ``vars/main.yml``,
|
||||
and only the backend address differs, because the dev stack reaches Nexus over
|
||||
a compose network alias while the role reaches it by its configured container.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
||||
from utils.cache.hosts import (
|
||||
host_records,
|
||||
hosts_serving,
|
||||
origin_of,
|
||||
repo_named,
|
||||
repo_records,
|
||||
)
|
||||
from utils.cache.yaml import load_yaml_any
|
||||
from utils.distros import distro_names
|
||||
from utils.roles.mapping import ROLE_FILE_META_VOLUMES, ROLE_FILE_VARS_MAIN
|
||||
|
||||
from . import PROJECT_ROOT
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
ROLE = "svc-cache-package"
|
||||
ROLE_DIR = PROJECT_ROOT / "roles" / ROLE
|
||||
ROLE_VARS = ROLE_DIR / ROLE_FILE_VARS_MAIN
|
||||
TEMPLATE_DIRS = (ROLE_DIR / "templates", PROJECT_ROOT / "compose" / "templates")
|
||||
UPSTREAMS_TEMPLATE = "upstreams.conf.j2"
|
||||
CONSUMER_TEMPLATE = "cache-consumer.yml.jinja"
|
||||
CONSUMER = "compose.cache-consumer.yml"
|
||||
|
||||
PYPI_REPO = "pypi-proxy"
|
||||
NPM_REPO = "npm-proxy"
|
||||
|
||||
BACKEND = "http://package-cache:8081"
|
||||
TEMPLATED = "{{"
|
||||
|
||||
|
||||
def _mount_target(volume: str, service: str) -> str:
|
||||
"""Return where *volume* is mounted on *service*, from meta/volumes.yml.
|
||||
|
||||
The role reads the same value through the ``volume`` lookup, which needs
|
||||
an inventory; the dev stack has none and reads the declaration directly,
|
||||
so the mount point still has one home.
|
||||
|
||||
Args:
|
||||
volume: the key in meta/volumes.yml.
|
||||
service: the service whose mount to return.
|
||||
"""
|
||||
declared = load_yaml_any(str(ROLE_DIR / ROLE_FILE_META_VOLUMES)) or {}
|
||||
mounts = (declared.get(volume) or {}).get("mounts") or []
|
||||
return next(
|
||||
(
|
||||
m["target"]
|
||||
for m in mounts
|
||||
if m.get("service") == service and m.get("target")
|
||||
),
|
||||
"",
|
||||
)
|
||||
|
||||
|
||||
def _tuning() -> dict[str, str]:
|
||||
"""Return the role's plain nginx settings, keyed as the template names them.
|
||||
|
||||
A value the role builds from its inventory is skipped: it has no meaning
|
||||
without one, and the dev stack supplies its own.
|
||||
"""
|
||||
loaded = load_yaml_any(str(ROLE_VARS), default_if_missing={}) or {}
|
||||
return {
|
||||
key: value
|
||||
for key, value in loaded.items()
|
||||
if key.startswith("CACHE_PACKAGE_") and TEMPLATED not in str(value)
|
||||
}
|
||||
|
||||
|
||||
def _render(template: str, **context) -> str:
|
||||
env = Environment(
|
||||
loader=FileSystemLoader([str(path) for path in TEMPLATE_DIRS]),
|
||||
trim_blocks=True,
|
||||
keep_trailing_newline=True,
|
||||
autoescape=False, # noqa: S701 - nginx and compose config, not markup
|
||||
)
|
||||
return env.get_template(template).render(**context)
|
||||
|
||||
|
||||
def upstreams_conf() -> str:
|
||||
"""Return the nginx frontend configuration the declarations produce."""
|
||||
return _render(
|
||||
UPSTREAMS_TEMPLATE,
|
||||
CACHE_PACKAGE_UPSTREAMS=list(host_records().values()),
|
||||
CACHE_PACKAGE_REPOS=repo_records(),
|
||||
CACHE_PACKAGE_BACKEND=BACKEND,
|
||||
CACHE_PACKAGE_CERTS_DIR=_mount_target("frontend_certs", "frontend"),
|
||||
**_tuning(),
|
||||
)
|
||||
|
||||
|
||||
def cache_consumer_yml() -> str:
|
||||
"""Return the compose override that points every cached host at the frontend."""
|
||||
return _render(CONSUMER_TEMPLATE, CACHE_PACKAGE_HOSTNAMES=list(host_records()))
|
||||
|
||||
|
||||
def pip_conf() -> str:
|
||||
"""Return the pip configuration pointing at the cached PyPI hosts."""
|
||||
repo = repo_named(PYPI_REPO)
|
||||
hosts = hosts_serving(PYPI_REPO)
|
||||
origin = origin_of(repo) if repo else ""
|
||||
index = next((entry for entry in hosts if entry.host == origin), None)
|
||||
if index is None:
|
||||
raise SystemExit(
|
||||
f"no cache: host declares itself the origin of {PYPI_REPO}, so pip "
|
||||
"has no index URL to point at"
|
||||
)
|
||||
return _render(
|
||||
"pip.conf.jinja",
|
||||
CACHE_PACKAGE_PYPI_INDEX=f"{index.scheme()}://{index.host}/simple/",
|
||||
CACHE_PACKAGE_PYPI_HOSTS=[entry.host for entry in hosts],
|
||||
)
|
||||
|
||||
|
||||
def npmrc() -> str:
|
||||
"""Return the npm configuration pointing at the cached registry."""
|
||||
hosts = hosts_serving(NPM_REPO)
|
||||
if not hosts:
|
||||
raise SystemExit(f"no cache: host serves {NPM_REPO}, so npm has no registry")
|
||||
registry = hosts[0]
|
||||
return _render(
|
||||
"npmrc.jinja",
|
||||
CACHE_PACKAGE_NPM_REGISTRY=f"{registry.scheme()}://{registry.host}/",
|
||||
)
|
||||
|
||||
|
||||
def apt_list(distro: str) -> str:
|
||||
"""Return one distribution's apt source, empty where it does not use apt.
|
||||
|
||||
Args:
|
||||
distro: the value INFINITO_DISTRO carries.
|
||||
"""
|
||||
name = f"apt-{distro}"
|
||||
repo = repo_named(name)
|
||||
hosts = hosts_serving(name)
|
||||
source = ""
|
||||
if repo and hosts:
|
||||
entry = hosts[0]
|
||||
path = entry.path_of(name).rstrip("/")
|
||||
source = f"{entry.scheme()}://{entry.host}{path}"
|
||||
return _render(
|
||||
"apt.list.jinja",
|
||||
CACHE_PACKAGE_APT_SOURCE=source,
|
||||
CACHE_PACKAGE_APT_SUITE=repo.distribution if repo else "",
|
||||
)
|
||||
|
||||
|
||||
def _write(target: Path, content: str) -> Path:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(content, encoding="utf-8")
|
||||
return target
|
||||
|
||||
|
||||
def render_artifacts(repo_root: Path, conf_path: Path) -> list[Path]:
|
||||
"""Write every generated cache file and return what was written.
|
||||
|
||||
A worktree configured to follow the primary checkout reads that
|
||||
checkout's map without writing it: rendering there would push this
|
||||
branch's declarations into another tree, which is the isolation the
|
||||
setting exists to provide.
|
||||
|
||||
Args:
|
||||
repo_root: the checkout being generated for.
|
||||
conf_path: where INFINITO_CACHE_PACKAGE_FRONTEND_CONF points.
|
||||
"""
|
||||
consumer_dir = repo_root / "compose" / "package-cache"
|
||||
written = [
|
||||
_write(repo_root / CONSUMER, cache_consumer_yml()),
|
||||
_write(consumer_dir / "pip.conf", pip_conf()),
|
||||
_write(consumer_dir / "npmrc", npmrc()),
|
||||
]
|
||||
written += [
|
||||
_write(consumer_dir / "apt" / f"{distro}.list", apt_list(distro))
|
||||
for distro in distro_names()
|
||||
]
|
||||
if conf_path.is_relative_to(repo_root):
|
||||
written.append(_write(conf_path, upstreams_conf()))
|
||||
return written
|
||||
Vendored
+8
@@ -35,7 +35,11 @@ from .infinito import (
|
||||
from .infinito.ca import bundle_candidates as ca_bundle_candidates
|
||||
from .infinito.ca import cert_host as ca_cert_host
|
||||
from .infinito.cache import conf as cache_conf
|
||||
from .infinito.cache import hosts as cache_hosts
|
||||
from .infinito.cache import http_hosts as cache_http_hosts
|
||||
from .infinito.cache import stack as cache_stack
|
||||
from .infinito.cache import tls as cache_tls
|
||||
from .infinito.cache import upstreams as cache_upstreams
|
||||
from .infinito.dir import backups as dir_backups
|
||||
from .infinito.dir import secrets as dir_secrets
|
||||
from .infinito.dir import var_lib as dir_var_lib
|
||||
@@ -80,6 +84,10 @@ ORDERED_HANDLERS = [
|
||||
running_on_act,
|
||||
running_on_github,
|
||||
cache_conf,
|
||||
cache_hosts,
|
||||
cache_http_hosts,
|
||||
cache_tls,
|
||||
cache_upstreams,
|
||||
cache_stack,
|
||||
tor_socks_port,
|
||||
is_wsl2,
|
||||
|
||||
+6
-1
@@ -1,6 +1,11 @@
|
||||
"""INFINITO_CACHE_PACKAGE_FRONTEND_CONF: nginx upstream map the shared
|
||||
package-cache frontend mounts, taken from the primary checkout unless
|
||||
INFINITO_CACHE_CONF_SOURCE says worktree."""
|
||||
INFINITO_CACHE_CONF_SOURCE says worktree.
|
||||
|
||||
The map itself is generated, never tracked, so no second copy of a hostname
|
||||
can drift; ``utils.cache.render.render_artifacts`` writes it once the
|
||||
value-set is built.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
"""INFINITO_CACHE_HOSTS: the hostnames the package cache fronts, collected
|
||||
from the ``cache:`` declarations every role makes in its meta/networks.yml."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from utils.cache.hosts import declarations
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from utils.env.builder import BuildContext, EnvBuilder
|
||||
|
||||
KEY = "INFINITO_CACHE_HOSTS"
|
||||
COMMENT = (
|
||||
"Hostnames the package cache fronts; derived from the roles' cache: declarations."
|
||||
)
|
||||
|
||||
|
||||
def apply(eb: EnvBuilder, ctx: BuildContext) -> None:
|
||||
eb.set(KEY, ",".join(entry.host for entry in declarations().hosts), comment=COMMENT)
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
"""INFINITO_CACHE_HTTP_HOSTS: the hostnames the frontend serves without TLS.
|
||||
|
||||
An image build reaches the cache by hostname, but it cannot verify the
|
||||
frontend's self-signed certificate unless its Dockerfile installs the CA, so
|
||||
only these are safe to hijack for every build. A build that does install the
|
||||
CA additionally gets the TLS hosts; `sys-svc-compose`'s override generator
|
||||
decides that per image.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from utils.cache.hosts import HTTP_PORT, declarations
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from utils.env.builder import BuildContext, EnvBuilder
|
||||
|
||||
KEY = "INFINITO_CACHE_HTTP_HOSTS"
|
||||
COMMENT = "Hostnames the cache frontend serves over plain HTTP; safe to hijack in any image build."
|
||||
|
||||
|
||||
def apply(eb: EnvBuilder, ctx: BuildContext) -> None:
|
||||
eb.set(
|
||||
KEY,
|
||||
",".join(
|
||||
entry.host for entry in declarations().hosts if HTTP_PORT in entry.listen
|
||||
),
|
||||
comment=COMMENT,
|
||||
)
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
"""INFINITO_CACHE_TLS_HOSTS: the hostnames the package-cache frontend needs a
|
||||
leaf certificate for, being those its cache: declarations serve over HTTPS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from utils.cache.hosts import HTTPS_PORT, declarations
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from utils.env.builder import BuildContext, EnvBuilder
|
||||
|
||||
KEY = "INFINITO_CACHE_TLS_HOSTS"
|
||||
COMMENT = "Hostnames the cache frontend needs a leaf cert for; derived from the roles' cache: declarations."
|
||||
|
||||
|
||||
def apply(eb: EnvBuilder, ctx: BuildContext) -> None:
|
||||
eb.set(
|
||||
KEY,
|
||||
" ".join(
|
||||
entry.host for entry in declarations().hosts if HTTPS_PORT in entry.listen
|
||||
),
|
||||
comment=COMMENT,
|
||||
)
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
"""INFINITO_CACHE_UPSTREAMS: every repository the package cache proxies, as
|
||||
``name|flavor|url|distribution|repodata_depth|content_max_age`` records
|
||||
separated by commas, from the cache: entries the roles declare in their
|
||||
meta/networks.yml.
|
||||
|
||||
`utils.cache.records` owns the format and the mirror resolution, so the role's
|
||||
bootstrap task and this key cannot drift apart.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from utils.cache.records import records
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from utils.env.builder import BuildContext, EnvBuilder
|
||||
|
||||
KEY = "INFINITO_CACHE_UPSTREAMS"
|
||||
MIRRORS_KEY = "INFINITO_APT_UBUNTU_MIRRORS"
|
||||
COMMENT = "Proxied repositories as name|flavor|url|distribution|repodata_depth|content_max_age; from the cache: declarations."
|
||||
|
||||
|
||||
def apply(eb: EnvBuilder, ctx: BuildContext) -> None:
|
||||
rendered = records(eb.get(MIRRORS_KEY) or ctx.static.get(MIRRORS_KEY, ""))
|
||||
if rendered:
|
||||
eb.set(KEY, rendered, comment=COMMENT)
|
||||
Vendored
+1
@@ -123,6 +123,7 @@ STATIC_KEYS = (
|
||||
"INFINITO_CACHE_REGISTRY_HOST_PATH",
|
||||
"INFINITO_CACHE_REGISTRY_CA_HOST_PATH",
|
||||
"INFINITO_CACHE_REGISTRY_IP",
|
||||
"INFINITO_CACHE_PACKAGE_CONTAINER",
|
||||
"INFINITO_CACHE_PACKAGE_HOST_PATH",
|
||||
"INFINITO_CACHE_PACKAGE_PORT",
|
||||
"INFINITO_CACHE_PACKAGE_FRONTEND_CA_DIR",
|
||||
|
||||
Reference in new issue
Block a user