From 5d05e78a68cceb96013225742f45deb5b36c8f1c Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 22 Oct 2026 07:48:49 +0200 Subject: [PATCH 1/7] fix(update): refresh dnf metadata for real and retry once after a push race Fedora deploys failed in `update (dnf)` with "Failed to download packages": the node's metadata still listed builds the master had already replaced, and every mirror answered 404 (#1138: less-704-4.fc44 four hours after the push; run 39684179395: OpenBao#0 swarm-mgr-01 and Nextcloud#9, less-704-4 and ca-certificates-2026.2.90). Two mechanisms lead there: - Under dnf5, ansible.builtin.dnf's `update_cache: true` expires no repository: it queries the repos before loading them (0 available repos before create_repos_from_system_configuration, 16 after), so metadata already cached on the node stays in use until metadata_expire. An explicit `dnf makecache --refresh` now runs before the upgrade. - A metalink can still hand out repodata from a mirror on the previous push. The upgrade now runs in a block whose rescue re-raises every other failure unchanged and, on exactly "Failed to download packages", runs `dnf clean metadata` and upgrades once more. Fixes #1138 Co-Authored-By: Claude Opus 5.5 (1M context) --- roles/update/tasks/fedora.yml | 36 ++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/roles/update/tasks/fedora.yml b/roles/update/tasks/fedora.yml index dcbae4c85c..7211b6d2b0 100644 --- a/roles/update/tasks/fedora.yml +++ b/roles/update/tasks/fedora.yml @@ -7,8 +7,34 @@ line: "retries=10" insertafter: "^\\[main\\]" -- name: "๐Ÿ”„ update (dnf) | refresh dnf cache and upgrade all installed packages" - ansible.builtin.dnf: # nocheck: dynamic-package - full-system upgrade, not a package install - name: "*" - state: latest - update_cache: true +# Exception: under dnf5, ansible.builtin.dnf's update_cache expires no repo (it queries the repos before loading them), so cached metadata survives until metadata_expire. +- name: "๐Ÿ”„ update (dnf) | re-fetch the metadata of every enabled repository" + become: true + ansible.builtin.command: + argv: [dnf, makecache, --refresh] + changed_when: false + +- name: "๐Ÿ”„ update (dnf) | upgrade all installed packages" + block: + - name: "๐Ÿ”„ update (dnf) | refresh dnf cache and upgrade all installed packages" + ansible.builtin.dnf: # nocheck: dynamic-package - full-system upgrade, not a package install + name: "*" + state: latest + update_cache: true + rescue: + - name: "๐Ÿ›‘ update (dnf) | re-raise anything but packages no mirror serves" # nocheck: rescue-recovery re-fetches metadata a mirror on the previous push served and retries once; the retry fails loud + ansible.builtin.fail: + msg: "{{ ansible_failed_result.msg }}" + when: ansible_failed_result.msg != 'Failed to download packages' + + - name: "๐Ÿงน update (dnf) | drop the metadata a mirror still on the previous push served" + become: true + ansible.builtin.command: + argv: [dnf, clean, metadata] + changed_when: true + + - name: "๐Ÿ”„ update (dnf) | upgrade once more against fresh metadata" + ansible.builtin.dnf: # nocheck: dynamic-package - full-system upgrade, not a package install + name: "*" + state: latest + update_cache: true From c65c5ce48cdaf8e604041b8d4aaef06e5a349476 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 22 Oct 2026 10:30:02 +0200 Subject: [PATCH 2/7] fix(update): pin only image tags the registry carries 7e82ba6d71 moved web-app-bigbluebutton's bbb-pads to v1.5.11, the tag https://raw.githubusercontent.com/bigbluebutton/docker/develop/repos/tags names for the bbb-pads source. alangecker/bbb-docker-pads has no such image (Docker Hub: newest v1.5.8), so every BigBlueButton deploy pulls a missing tag and test-external fails "These pinned Docker images do not exist in their registry (HTTP 404): web-app-bigbluebutton/bbb-pads". bbb-pads goes back to v1.5.8. The version-source updater now knows the image a `version` pin tags (pulled directly or as the base of a custom build) and withholds a bump when the registry answers 404 for the new tag; an indeterminate answer does not block it, the same line test-external draws. Run against main, the updater now reports no update for bbb-pads instead of v1.5.11. Co-Authored-By: Claude Opus 5.5 (1M context) --- roles/web-app-bigbluebutton/meta/services.yml | 2 +- tests/unit/python/utils/update/test_source.py | 30 ++++++++++++++++++- utils/update/source.py | 14 +++++++-- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/roles/web-app-bigbluebutton/meta/services.yml b/roles/web-app-bigbluebutton/meta/services.yml index 1774c3f158..e93411f153 100644 --- a/roles/web-app-bigbluebutton/meta/services.yml +++ b/roles/web-app-bigbluebutton/meta/services.yml @@ -152,7 +152,7 @@ etherpad: # nocheck: dynamic-flag: fixed upstream-compose sub-service, not group-gated bbb-pads: image: alangecker/bbb-docker-pads - version: "v1.5.11" + version: "v1.5.8" update: type: http_regex url: https://raw.githubusercontent.com/bigbluebutton/docker/develop/repos/tags diff --git a/tests/unit/python/utils/update/test_source.py b/tests/unit/python/utils/update/test_source.py index e10f86c77f..df905623d2 100644 --- a/tests/unit/python/utils/update/test_source.py +++ b/tests/unit/python/utils/update/test_source.py @@ -179,7 +179,10 @@ class TestOutdated(unittest.TestCase): class TestTemplate(unittest.TestCase): def _outdated(self, services: str, upstream: str) -> list[str]: root = _repo(services) - with mock.patch.object(module, "documents", return_value=upstream): + with ( + mock.patch.object(module, "documents", return_value=upstream), + mock.patch.object(module, "manifest_exists", return_value=True), + ): updates = module.outdated(module.collect_entries(root), root) return [update.latest for update in updates] @@ -206,6 +209,31 @@ class TestTemplate(unittest.TestCase): ) +class TestPulledImage(unittest.TestCase): + def _outdated(self, exists: bool | None) -> tuple[list[str], mock.MagicMock]: + root = _repo(COMPOUND) + with ( + mock.patch.object( + module, "documents", return_value="core v3.0.39\npart v1.11.3\n" + ), + mock.patch.object(module, "manifest_exists", return_value=exists) as probe, + ): + updates = module.outdated(module.collect_entries(root), root) + return [update.latest for update in updates], probe + + def test_a_tag_the_image_registry_lacks_is_not_pinned(self) -> None: + latest, probe = self._outdated(False) + self.assertEqual(latest, []) + probe.assert_called_once_with("example/part", "v1.11.3-v3.0.39") + + def test_an_indeterminate_registry_answer_keeps_the_bump(self) -> None: + self.assertEqual(self._outdated(None)[0], ["v1.11.3-v3.0.39"]) + + def test_only_the_version_of_a_pulled_image_carries_the_image(self) -> None: + entries = {e.key: e.image for e in module.collect_entries(_repo())} + self.assertEqual(entries, {"app_version": "", "version": "example"}) + + class TestAddons(unittest.TestCase): def test_only_a_declared_or_github_released_addon_pin_is_collected(self) -> None: entries = module.collect_entries(_addon_repo()) diff --git a/utils/update/source.py b/utils/update/source.py index 0cf9166337..3d6405feaa 100644 --- a/utils/update/source.py +++ b/utils/update/source.py @@ -39,6 +39,7 @@ from urllib.parse import urlencode from utils.annotations.suppress import is_suppressed_at from utils.cache.files import read_text from utils.cache.yaml import load_yaml +from utils.docker.registry import manifest_exists from utils.roles.mapping import ROLE_FILE_META_SERVICES from utils.update.addons import GITHUB_RELEASES_CATALOG, iter_addon_files from utils.update.base import ( @@ -83,6 +84,10 @@ class VersionSourceEntry: config_path: file that carries the pin. line: 1-indexed line of the pin. addon: whether the pin sits in ``meta/addons/.yml``. + image: image the pin tags when it is the entity's ``version``, pulled + directly or as the base of a ``custom`` build; a bump to a tag + its registry answers 404 for is withheld, an indeterminate answer + does not block it. """ role: str @@ -93,6 +98,7 @@ class VersionSourceEntry: config_path: Path line: int addon: bool = False + image: str = "" @dataclass(frozen=True) @@ -442,6 +448,7 @@ def collect_entries(repo_root: Path) -> list[VersionSourceEntry]: source=source, config_path=config_path, line=line, + image=config.get("image", "") if key == DEFAULT_KEY else "", ) ) return entries + _addon_entries(repo_root) @@ -458,8 +465,11 @@ def outdated( candidates(entry, repo_root), assembled=bool(entry.source.get("template")), ) - if newest: - updates.append(VersionSourceUpdate(entry=entry, latest=newest)) + if not newest: + continue + if entry.image and manifest_exists(entry.image, newest) is False: + continue + updates.append(VersionSourceUpdate(entry=entry, latest=newest)) return updates From 88d76aeea0a81df25f756354f655882d04e14707 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 22 Oct 2026 10:30:32 +0200 Subject: [PATCH 3/7] fix(update): resolve CentOS Stream packages from mirror.stream.centos.org MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since 2026-10-21 18:45Z part of the metalinks mirrors.centos.org hands to CI runners still expects the previous BaseOS repomd.xml (sha512 bed4f4โ€ฆ) while every mirror and the master already serve the current one (2f4550โ€ฆ). dnf then rejects every mirror ("checksum doesn't match", "All mirrors were tried", Nextcloud#2, Nextcloud#10, Mattermost#1 in run 39684179395) or resolves BaseOS from a mirror still on the previous push while AppStream is current ("nothing provides rpm = 4.19.1.1-27.el10 needed by rpm-build-4.19.1.1-27.el10", Baserow#0, Akaunting#0). roles/update/files/centos_mirror.sh replaces every metalink of the CentOS Stream repo files with the matching baseurl on mirror.stream.centos.org and prints what it pinned. The Dockerfile runs it before its first dnf call, so the dev and swarm-node images never consult a metalink; roles/update reaches fedora.yml through a new centos.yml that runs it first, so production hosts resolve against the same source. The script is idempotent and prints nothing once every section is pinned. Co-Authored-By: Claude Opus 5.5 (1M context) --- Dockerfile | 2 ++ roles/update/README.md | 11 ++++--- roles/update/files/shell/centos_mirror.sh | 38 +++++++++++++++++++++++ roles/update/tasks/centos.yml | 8 +++++ roles/update/tasks/main.yml | 2 +- 5 files changed, 56 insertions(+), 5 deletions(-) create mode 100755 roles/update/files/shell/centos_mirror.sh create mode 100644 roles/update/tasks/centos.yml diff --git a/Dockerfile b/Dockerfile index 25e3f98710..ab7e98093f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,7 @@ RUN set -euo pipefail; \ COPY default.env ${INFINITO_SRC_DIR}/default.env COPY scripts/install ${INFINITO_SRC_DIR}/scripts/install COPY roles/dev-python/files/shell ${INFINITO_SRC_DIR}/roles/dev-python/files/shell +COPY roles/update/files/shell/centos_mirror.sh ${INFINITO_SRC_DIR}/roles/update/files/shell/centos_mirror.sh COPY requirements ${INFINITO_SRC_DIR}/requirements COPY utils/__init__.py ${INFINITO_SRC_DIR}/utils/__init__.py COPY utils/install/__init__.py ${INFINITO_SRC_DIR}/utils/install/__init__.py @@ -36,6 +37,7 @@ COPY utils/install/collections.py ${INFINITO_SRC_DIR}/utils/install/collections. # hadolint ignore=DL3008,DL3033,DL3041,SC1090,SC3040 RUN set -euo pipefail; \ + /bin/bash "${INFINITO_SRC_DIR}/roles/update/files/shell/centos_mirror.sh"; \ source <(grep -hE '^INFINITO_PYTHON_INSTALL_SCRIPT=' "${INFINITO_SRC_DIR}/default.env"); \ /bin/bash "${INFINITO_SRC_DIR}/${INFINITO_PYTHON_INSTALL_SCRIPT:?}" ensure; \ VENV="${INFINITO_VENV_DIR}" bash "${INFINITO_SRC_DIR}/scripts/install/venv.sh"; \ diff --git a/roles/update/README.md b/roles/update/README.md index fbdb3e21fd..c05361eee9 100644 --- a/roles/update/README.md +++ b/roles/update/README.md @@ -13,18 +13,21 @@ installed package on the host. This role refreshes the package index and upgrades every installed package on the host. `tasks/main.yml` reads `ansible_facts['distribution']` and dispatches -to one of three task files grouped by package-manager family. Debian and -Ubuntu both use `apt`, Fedora and CentOS both use `dnf`, so each pair shares a -single task file. A separate `unsupported.yml` fails loudly with a clear +to one task file per package-manager family. Debian and Ubuntu both use +`apt`, so they share a single task file; Fedora and CentOS both use `dnf`, and +CentOS reaches `fedora.yml` through `centos.yml`. A separate `unsupported.yml` fails loudly with a clear message when a host runs a distribution that this role does not yet cover. | `ansible_facts['distribution']` | Task file | Module | | ------------------------------- | ----------------- | -------------------------- | | `Archlinux` | `arch.yml` | `community.general.pacman` | | `Debian` / `Ubuntu` | `debian.yml` | `ansible.builtin.apt` | -| `Fedora` / `CentOS` | `fedora.yml` | `ansible.builtin.dnf` | +| `Fedora` | `fedora.yml` | `ansible.builtin.dnf` | +| `CentOS` | `centos.yml` | `ansible.builtin.dnf` | | anything else | `unsupported.yml` | `ansible.builtin.fail` | +`centos.yml` first runs [centos_mirror.sh](files/shell/centos_mirror.sh), which replaces every `metalink=` of the CentOS Stream repo files with a `baseurl=` on `mirror.stream.centos.org`, and then includes `fedora.yml`. The [Dockerfile](../../Dockerfile) runs the same script before its first `dnf` call, so the CentOS dev and swarm-node images resolve against the same source. + ## Cosmos The diagram places Package update in the Infinito.Nexus cosmos: the components it deploys (capabilities), the central services it consumes (dependencies), and its outward reach (federation and bridged external networks). diff --git a/roles/update/files/shell/centos_mirror.sh b/roles/update/files/shell/centos_mirror.sh new file mode 100755 index 0000000000..d44cbc66aa --- /dev/null +++ b/roles/update/files/shell/centos_mirror.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Param: none. Rewrites the CentOS Stream repo files in /etc/yum.repos.d in place and prints the metalink repo of every section it pinned; prints nothing when there is nothing left to pin. +set -euo pipefail + +master="https://mirror.stream.centos.org" +repo_dir="/etc/yum.repos.d" + +[[ -f "${repo_dir}/centos.repo" ]] || exit 0 + +pin() { + local metalink_repo="$1" path="$2" file + for file in "${repo_dir}/centos.repo" "${repo_dir}/centos-addons.repo"; do + [[ -f "${file}" ]] || continue + grep -q -E "^metalink=https://mirrors\.centos\.org/metalink\?repo=${metalink_repo}&" "${file}" || continue + sed -i -E "s|^metalink=https://mirrors\.centos\.org/metalink\?repo=${metalink_repo}&.*\$|baseurl=${master}/${path}/|" "${file}" + echo "${metalink_repo/\[\$\]stream/\$stream}" + done +} + +declare -A directories=( + [baseos]=BaseOS + [appstream]=AppStream + [crb]=CRB + [highavailability]=HighAvailability + [nfv]=NFV + [rt]=RT + [resilientstorage]=ResilientStorage +) + +for name in "${!directories[@]}"; do + directory="${directories[${name}]}" + pin "centos-${name}-[\$]stream" "\$stream/${directory}/\$basearch/os" + pin "centos-${name}-debug-[\$]stream" "\$stream/${directory}/\$basearch/debug/tree" + pin "centos-${name}-source-[\$]stream" "\$stream/${directory}/source/tree" +done + +pin "centos-extras-sig-extras-common-[\$]stream" "SIGs/\$stream/extras/\$basearch/extras-common" +pin "centos-extras-sig-extras-common-source-[\$]stream" "SIGs/\$stream/extras/source/extras-common" diff --git a/roles/update/tasks/centos.yml b/roles/update/tasks/centos.yml new file mode 100644 index 0000000000..2908c2ddc5 --- /dev/null +++ b/roles/update/tasks/centos.yml @@ -0,0 +1,8 @@ +--- +- name: "๐Ÿ“Œ update (dnf) | pin the CentOS Stream repos to mirror.stream.centos.org" + become: true + ansible.builtin.script: shell/centos_mirror.sh + register: update_centos_pinned + changed_when: update_centos_pinned.stdout | length > 0 + +- include_tasks: fedora.yml diff --git a/roles/update/tasks/main.yml b/roles/update/tasks/main.yml index 7592fdf08e..6a4341e2b8 100644 --- a/roles/update/tasks/main.yml +++ b/roles/update/tasks/main.yml @@ -10,7 +10,7 @@ 'Debian': 'debian.yml', 'Ubuntu': 'debian.yml', 'Fedora': 'fedora.yml', - 'CentOS': 'fedora.yml', + 'CentOS': 'centos.yml', }[ansible_facts['distribution']] | default('unsupported.yml') }} From e7e3856b09b903e81188c4ee8a4ea033fd8669b8 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 22 Oct 2026 10:30:57 +0200 Subject: [PATCH 4/7] fix(update): match the dnf download failure by prefix Under the dnf4 backend (CentOS) ansible.builtin.dnf reports `Failed to download packages: ` (module_utils/_embed/dnf.py), under dnf5 (Fedora) the bare `Failed to download packages`. The exact comparison re-raised the CentOS form instead of retrying. The rescue now re-raises everything that does not start with that message. Co-Authored-By: Claude Opus 5.5 (1M context) --- roles/update/tasks/fedora.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/update/tasks/fedora.yml b/roles/update/tasks/fedora.yml index 7211b6d2b0..a7ad389c9f 100644 --- a/roles/update/tasks/fedora.yml +++ b/roles/update/tasks/fedora.yml @@ -25,7 +25,7 @@ - name: "๐Ÿ›‘ update (dnf) | re-raise anything but packages no mirror serves" # nocheck: rescue-recovery re-fetches metadata a mirror on the previous push served and retries once; the retry fails loud ansible.builtin.fail: msg: "{{ ansible_failed_result.msg }}" - when: ansible_failed_result.msg != 'Failed to download packages' + when: ansible_failed_result.msg is not match('Failed to download packages') - name: "๐Ÿงน update (dnf) | drop the metadata a mirror still on the previous push served" become: true From 0f9bdca445c5c35560cadd05d0be07687c6dcf8d Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 22 Oct 2026 10:30:57 +0200 Subject: [PATCH 5/7] fix(install): refresh dnf metadata on every retried install scripts/install/filesystem.sh retried `dnf install -y zfs` three times against the same metadata during the Fedora image build of main run 39684179395 (XMPP#0): every attempt listed pcp-*-7.2.2-1.fc44, every mirror answered 404, "Failed to download packages" after 3 attempts. Each retried dnf install now runs with --refresh, so every attempt resolves against freshly fetched metadata instead of repeating the one that lists packages no mirror serves anymore. Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/install/filesystem.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/install/filesystem.sh b/scripts/install/filesystem.sh index 61f8428cdc..31b6b52136 100755 --- a/scripts/install/filesystem.sh +++ b/scripts/install/filesystem.sh @@ -112,13 +112,13 @@ install_pacman() { install_dnf_native() { case "${ID:-}" in fedora) - retry dnf install -y e2fsprogs btrfs-progs + retry dnf --refresh install -y e2fsprogs btrfs-progs return ;; esac log "adding EPEL, the only place the EL family carries btrfs-progs" - retry dnf install -y epel-release - retry dnf install -y e2fsprogs btrfs-progs + retry dnf --refresh install -y epel-release + retry dnf --refresh install -y e2fsprogs btrfs-progs } # Param: $1 path segment of the zfsonlinux repository, `fedora` or `epel` @@ -155,7 +155,7 @@ install_dnf() { log "zfs userland offered by the repository: ${zfs_version}" if ! command -v rpmbuild >/dev/null 2>&1; then - retry dnf install -y rpm-build + retry dnf --refresh install -y rpm-build fi local topdir=/tmp/zfs-kmod-container @@ -180,7 +180,7 @@ SPEC rpmbuild --define "_topdir ${topdir}" -bb "${topdir}/SPECS/zfs-kmod-container.spec" dnf install -y "${topdir}"/RPMS/noarch/zfs-kmod-container-*.rpm - retry dnf install -y zfs + retry dnf --refresh install -y zfs if rpm -qa 'kernel-devel*' 'kernel-debug*' 'zfs-dkms' | grep -q .; then log "ERROR: the transaction pulled kernel or DKMS packages, so the dummy did not take" From ba9e51ae850f22d4a32f2f6847263afe1fe6aba6 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 22 Oct 2026 10:31:12 +0200 Subject: [PATCH 6/7] fix(tests): keep the workspace minimal deploy off the public asset host compose/01_minimal.sh disabled every variable service of the dashboard, asset included, and ran both matrix rounds; round 1 of web-app-dashboard pins the asset provider off as well. Without web-svc-asset the dashboard renders its logo and favicon from the public https://file.infinito.nexus/assets, so the workspace test depended on that host: while it does not answer, Playwright's no-console-noise spec fails on `net::ERR_CONNECTION_REFUSED` for the logo (main run 39684179395, ubuntu and centos). The test now keeps the asset provider out of the disable set and pins round 0, so the dashboard serves its branding from the deployment's own file service. Round 1 stays covered by the regular dashboard matrix. Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/tests/workspace/compose/01_minimal.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/tests/workspace/compose/01_minimal.sh b/scripts/tests/workspace/compose/01_minimal.sh index 783c7ffa06..1c718d388a 100755 --- a/scripts/tests/workspace/compose/01_minimal.sh +++ b/scripts/tests/workspace/compose/01_minimal.sh @@ -11,8 +11,9 @@ source "${SCRIPT_DIR}/../utils/cache.sh" echo "Snapshotting cache counters before the deploy." CACHE_BEFORE="$(cache_snapshot)" -echo "Deploying dashboard with all variable (group-conditional) services disabled to speed up the deploy and verify disable= suppresses them from the inventory." -make compose-deploy mode=reinstall apps="${DASHBOARD_APP}" disable="$(variable_services dashboard javascript)" +echo "Deploying dashboard with all variable (group-conditional) services disabled, except the asset provider, to speed up the deploy and verify disable= suppresses them from the inventory." +# Exception: without the asset provider the dashboard loads its logo from the public file.infinito.nexus, which ties this test to that host's uptime; variant 1 pins the provider off, so only round 0 runs here. +make compose-deploy mode=reinstall apps="${DASHBOARD_APP}" variant=0 disable="$(variable_services dashboard javascript asset)" inspect echo "Actively probing both caches to confirm pull-through works end-to-end." From 0986957e523c92cc5ede90d8829355e9f8501401 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 22 Oct 2026 10:33:49 +0200 Subject: [PATCH 7/7] fix(proxy): strip upstream headers on every proxying location fragment proxy_hide_header is replaced, not merged, by a location that sets its own, so upstream_strips.conf.j2 only takes effect where a location includes it. The health locations of web-app-prometheus (/healthz/ready, /_live_probe), which every standard vhost includes, and the extra locations of web-app-bluesky (/config, /ipcc) and web-app-opentalk (/v1/, /signaling, /livekit) proxied without it, so X-Powered-By and the upstream's Strict-Transport-Security leaked on those paths. All seven locations now include the strips. A new lint checks every role template without a server block, i.e. every fragment the front proxy renders into a vhost, and fails on a proxying location that lacks them. Templates with their own server block are app-internal nginx configs behind such a location and stay out of scope. Closes #1124 Co-Authored-By: Claude Opus 5.5 (1M context) --- .../templates/extra_locations.conf.j2 | 2 + .../templates/nginx/extra.conf.j2 | 3 + .../templates/nginx/healthz.conf.j2 | 2 + .../jinja/test_proxy_location_strips.py | 90 +++++++++++++++++++ 4 files changed, 97 insertions(+) create mode 100644 tests/lint/ansible/jinja/test_proxy_location_strips.py diff --git a/roles/web-app-bluesky/templates/extra_locations.conf.j2 b/roles/web-app-bluesky/templates/extra_locations.conf.j2 index adf9f6d102..169661909b 100644 --- a/roles/web-app-bluesky/templates/extra_locations.conf.j2 +++ b/roles/web-app-bluesky/templates/extra_locations.conf.j2 @@ -10,6 +10,7 @@ location = /config { set $up_host "{{ BLUESKY_CONFIG_UPSTREAM_URL | regex_replace('^https?://', '') | regex_replace('/.*$', '') }}"; set $bluesky_config_upstream "{{ BLUESKY_CONFIG_UPSTREAM_URL }}"; proxy_pass $bluesky_config_upstream; + {% include 'roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2' %} proxy_set_header Host $up_host; proxy_http_version 1.1; proxy_set_header Connection ""; @@ -24,6 +25,7 @@ location = /ipcc { set $up_host "bsky.app"; set $bluesky_ipcc_upstream "https://bsky.app/ipcc"; proxy_pass $bluesky_ipcc_upstream; + {% include 'roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2' %} proxy_set_header Host $up_host; proxy_http_version 1.1; proxy_set_header Connection ""; diff --git a/roles/web-app-opentalk/templates/nginx/extra.conf.j2 b/roles/web-app-opentalk/templates/nginx/extra.conf.j2 index 35f513c64a..378849a174 100644 --- a/roles/web-app-opentalk/templates/nginx/extra.conf.j2 +++ b/roles/web-app-opentalk/templates/nginx/extra.conf.j2 @@ -5,6 +5,7 @@ location /v1/ { proxy_pass http://127.0.0.1:{{ OPENTALK_CTRL_HTTP_PORT }}; + {% include 'roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2' %} proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -17,6 +18,7 @@ location /v1/ { location /signaling { proxy_pass http://127.0.0.1:{{ OPENTALK_CTRL_HTTP_PORT }}; + {% include 'roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2' %} proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; @@ -30,6 +32,7 @@ location /signaling { location /livekit { proxy_pass http://127.0.0.1:{{ OPENTALK_LIVEKIT_HTTP_PORT }}; + {% include 'roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2' %} proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; diff --git a/roles/web-app-prometheus/templates/nginx/healthz.conf.j2 b/roles/web-app-prometheus/templates/nginx/healthz.conf.j2 index 914f08201d..a034241e6a 100644 --- a/roles/web-app-prometheus/templates/nginx/healthz.conf.j2 +++ b/roles/web-app-prometheus/templates/nginx/healthz.conf.j2 @@ -54,6 +54,7 @@ internal; {{ prometheus_backend }} {% include 'roles/sys-svc-proxy/templates/headers/forwarded.conf.j2' %} + {% include 'roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2' %} proxy_connect_timeout 2s; proxy_read_timeout 3s; proxy_intercept_errors off; @@ -63,6 +64,7 @@ access_log off; {{ prometheus_ready_backend }} {% include 'roles/sys-svc-proxy/templates/headers/forwarded.conf.j2' %} + {% include 'roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2' %} proxy_connect_timeout 3s; proxy_read_timeout 5s; proxy_intercept_errors off; diff --git a/tests/lint/ansible/jinja/test_proxy_location_strips.py b/tests/lint/ansible/jinja/test_proxy_location_strips.py new file mode 100644 index 0000000000..93b750fc31 --- /dev/null +++ b/tests/lint/ansible/jinja/test_proxy_location_strips.py @@ -0,0 +1,90 @@ +"""Lint: every proxying location the front proxy includes strips upstream headers. + +``proxy_hide_header`` is replaced, not merged, by a location that sets its own, +so ``upstream_strips.conf.j2`` only takes effect where a location includes it. +A location fragment without it leaks ``X-Powered-By`` and the upstream's +``Strict-Transport-Security`` on that path alone. + +Scope: role templates without a ``server`` block. Those are the fragments the +front proxy renders into a vhost; a template with its own ``server`` block is +the config of an app-internal nginx that sits behind such a location. +""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path + +from utils.cache.files import iter_project_files, read_text + +from . import PROJECT_ROOT + +_STRIPS = "roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2" +_SERVER_BLOCK = re.compile(r"^\s*server\s*\{", re.MULTILINE) +_LOCATION_OPEN = re.compile(r"^\s*location\b[^{]*\{\s*$") +_PROXYING = re.compile(r"\bproxy_pass\b|lookup\('proxy_pass'|_backend\s*\}\}") + + +def _role_templates() -> list[str]: + return sorted( + path + for path in iter_project_files(extensions=(".j2",), exclude_tests=True) + if "/roles/" in path and "/templates/" in path + ) + + +def _location_blocks(lines: list[str]) -> list[tuple[int, str]]: + blocks: list[tuple[int, str]] = [] + index = 0 + while index < len(lines): + if not _LOCATION_OPEN.match(lines[index]): + index += 1 + continue + start = index + depth = lines[index].count("{") - lines[index].count("}") + body: list[str] = [] + index += 1 + while index < len(lines) and depth > 0: + depth += lines[index].count("{") - lines[index].count("}") + body.append(lines[index]) + index += 1 + blocks.append((start + 1, "\n".join(body))) + return blocks + + +class TestProxyLocationStrips(unittest.TestCase): + def test_at_least_one_fragment_is_checked(self) -> None: + """Otherwise the assertion below passes over an empty list.""" + checked = [ + path + for path in _role_templates() + if not _SERVER_BLOCK.search(read_text(path)) + and any( + _PROXYING.search(body) + for _, body in _location_blocks(read_text(path).splitlines()) + ) + ] + self.assertTrue(checked, "no proxying location fragment found to check") + + def test_proxying_locations_include_the_strips(self) -> None: + findings: list[str] = [] + for path in _role_templates(): + text = read_text(path) + if _SERVER_BLOCK.search(text): + continue + rel = Path(path).relative_to(PROJECT_ROOT).as_posix() + for line, body in _location_blocks(text.splitlines()): + if _PROXYING.search(body) and _STRIPS not in body: + findings.append(f"- {rel}:{line}") + if findings: + self.fail( + "These proxying locations do not include the upstream header strips, " + "so X-Powered-By and the upstream HSTS leak on their path:\n" + + "\n".join(findings) + + f"\n\nFix: add `{{% include '{_STRIPS}' %}}` inside each location." + ) + + +if __name__ == "__main__": + unittest.main()