diff --git a/Dockerfile b/Dockerfile index 25e3f98710..ab7e98093f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,7 @@ RUN set -euo pipefail; \ COPY default.env ${INFINITO_SRC_DIR}/default.env COPY scripts/install ${INFINITO_SRC_DIR}/scripts/install COPY roles/dev-python/files/shell ${INFINITO_SRC_DIR}/roles/dev-python/files/shell +COPY roles/update/files/shell/centos_mirror.sh ${INFINITO_SRC_DIR}/roles/update/files/shell/centos_mirror.sh COPY requirements ${INFINITO_SRC_DIR}/requirements COPY utils/__init__.py ${INFINITO_SRC_DIR}/utils/__init__.py COPY utils/install/__init__.py ${INFINITO_SRC_DIR}/utils/install/__init__.py @@ -36,6 +37,7 @@ COPY utils/install/collections.py ${INFINITO_SRC_DIR}/utils/install/collections. # hadolint ignore=DL3008,DL3033,DL3041,SC1090,SC3040 RUN set -euo pipefail; \ + /bin/bash "${INFINITO_SRC_DIR}/roles/update/files/shell/centos_mirror.sh"; \ source <(grep -hE '^INFINITO_PYTHON_INSTALL_SCRIPT=' "${INFINITO_SRC_DIR}/default.env"); \ /bin/bash "${INFINITO_SRC_DIR}/${INFINITO_PYTHON_INSTALL_SCRIPT:?}" ensure; \ VENV="${INFINITO_VENV_DIR}" bash "${INFINITO_SRC_DIR}/scripts/install/venv.sh"; \ diff --git a/roles/update/README.md b/roles/update/README.md index fbdb3e21fd..c05361eee9 100644 --- a/roles/update/README.md +++ b/roles/update/README.md @@ -13,18 +13,21 @@ installed package on the host. This role refreshes the package index and upgrades every installed package on the host. `tasks/main.yml` reads `ansible_facts['distribution']` and dispatches -to one of three task files grouped by package-manager family. Debian and -Ubuntu both use `apt`, Fedora and CentOS both use `dnf`, so each pair shares a -single task file. A separate `unsupported.yml` fails loudly with a clear +to one task file per package-manager family. Debian and Ubuntu both use +`apt`, so they share a single task file; Fedora and CentOS both use `dnf`, and +CentOS reaches `fedora.yml` through `centos.yml`. A separate `unsupported.yml` fails loudly with a clear message when a host runs a distribution that this role does not yet cover. | `ansible_facts['distribution']` | Task file | Module | | ------------------------------- | ----------------- | -------------------------- | | `Archlinux` | `arch.yml` | `community.general.pacman` | | `Debian` / `Ubuntu` | `debian.yml` | `ansible.builtin.apt` | -| `Fedora` / `CentOS` | `fedora.yml` | `ansible.builtin.dnf` | +| `Fedora` | `fedora.yml` | `ansible.builtin.dnf` | +| `CentOS` | `centos.yml` | `ansible.builtin.dnf` | | anything else | `unsupported.yml` | `ansible.builtin.fail` | +`centos.yml` first runs [centos_mirror.sh](files/shell/centos_mirror.sh), which replaces every `metalink=` of the CentOS Stream repo files with a `baseurl=` on `mirror.stream.centos.org`, and then includes `fedora.yml`. The [Dockerfile](../../Dockerfile) runs the same script before its first `dnf` call, so the CentOS dev and swarm-node images resolve against the same source. + ## Cosmos The diagram places Package update in the Infinito.Nexus cosmos: the components it deploys (capabilities), the central services it consumes (dependencies), and its outward reach (federation and bridged external networks). diff --git a/roles/update/files/shell/centos_mirror.sh b/roles/update/files/shell/centos_mirror.sh new file mode 100755 index 0000000000..d44cbc66aa --- /dev/null +++ b/roles/update/files/shell/centos_mirror.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Param: none. Rewrites the CentOS Stream repo files in /etc/yum.repos.d in place and prints the metalink repo of every section it pinned; prints nothing when there is nothing left to pin. +set -euo pipefail + +master="https://mirror.stream.centos.org" +repo_dir="/etc/yum.repos.d" + +[[ -f "${repo_dir}/centos.repo" ]] || exit 0 + +pin() { + local metalink_repo="$1" path="$2" file + for file in "${repo_dir}/centos.repo" "${repo_dir}/centos-addons.repo"; do + [[ -f "${file}" ]] || continue + grep -q -E "^metalink=https://mirrors\.centos\.org/metalink\?repo=${metalink_repo}&" "${file}" || continue + sed -i -E "s|^metalink=https://mirrors\.centos\.org/metalink\?repo=${metalink_repo}&.*\$|baseurl=${master}/${path}/|" "${file}" + echo "${metalink_repo/\[\$\]stream/\$stream}" + done +} + +declare -A directories=( + [baseos]=BaseOS + [appstream]=AppStream + [crb]=CRB + [highavailability]=HighAvailability + [nfv]=NFV + [rt]=RT + [resilientstorage]=ResilientStorage +) + +for name in "${!directories[@]}"; do + directory="${directories[${name}]}" + pin "centos-${name}-[\$]stream" "\$stream/${directory}/\$basearch/os" + pin "centos-${name}-debug-[\$]stream" "\$stream/${directory}/\$basearch/debug/tree" + pin "centos-${name}-source-[\$]stream" "\$stream/${directory}/source/tree" +done + +pin "centos-extras-sig-extras-common-[\$]stream" "SIGs/\$stream/extras/\$basearch/extras-common" +pin "centos-extras-sig-extras-common-source-[\$]stream" "SIGs/\$stream/extras/source/extras-common" diff --git a/roles/update/tasks/centos.yml b/roles/update/tasks/centos.yml new file mode 100644 index 0000000000..2908c2ddc5 --- /dev/null +++ b/roles/update/tasks/centos.yml @@ -0,0 +1,8 @@ +--- +- name: "๐Ÿ“Œ update (dnf) | pin the CentOS Stream repos to mirror.stream.centos.org" + become: true + ansible.builtin.script: shell/centos_mirror.sh + register: update_centos_pinned + changed_when: update_centos_pinned.stdout | length > 0 + +- include_tasks: fedora.yml diff --git a/roles/update/tasks/fedora.yml b/roles/update/tasks/fedora.yml index dcbae4c85c..a7ad389c9f 100644 --- a/roles/update/tasks/fedora.yml +++ b/roles/update/tasks/fedora.yml @@ -7,8 +7,34 @@ line: "retries=10" insertafter: "^\\[main\\]" -- name: "๐Ÿ”„ update (dnf) | refresh dnf cache and upgrade all installed packages" - ansible.builtin.dnf: # nocheck: dynamic-package - full-system upgrade, not a package install - name: "*" - state: latest - update_cache: true +# Exception: under dnf5, ansible.builtin.dnf's update_cache expires no repo (it queries the repos before loading them), so cached metadata survives until metadata_expire. +- name: "๐Ÿ”„ update (dnf) | re-fetch the metadata of every enabled repository" + become: true + ansible.builtin.command: + argv: [dnf, makecache, --refresh] + changed_when: false + +- name: "๐Ÿ”„ update (dnf) | upgrade all installed packages" + block: + - name: "๐Ÿ”„ update (dnf) | refresh dnf cache and upgrade all installed packages" + ansible.builtin.dnf: # nocheck: dynamic-package - full-system upgrade, not a package install + name: "*" + state: latest + update_cache: true + rescue: + - name: "๐Ÿ›‘ update (dnf) | re-raise anything but packages no mirror serves" # nocheck: rescue-recovery re-fetches metadata a mirror on the previous push served and retries once; the retry fails loud + ansible.builtin.fail: + msg: "{{ ansible_failed_result.msg }}" + when: ansible_failed_result.msg is not match('Failed to download packages') + + - name: "๐Ÿงน update (dnf) | drop the metadata a mirror still on the previous push served" + become: true + ansible.builtin.command: + argv: [dnf, clean, metadata] + changed_when: true + + - name: "๐Ÿ”„ update (dnf) | upgrade once more against fresh metadata" + ansible.builtin.dnf: # nocheck: dynamic-package - full-system upgrade, not a package install + name: "*" + state: latest + update_cache: true diff --git a/roles/update/tasks/main.yml b/roles/update/tasks/main.yml index 7592fdf08e..6a4341e2b8 100644 --- a/roles/update/tasks/main.yml +++ b/roles/update/tasks/main.yml @@ -10,7 +10,7 @@ 'Debian': 'debian.yml', 'Ubuntu': 'debian.yml', 'Fedora': 'fedora.yml', - 'CentOS': 'fedora.yml', + 'CentOS': 'centos.yml', }[ansible_facts['distribution']] | default('unsupported.yml') }} diff --git a/roles/web-app-bigbluebutton/meta/services.yml b/roles/web-app-bigbluebutton/meta/services.yml index 1774c3f158..e93411f153 100644 --- a/roles/web-app-bigbluebutton/meta/services.yml +++ b/roles/web-app-bigbluebutton/meta/services.yml @@ -152,7 +152,7 @@ etherpad: # nocheck: dynamic-flag: fixed upstream-compose sub-service, not group-gated bbb-pads: image: alangecker/bbb-docker-pads - version: "v1.5.11" + version: "v1.5.8" update: type: http_regex url: https://raw.githubusercontent.com/bigbluebutton/docker/develop/repos/tags diff --git a/roles/web-app-bluesky/templates/extra_locations.conf.j2 b/roles/web-app-bluesky/templates/extra_locations.conf.j2 index adf9f6d102..169661909b 100644 --- a/roles/web-app-bluesky/templates/extra_locations.conf.j2 +++ b/roles/web-app-bluesky/templates/extra_locations.conf.j2 @@ -10,6 +10,7 @@ location = /config { set $up_host "{{ BLUESKY_CONFIG_UPSTREAM_URL | regex_replace('^https?://', '') | regex_replace('/.*$', '') }}"; set $bluesky_config_upstream "{{ BLUESKY_CONFIG_UPSTREAM_URL }}"; proxy_pass $bluesky_config_upstream; + {% include 'roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2' %} proxy_set_header Host $up_host; proxy_http_version 1.1; proxy_set_header Connection ""; @@ -24,6 +25,7 @@ location = /ipcc { set $up_host "bsky.app"; set $bluesky_ipcc_upstream "https://bsky.app/ipcc"; proxy_pass $bluesky_ipcc_upstream; + {% include 'roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2' %} proxy_set_header Host $up_host; proxy_http_version 1.1; proxy_set_header Connection ""; diff --git a/roles/web-app-opentalk/templates/nginx/extra.conf.j2 b/roles/web-app-opentalk/templates/nginx/extra.conf.j2 index 35f513c64a..378849a174 100644 --- a/roles/web-app-opentalk/templates/nginx/extra.conf.j2 +++ b/roles/web-app-opentalk/templates/nginx/extra.conf.j2 @@ -5,6 +5,7 @@ location /v1/ { proxy_pass http://127.0.0.1:{{ OPENTALK_CTRL_HTTP_PORT }}; + {% include 'roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2' %} proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -17,6 +18,7 @@ location /v1/ { location /signaling { proxy_pass http://127.0.0.1:{{ OPENTALK_CTRL_HTTP_PORT }}; + {% include 'roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2' %} proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; @@ -30,6 +32,7 @@ location /signaling { location /livekit { proxy_pass http://127.0.0.1:{{ OPENTALK_LIVEKIT_HTTP_PORT }}; + {% include 'roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2' %} proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; diff --git a/roles/web-app-prometheus/templates/nginx/healthz.conf.j2 b/roles/web-app-prometheus/templates/nginx/healthz.conf.j2 index 914f08201d..a034241e6a 100644 --- a/roles/web-app-prometheus/templates/nginx/healthz.conf.j2 +++ b/roles/web-app-prometheus/templates/nginx/healthz.conf.j2 @@ -54,6 +54,7 @@ internal; {{ prometheus_backend }} {% include 'roles/sys-svc-proxy/templates/headers/forwarded.conf.j2' %} + {% include 'roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2' %} proxy_connect_timeout 2s; proxy_read_timeout 3s; proxy_intercept_errors off; @@ -63,6 +64,7 @@ access_log off; {{ prometheus_ready_backend }} {% include 'roles/sys-svc-proxy/templates/headers/forwarded.conf.j2' %} + {% include 'roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2' %} proxy_connect_timeout 3s; proxy_read_timeout 5s; proxy_intercept_errors off; diff --git a/scripts/install/filesystem.sh b/scripts/install/filesystem.sh index 61f8428cdc..31b6b52136 100755 --- a/scripts/install/filesystem.sh +++ b/scripts/install/filesystem.sh @@ -112,13 +112,13 @@ install_pacman() { install_dnf_native() { case "${ID:-}" in fedora) - retry dnf install -y e2fsprogs btrfs-progs + retry dnf --refresh install -y e2fsprogs btrfs-progs return ;; esac log "adding EPEL, the only place the EL family carries btrfs-progs" - retry dnf install -y epel-release - retry dnf install -y e2fsprogs btrfs-progs + retry dnf --refresh install -y epel-release + retry dnf --refresh install -y e2fsprogs btrfs-progs } # Param: $1 path segment of the zfsonlinux repository, `fedora` or `epel` @@ -155,7 +155,7 @@ install_dnf() { log "zfs userland offered by the repository: ${zfs_version}" if ! command -v rpmbuild >/dev/null 2>&1; then - retry dnf install -y rpm-build + retry dnf --refresh install -y rpm-build fi local topdir=/tmp/zfs-kmod-container @@ -180,7 +180,7 @@ SPEC rpmbuild --define "_topdir ${topdir}" -bb "${topdir}/SPECS/zfs-kmod-container.spec" dnf install -y "${topdir}"/RPMS/noarch/zfs-kmod-container-*.rpm - retry dnf install -y zfs + retry dnf --refresh install -y zfs if rpm -qa 'kernel-devel*' 'kernel-debug*' 'zfs-dkms' | grep -q .; then log "ERROR: the transaction pulled kernel or DKMS packages, so the dummy did not take" diff --git a/scripts/tests/workspace/compose/01_minimal.sh b/scripts/tests/workspace/compose/01_minimal.sh index 783c7ffa06..1c718d388a 100755 --- a/scripts/tests/workspace/compose/01_minimal.sh +++ b/scripts/tests/workspace/compose/01_minimal.sh @@ -11,8 +11,9 @@ source "${SCRIPT_DIR}/../utils/cache.sh" echo "Snapshotting cache counters before the deploy." CACHE_BEFORE="$(cache_snapshot)" -echo "Deploying dashboard with all variable (group-conditional) services disabled to speed up the deploy and verify disable= suppresses them from the inventory." -make compose-deploy mode=reinstall apps="${DASHBOARD_APP}" disable="$(variable_services dashboard javascript)" +echo "Deploying dashboard with all variable (group-conditional) services disabled, except the asset provider, to speed up the deploy and verify disable= suppresses them from the inventory." +# Exception: without the asset provider the dashboard loads its logo from the public file.infinito.nexus, which ties this test to that host's uptime; variant 1 pins the provider off, so only round 0 runs here. +make compose-deploy mode=reinstall apps="${DASHBOARD_APP}" variant=0 disable="$(variable_services dashboard javascript asset)" inspect echo "Actively probing both caches to confirm pull-through works end-to-end." diff --git a/tests/lint/ansible/jinja/test_proxy_location_strips.py b/tests/lint/ansible/jinja/test_proxy_location_strips.py new file mode 100644 index 0000000000..93b750fc31 --- /dev/null +++ b/tests/lint/ansible/jinja/test_proxy_location_strips.py @@ -0,0 +1,90 @@ +"""Lint: every proxying location the front proxy includes strips upstream headers. + +``proxy_hide_header`` is replaced, not merged, by a location that sets its own, +so ``upstream_strips.conf.j2`` only takes effect where a location includes it. +A location fragment without it leaks ``X-Powered-By`` and the upstream's +``Strict-Transport-Security`` on that path alone. + +Scope: role templates without a ``server`` block. Those are the fragments the +front proxy renders into a vhost; a template with its own ``server`` block is +the config of an app-internal nginx that sits behind such a location. +""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path + +from utils.cache.files import iter_project_files, read_text + +from . import PROJECT_ROOT + +_STRIPS = "roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2" +_SERVER_BLOCK = re.compile(r"^\s*server\s*\{", re.MULTILINE) +_LOCATION_OPEN = re.compile(r"^\s*location\b[^{]*\{\s*$") +_PROXYING = re.compile(r"\bproxy_pass\b|lookup\('proxy_pass'|_backend\s*\}\}") + + +def _role_templates() -> list[str]: + return sorted( + path + for path in iter_project_files(extensions=(".j2",), exclude_tests=True) + if "/roles/" in path and "/templates/" in path + ) + + +def _location_blocks(lines: list[str]) -> list[tuple[int, str]]: + blocks: list[tuple[int, str]] = [] + index = 0 + while index < len(lines): + if not _LOCATION_OPEN.match(lines[index]): + index += 1 + continue + start = index + depth = lines[index].count("{") - lines[index].count("}") + body: list[str] = [] + index += 1 + while index < len(lines) and depth > 0: + depth += lines[index].count("{") - lines[index].count("}") + body.append(lines[index]) + index += 1 + blocks.append((start + 1, "\n".join(body))) + return blocks + + +class TestProxyLocationStrips(unittest.TestCase): + def test_at_least_one_fragment_is_checked(self) -> None: + """Otherwise the assertion below passes over an empty list.""" + checked = [ + path + for path in _role_templates() + if not _SERVER_BLOCK.search(read_text(path)) + and any( + _PROXYING.search(body) + for _, body in _location_blocks(read_text(path).splitlines()) + ) + ] + self.assertTrue(checked, "no proxying location fragment found to check") + + def test_proxying_locations_include_the_strips(self) -> None: + findings: list[str] = [] + for path in _role_templates(): + text = read_text(path) + if _SERVER_BLOCK.search(text): + continue + rel = Path(path).relative_to(PROJECT_ROOT).as_posix() + for line, body in _location_blocks(text.splitlines()): + if _PROXYING.search(body) and _STRIPS not in body: + findings.append(f"- {rel}:{line}") + if findings: + self.fail( + "These proxying locations do not include the upstream header strips, " + "so X-Powered-By and the upstream HSTS leak on their path:\n" + + "\n".join(findings) + + f"\n\nFix: add `{{% include '{_STRIPS}' %}}` inside each location." + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/update/test_source.py b/tests/unit/python/utils/update/test_source.py index e10f86c77f..df905623d2 100644 --- a/tests/unit/python/utils/update/test_source.py +++ b/tests/unit/python/utils/update/test_source.py @@ -179,7 +179,10 @@ class TestOutdated(unittest.TestCase): class TestTemplate(unittest.TestCase): def _outdated(self, services: str, upstream: str) -> list[str]: root = _repo(services) - with mock.patch.object(module, "documents", return_value=upstream): + with ( + mock.patch.object(module, "documents", return_value=upstream), + mock.patch.object(module, "manifest_exists", return_value=True), + ): updates = module.outdated(module.collect_entries(root), root) return [update.latest for update in updates] @@ -206,6 +209,31 @@ class TestTemplate(unittest.TestCase): ) +class TestPulledImage(unittest.TestCase): + def _outdated(self, exists: bool | None) -> tuple[list[str], mock.MagicMock]: + root = _repo(COMPOUND) + with ( + mock.patch.object( + module, "documents", return_value="core v3.0.39\npart v1.11.3\n" + ), + mock.patch.object(module, "manifest_exists", return_value=exists) as probe, + ): + updates = module.outdated(module.collect_entries(root), root) + return [update.latest for update in updates], probe + + def test_a_tag_the_image_registry_lacks_is_not_pinned(self) -> None: + latest, probe = self._outdated(False) + self.assertEqual(latest, []) + probe.assert_called_once_with("example/part", "v1.11.3-v3.0.39") + + def test_an_indeterminate_registry_answer_keeps_the_bump(self) -> None: + self.assertEqual(self._outdated(None)[0], ["v1.11.3-v3.0.39"]) + + def test_only_the_version_of_a_pulled_image_carries_the_image(self) -> None: + entries = {e.key: e.image for e in module.collect_entries(_repo())} + self.assertEqual(entries, {"app_version": "", "version": "example"}) + + class TestAddons(unittest.TestCase): def test_only_a_declared_or_github_released_addon_pin_is_collected(self) -> None: entries = module.collect_entries(_addon_repo()) diff --git a/utils/update/source.py b/utils/update/source.py index 0cf9166337..3d6405feaa 100644 --- a/utils/update/source.py +++ b/utils/update/source.py @@ -39,6 +39,7 @@ from urllib.parse import urlencode from utils.annotations.suppress import is_suppressed_at from utils.cache.files import read_text from utils.cache.yaml import load_yaml +from utils.docker.registry import manifest_exists from utils.roles.mapping import ROLE_FILE_META_SERVICES from utils.update.addons import GITHUB_RELEASES_CATALOG, iter_addon_files from utils.update.base import ( @@ -83,6 +84,10 @@ class VersionSourceEntry: config_path: file that carries the pin. line: 1-indexed line of the pin. addon: whether the pin sits in ``meta/addons/.yml``. + image: image the pin tags when it is the entity's ``version``, pulled + directly or as the base of a ``custom`` build; a bump to a tag + its registry answers 404 for is withheld, an indeterminate answer + does not block it. """ role: str @@ -93,6 +98,7 @@ class VersionSourceEntry: config_path: Path line: int addon: bool = False + image: str = "" @dataclass(frozen=True) @@ -442,6 +448,7 @@ def collect_entries(repo_root: Path) -> list[VersionSourceEntry]: source=source, config_path=config_path, line=line, + image=config.get("image", "") if key == DEFAULT_KEY else "", ) ) return entries + _addon_entries(repo_root) @@ -458,8 +465,11 @@ def outdated( candidates(entry, repo_root), assembled=bool(entry.source.get("template")), ) - if newest: - updates.append(VersionSourceUpdate(entry=entry, latest=newest)) + if not newest: + continue + if entry.image and manifest_exists(entry.image, newest) is False: + continue + updates.append(VersionSourceUpdate(entry=entry, latest=newest)) return updates