diff --git a/TODO.md b/TODO.md index 273e3021a6..be8e6ea770 100644 --- a/TODO.md +++ b/TODO.md @@ -4,7 +4,7 @@ - Implement rbac administration interface - Enable IP6 for docker. - backup docker to local für ca optimieren -- Drop the `php_memory_unlimited` mount ([memory-unlimited.ini](roles/sys-svc-compose-php/files/memory-unlimited.ini), `memory_limit = -1`) from each role whose image ships PHP 8.2.35, 8.3.36, 8.4.27, 8.5.12 or later, and the file, its install task and `FILE_PHP_MEMORY_UNLIMITED_INI` once no role mounts it. It is the interim mitigation for CVE-2026-109618 and overrides the per-role `service_memory_mb` share, so a single request can exhaust the container instead of hitting a PHP limit. Still mounted by `web-app-magento` (`markoshust/magento-php:8.4-fpm`, PHP 8.4.21), `web-app-mailu` (`ghcr.io/heviat/webmail:2024.06`, the OIDC flavor, built 2025-04-04), `web-app-phpldapadmin` (`phpldapadmin/phpldapadmin:2.3.11`, built 2026-05-15) and `web-app-shopware` (`ghcr.io/shopware/docker-base:8.3`, PHP 8.3.25). `web-app-magento` keeps the 756M its `nginx.conf.sample` (`PHP_VALUE`) and `pub/.user.ini` set and keeps `post_max_size` below a quarter of that limit. +- Drop the `php_memory_unlimited` mount ([memory-unlimited.ini](roles/sys-svc-compose-php/files/memory-unlimited.ini), `memory_limit = -1`) from each role whose image ships PHP 8.2.35, 8.3.36, 8.4.27, 8.5.12 or later, and the file, its install task and `FILE_PHP_MEMORY_UNLIMITED_INI` once no role mounts it. It is the interim mitigation for CVE-2026-109618 and overrides the per-role `service_memory_mb` share, so a single request can exhaust the container instead of hitting a PHP limit. Still mounted by `web-app-magento` (`markoshust/magento-php:8.4-fpm`, PHP 8.4.21), `web-app-mailu` (`ghcr.io/heviat/webmail:2024.06`, the OIDC flavor, built 2025-04-04), `web-app-phpldapadmin` (`phpldapadmin/phpldapadmin:2.3.11`, built 2026-05-15) and `web-app-shopware` (`ghcr.io/shopware/docker-base:8.3`, PHP 8.3.25). `web-app-fusiondirectory` (`nfrastack/fusiondirectory:1.5`, PHP 7.4.33 on Alpine 3.15, end of life and never patched) carries it as `PHP_MEMORY_LIMIT=-1` in `templates/env.j2` instead (the image's pool sets `php_admin_value[memory_limit]` from it, so a conf.d ini cannot reach FPM) and drops it under the same rule. `web-app-magento` keeps the 756M its `nginx.conf.sample` (`PHP_VALUE`) and `pub/.user.ini` set and keeps `post_max_size` below a quarter of that limit. - Fork [rpardini/docker-registry-proxy](https://github.com/rpardini/docker-registry-proxy) and teach the bundled nginx to handle GCP Artifact Registry redirect URLs (`/artifacts-downloads/namespaces//repositories//downloads/`). Currently `gcr.io` blob pulls are bypassed via `NO_PROXY=…,gcr.io,storage.googleapis.com,googleusercontent.com` in [compose/registry-cache/proxy.conf](compose/registry-cache/proxy.conf), which is a workaround that disables caching for those registries. Long-term: rebase upstream's `proxy.conf` / `sub_filter` rules to recognise the new redirect shape, ship the fix in our fork, point `compose/registry-cache/Dockerfile` (or its image tag) at the fork, and drop the `NO_PROXY` bypass. ## Testing diff --git a/group_vars/all/04_webserver.yml b/group_vars/all/04_webserver.yml index 5168689cb9..45f467e50c 100644 --- a/group_vars/all/04_webserver.yml +++ b/group_vars/all/04_webserver.yml @@ -25,6 +25,18 @@ WEBSERVER_WORKER_CONNECTIONS: >- ] | min }} +# Not WEBSERVER_WORKER_*: their application_id is whichever role loaded the shared proxy first, whose limits then cap every vhost. +WEBSERVER_PROXY_WORKER_PROCESSES: >- + {{ + [ + (lookup('applications') | resource_filter('svc-prx-openresty', 'cpus', '', RESOURCE_CPUS)) | float | int, + 1 + ] | max + }} + +WEBSERVER_PROXY_WORKER_CONNECTIONS: >- + {{ lookup('applications') | resource_filter('svc-prx-openresty', 'pids_limit', '', RESOURCE_PIDS_LIMIT) | int }} + # Response security headers of every vhost; an app overrides any key under `security_headers` in its meta/server.yml. WEBSERVER_SECURITY_HEADERS: nosniff: false # X-Content-Type-Options: nosniff diff --git a/roles/svc-bkp-volume-2-local/files/python/recover.py b/roles/svc-bkp-volume-2-local/files/python/recover.py index 5d4d0829bf..65022e973f 100755 --- a/roles/svc-bkp-volume-2-local/files/python/recover.py +++ b/roles/svc-bkp-volume-2-local/files/python/recover.py @@ -25,6 +25,7 @@ Usage: from __future__ import annotations import argparse +import os import subprocess import sys from pathlib import Path @@ -50,6 +51,7 @@ class VolumeRecovery(DirectoryRecovery): service_backup: bool = True, docker_host: str | None = None, ) -> None: + host = docker_host.split("://", 1)[-1] if docker_host else None docker = ["docker", *(["-H", docker_host] if docker_host else [])] mountpoint = subprocess.run( [*docker, "volume", "inspect", "--format", "{{.Mountpoint}}", volume], @@ -65,8 +67,8 @@ class VolumeRecovery(DirectoryRecovery): ).stdout.strip() if options not in ("", "null", "{}"): probe = ( - ["ssh", docker_host.split("://", 1)[-1], "mountpoint", "-q", mountpoint] - if docker_host + ["ssh", host, "mountpoint", "-q", mountpoint] + if host else ["mountpoint", "-q", mountpoint] ) if subprocess.run(probe, check=False).returncode != 0: @@ -77,11 +79,18 @@ class VolumeRecovery(DirectoryRecovery): "disk and be shadowed by the real backing store on the next mount. " "Start a container that holds the volume, then retry." ) - target = ( - f"{docker_host.split('://', 1)[-1]}:{mountpoint}" - if docker_host - else mountpoint + # Exception: rsync >= 3.5.0 refuses a destination symlink not owned by root, and svc-opt-ssd-hdd's _data is one. + resolved = ( + subprocess.run( + ["ssh", host, "realpath", "-e", "--", mountpoint], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if host + else os.path.realpath(mountpoint) ) + target = f"{host}:{resolved}" if host else resolved super().__init__(source_dir, target, service_backup=service_backup) diff --git a/roles/sys-svc-webserver-core/templates/nginx.conf.j2 b/roles/sys-svc-webserver-core/templates/nginx.conf.j2 index e8cfe19861..97b3e84dbf 100644 --- a/roles/sys-svc-webserver-core/templates/nginx.conf.j2 +++ b/roles/sys-svc-webserver-core/templates/nginx.conf.j2 @@ -1,7 +1,7 @@ -worker_processes {{ WEBSERVER_WORKER_PROCESSES }}; +worker_processes {{ WEBSERVER_PROXY_WORKER_PROCESSES }}; events { - worker_connections {{ WEBSERVER_WORKER_CONNECTIONS }}; + worker_connections {{ WEBSERVER_PROXY_WORKER_CONNECTIONS }}; } http { diff --git a/roles/web-app-akaunting/TODO.md b/roles/web-app-akaunting/TODO.md index 437f188b19..08cc6561fb 100644 --- a/roles/web-app-akaunting/TODO.md +++ b/roles/web-app-akaunting/TODO.md @@ -1,5 +1,4 @@ # To-dos - Enable OIDC -- Move off PHP 8.1. The role builds the image from `akaunting/docker` (`master`), whose Dockerfile starts `FROM php:8.1-apache` and runs PHP 8.1.34, which is end of life and gets no upstream fix for CVE-2026-109618 and CVE-2026-109619. Build on a PHP 8.2 or later base instead. - Add OIDC auto-provisioning so a fresh Keycloak user that lands on the akaunting tile lands on a working akaunting user account instead of a "no permission" surface. Until that is wired, BOTH the playwright biber and administrator personas are blocked via `PERSONA_BIBER_BLOCKED=true` and `PERSONA_ADMINISTRATOR_BLOCKED=true` in `templates/playwright.env.j2` (req 019 Rule 11 explicit opt-out). Once auto-provisioning works, drop both opt-out flags so the personas drive the real journey. diff --git a/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 b/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 index fbcaaa9711..e0a2c213b8 100644 --- a/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 +++ b/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 @@ -30,6 +30,8 @@ services: {{ service_name }}: {{ lookup('container_image', application_id, service_name) }} {{ lookup('container_volumes', service_name) | indent(4) }} + environment: + JAVA_OPTS: "-Dpostgres.transactionIsolation=READ_COMMITTED" {% set service_name = 'bbb-graphql-server' %} {{ service_name }}: {{ lookup('container_image', application_id, service_name) }} diff --git a/roles/web-app-funkwhale/meta/services.yml b/roles/web-app-funkwhale/meta/services.yml index 19774a94f1..e009110245 100644 --- a/roles/web-app-funkwhale/meta/services.yml +++ b/roles/web-app-funkwhale/meta/services.yml @@ -11,7 +11,7 @@ funkwhale: # nocheck: default-placement-manager embedded typesense index; node-local in swarm placement: manager image: funkwhale/front - version: 2.0.12 + version: 2.0.13 min_storage: 224MB ports: internal: @@ -36,7 +36,7 @@ api: flavor: curl path: api/v1/instance/nodeinfo/2.0 image: funkwhale/api - version: 2.0.12 + version: 2.0.13 min_storage: 588MB ports: internal: diff --git a/roles/web-app-fusiondirectory/TODO.md b/roles/web-app-fusiondirectory/TODO.md index f20419e2c3..1e47da11c4 100644 --- a/roles/web-app-fusiondirectory/TODO.md +++ b/roles/web-app-fusiondirectory/TODO.md @@ -1,4 +1,4 @@ # To-dos - Move off PHP 7.4. The pinned `nfrastack/fusiondirectory:1.5` image builds on `ghcr.io/nfrastack/container-nginx-php-fpm` with PHP 7.4 on Alpine 3.15, which is end of life and gets no fix for CVE-2026-109618 and CVE-2026-109619. - - No FusionDirectory release runs on a supported PHP yet: upstream certifies 1.4 to 1.6 for PHP 7.4 only, `nfrastack/container-fusiondirectory` builds its image only on PHP 7.4 and Alpine 3.15, and the PHP 8 port lives on upstream's unreleased `php8` branch. Until then the shared hardening ini mounted at `/etc/php7/conf.d/zz-infinito-hardening.ini` carries the mitigations for both CVEs. Options: rebuild the image on upstream's first `php8` release, or build `nfrastack/container-fusiondirectory` against a pinned `php8` commit on `container-nginx-php-fpm` 8.4 before that release. + - No FusionDirectory release runs on a supported PHP yet: upstream certifies 1.4 to 1.6 for PHP 7.4 only, `nfrastack/container-fusiondirectory` builds its image only on PHP 7.4 and Alpine 3.15, and the PHP 8 port lives on upstream's unreleased `php8` branch. Until then the shared hardening ini mounted at `/etc/php7/conf.d/zz-infinito-hardening.ini` carries the CVE-2026-109619 mitigation and `PHP_MEMORY_LIMIT=-1` in `templates/env.j2` the CVE-2026-109618 one. Options: rebuild the image on upstream's first `php8` release, or build `nfrastack/container-fusiondirectory` against a pinned `php8` commit on `container-nginx-php-fpm` 8.4 before that release. diff --git a/roles/web-app-fusiondirectory/templates/env.j2 b/roles/web-app-fusiondirectory/templates/env.j2 index a6fd6a7bf4..340112e5fe 100644 --- a/roles/web-app-fusiondirectory/templates/env.j2 +++ b/roles/web-app-fusiondirectory/templates/env.j2 @@ -8,3 +8,4 @@ LDAP01_PORT={{ lookup('config', 'svc-db-openldap', 'services.openldap.ports.loca LDAP01_ADMIN_DN={{ LDAP.DN.ADMINISTRATOR.DATA }} LDAP01_ADMIN_PASS={{ LDAP.BIND_CREDENTIAL | dotenv_quote }} LDAP01_BASE_DN={{ LDAP.DN.ROOT }} +PHP_MEMORY_LIMIT=-1 diff --git a/roles/web-app-matrix/meta/variants.yml b/roles/web-app-matrix/meta/variants.yml index 537076f63d..a55875e719 100644 --- a/roles/web-app-matrix/meta/variants.yml +++ b/roles/web-app-matrix/meta/variants.yml @@ -50,6 +50,8 @@ container_backup: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true + element: + force_verification: false matrix: flavor: ansible coturn: diff --git a/roles/web-app-openproject/tasks/01_manager_ops.yml b/roles/web-app-openproject/tasks/01_manager_ops.yml index bd0331f1d7..39bb5a9a80 100644 --- a/roles/web-app-openproject/tasks/01_manager_ops.yml +++ b/roles/web-app-openproject/tasks/01_manager_ops.yml @@ -29,6 +29,18 @@ - name: 🚿 flush docker service meta: flush_handlers +- name: ⏳ Wait until the seeder has committed the seed (swarm) + ansible.builtin.script: "{{ lookup('path_absolute', 'roles/sys-svc-compose/files/shell/swarm/job_state.sh') }}" + environment: + JOB: "{{ lookup('container_service', application_id, OPENPROJECT_SEEDER_SERVICE) }}" + register: openproject_seeder_state + changed_when: false + retries: 120 + delay: 10 + until: openproject_seeder_state.rc != 1 + failed_when: openproject_seeder_state.rc != 0 + when: DEPLOYMENT_MODE == 'swarm' + - name: 🚀 Deploy Settings include_tasks: 02_settings.yml diff --git a/roles/web-app-openproject/templates/env.j2 b/roles/web-app-openproject/templates/env.j2 index cc684cdccc..bbbb560daf 100644 --- a/roles/web-app-openproject/templates/env.j2 +++ b/roles/web-app-openproject/templates/env.j2 @@ -19,7 +19,7 @@ OPENPROJECT_AUTH__SOURCE__SSO_LOGOUT__URL={{ OPENPROJECT_SSO_LOGOUT_URL }} {% endif %} POSTGRES_PASSWORD={{ lookup('database', application_id, 'password') | dotenv_quote }} -DATABASE_URL={{ lookup('database', application_id, 'url_full') }}?pool=20&encoding=unicode&reconnect=true +DATABASE_URL={{ lookup('database', application_id, 'url_full') }}?pool=20&encoding=unicode&reconnect=true&keepalives=1&keepalives_idle=60&keepalives_interval=10&keepalives_count=6&tcp_user_timeout=120000 DB_POOL={{ OPENPROJECT_DB_POOL }} RAILS_MAX_THREADS={{ OPENPROJECT_RAILS_MAX_THREADS }} RAILS_MIN_THREADS=4 diff --git a/roles/web-app-socialhome/meta/services.yml b/roles/web-app-socialhome/meta/services.yml index 4f9f1999bd..07717ac4ed 100644 --- a/roles/web-app-socialhome/meta/services.yml +++ b/roles/web-app-socialhome/meta/services.yml @@ -13,7 +13,7 @@ socialhome: # nocheck: default-placement-manager embedded sqlite in a node-local volume; no clustered storage placement: manager image: ghcr.io/social-home-io/socialhome - version: "2026.10.29" + version: "2026.10.30" ports: local: http: 8089 diff --git a/tests/unit/python/group_vars/test_webserver_worker_limits.py b/tests/unit/python/group_vars/test_webserver_worker_limits.py new file mode 100644 index 0000000000..16a9997311 --- /dev/null +++ b/tests/unit/python/group_vars/test_webserver_worker_limits.py @@ -0,0 +1,146 @@ +"""The shared proxy's nginx.conf takes its worker limits from the proxy alone. + +``nginx.conf`` of ``sys-svc-webserver-core`` renders once, in the scope of +whichever role loads ``svc-prx-openresty`` first. ``WEBSERVER_WORKER_*`` pair the +proxy with that role's ``application_id``, so a 64-pid role capped every vhost at +64 worker_connections. +""" + +from __future__ import annotations + +import unittest +from pathlib import Path +from typing import Any, ClassVar + +from ansible.parsing.dataloader import DataLoader +from ansible.template import Templar, trust_as_template + +from utils.cache import _reset_cache_for_tests +from utils.cache.carrier import ( + APPLICATIONS_RENDERED_FACT, + merged_applications_cache_key, +) +from utils.cache.files import PROJECT_ROOT, read_text +from utils.cache.yaml import load_yaml + +_WEBSERVER_VARS = Path(PROJECT_ROOT) / "group_vars" / "all" / "04_webserver.yml" +_NGINX_CONF = ( + Path(PROJECT_ROOT) + / "roles" + / "sys-svc-webserver-core" + / "templates" + / "nginx.conf.j2" +) + +_FIRST_LOADER = "web-svc-mirror" + + +def _applications(proxy_cpus: float) -> dict[str, Any]: + """Return a proxy with 256 pids and a first-loading role with 64 pids, 1 cpu. + + Args: + proxy_cpus: the proxy's ``cpus``. + """ + return { + "svc-prx-openresty": { + "services": {"openresty": {"cpus": proxy_cpus, "pids_limit": 256}} + }, + _FIRST_LOADER: {"services": {"mirror": {"cpus": 1, "pids_limit": 64}}}, + } + + +class TestWebserverWorkerLimits(unittest.TestCase): + group_vars: ClassVar[dict[str, Any]] + + @classmethod + def setUpClass(cls) -> None: + cls.group_vars = load_yaml(_WEBSERVER_VARS) + + def setUp(self) -> None: + _reset_cache_for_tests() + self.addCleanup(_reset_cache_for_tests) + + def _render( + self, expression: str, applications: dict[str, Any], **scope: Any + ) -> str: + """Render *expression* through Ansible's Templar over the group_vars. + + Args: + expression: Jinja source, e.g. ``"{{ WEBSERVER_WORKER_CONNECTIONS }}"``. + applications: payload ``lookup('applications')`` serves. + scope: play variables added on top of ``application_id``. + + Returns: + The rendered text, stripped. + """ + variables: dict[str, Any] = { + key: trust_as_template(value) if isinstance(value, str) else value + for key, value in self.group_vars.items() + } + variables.update( + application_id=_FIRST_LOADER, + RESOURCE_CPUS="0.5", + RESOURCE_PIDS_LIMIT=512, + **scope, + ) + key = merged_applications_cache_key(variables) + variables[APPLICATIONS_RENDERED_FACT] = { + "key": [key[0], list(key[1])], + "applications": applications, + } + templar = Templar(loader=DataLoader(), variables=variables) + return str(templar.template(trust_as_template(expression))).strip() + + def test_proxy_connections_follow_the_proxy_alone(self) -> None: + for scope in ({}, {"service_name": "mirror"}): + with self.subTest(**scope): + self.assertEqual( + self._render( + "{{ WEBSERVER_PROXY_WORKER_CONNECTIONS }}", + _applications(4), + **scope, + ), + "256", + ) + + def test_proxy_processes_follow_the_proxy_alone(self) -> None: + for scope in ({}, {"service_name": "mirror"}): + with self.subTest(**scope): + self.assertEqual( + self._render( + "{{ WEBSERVER_PROXY_WORKER_PROCESSES }}", + _applications(4), + **scope, + ), + "4", + ) + + def test_a_sub_cpu_proxy_still_runs_one_worker(self) -> None: + self.assertEqual( + self._render("{{ WEBSERVER_PROXY_WORKER_PROCESSES }}", _applications(0.5)), + "1", + ) + + def test_app_internal_limits_still_pair_with_the_play(self) -> None: + applications = _applications(4) + self.assertEqual( + self._render("{{ WEBSERVER_WORKER_CONNECTIONS }}", applications), "64" + ) + self.assertEqual( + self._render("{{ WEBSERVER_WORKER_PROCESSES }}", applications), "1" + ) + + def test_shared_proxy_nginx_conf_uses_the_proxy_limits(self) -> None: + worker_lines = [ + line.strip() + for line in read_text(str(_NGINX_CONF)).splitlines() + if line.strip().startswith("worker_") + ] + rendered = self._render("\n".join(worker_lines), _applications(4)) + self.assertEqual( + rendered.splitlines(), ["worker_processes 4;", "worker_connections 256;"] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/recovery/test_role_recover_scripts.py b/tests/unit/python/utils/recovery/test_role_recover_scripts.py index a2756be5e6..b86d30f748 100644 --- a/tests/unit/python/utils/recovery/test_role_recover_scripts.py +++ b/tests/unit/python/utils/recovery/test_role_recover_scripts.py @@ -69,6 +69,65 @@ class VolumeRecoverTests(unittest.TestCase): self.assertEqual(rec.target_dir, mount) self.assertFalse(rec.service_backup) + def test_restores_into_the_target_of_a_symlinked_mountpoint(self): + mod = _load("svc-bkp-volume-2-local", "volume_recover") + with tempfile.TemporaryDirectory() as td: + source = Path(td) / "source" + source.mkdir() + moved = Path(td) / "hdd" / "myvol" + moved.mkdir(parents=True) + mount = Path(td) / "volumes" / "myvol" / "_data" + mount.parent.mkdir(parents=True) + mount.symlink_to(moved) + with mock.patch.object( + mod.subprocess, + "run", + side_effect=[ + mock.Mock(stdout=f"{mount}\n"), + mock.Mock(stdout="null\n"), + ], + ): + rec = mod.VolumeRecovery(str(source), "myvol", service_backup=False) + self.assertEqual(rec.target_dir, moved.resolve()) + with mock.patch.object(mod.subprocess, "run") as run: + rec.restore() + self.assertEqual(run.call_args.args[0][-1], f"{moved.resolve()}/") + + def test_resolves_a_remote_mountpoint_on_the_remote_host(self): + mod = _load("svc-bkp-volume-2-local", "volume_recover") + with tempfile.TemporaryDirectory() as td: + source = Path(td) / "source" + source.mkdir() + with mock.patch.object( + mod.subprocess, + "run", + side_effect=[ + mock.Mock(stdout="/var/lib/docker/volumes/x/_data\n"), + mock.Mock(stdout="null\n"), + mock.Mock(stdout="/mnt/hdd/data/docker/volumes/x\n"), + ], + ) as run: + rec = mod.VolumeRecovery( + str(source), + "myvol", + service_backup=False, + docker_host="ssh://root@node-1", + ) + self.assertEqual( + run.call_args_list[2].args[0], + [ + "ssh", + "root@node-1", + "realpath", + "-e", + "--", + "/var/lib/docker/volumes/x/_data", + ], + ) + self.assertEqual( + str(rec.target_dir), "root@node-1:/mnt/hdd/data/docker/volumes/x" + ) + def test_refuses_unmounted_backing_store(self): mod = _load("svc-bkp-volume-2-local", "volume_recover") with tempfile.TemporaryDirectory() as td: