From cd6dc485379b06dd6b9cffcd11e32338a582da5e Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 07:34:47 +0000 Subject: [PATCH 01/10] update(images): update Docker image versions (web-app-socialhome) --- roles/web-app-socialhome/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-socialhome/meta/services.yml b/roles/web-app-socialhome/meta/services.yml index 4f9f1999bd..07717ac4ed 100644 --- a/roles/web-app-socialhome/meta/services.yml +++ b/roles/web-app-socialhome/meta/services.yml @@ -13,7 +13,7 @@ socialhome: # nocheck: default-placement-manager embedded sqlite in a node-local volume; no clustered storage placement: manager image: ghcr.io/social-home-io/socialhome - version: "2026.10.29" + version: "2026.10.30" ports: local: http: 8089 From c1ff8e5d11a6c837efd9980e1bd0f063863e300e Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 30 Oct 2026 08:57:32 +0100 Subject: [PATCH 02/10] fix(web-app-bigbluebutton): pin apps-akka's transaction isolation At BBB boot, bbb-graphql-server's start.sh runs pg_terminate_backend and drops bbb_graphql. When that kills apps-akka's first HikariCP connection while the pool detects the driver's default transaction isolation, HikariCP 4.0.3 swallows the 08xxx error, marks detection as done and keeps isolation -1 for the pool's lifetime. Every later connection setup then calls setTransactionIsolation(-1), which pgjdbc rejects with "Transaction isolation level -1 not supported", so meetings never reach the database and the client shows "Oops" (about 1 in 8 boots). Pin the isolation explicitly so HikariCP never falls back to the detected value: slick-hikaricp reads postgres.transactionIsolation, prepends TRANSACTION_ and hands it to HikariCP, hence the value must be exactly READ_COMMITTED (the PostgreSQL default, so normal boots behave as before). The flag goes into apps-akka's own compose environment as JAVA_OPTS, which the sbt-native-packager start script honours, instead of env.j2, because compose_mods pins env.j2 onto every bbb service and would hand the flag to bbb-web and fsesl-akka as well. Co-Authored-By: Claude Opus 5.5 --- roles/web-app-bigbluebutton/templates/compose.override.yml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 b/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 index fbcaaa9711..e0a2c213b8 100644 --- a/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 +++ b/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 @@ -30,6 +30,8 @@ services: {{ service_name }}: {{ lookup('container_image', application_id, service_name) }} {{ lookup('container_volumes', service_name) | indent(4) }} + environment: + JAVA_OPTS: "-Dpostgres.transactionIsolation=READ_COMMITTED" {% set service_name = 'bbb-graphql-server' %} {{ service_name }}: {{ lookup('container_image', application_id, service_name) }} From 46fe01179680f4de4e2cfc2649382695f7151700 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 30 Oct 2026 08:57:44 +0100 Subject: [PATCH 03/10] fix(web-app-fusiondirectory): lift the PHP memory_limit in the FPM pool #1259 moved the interim CVE-2026-109618 mitigation (memory_limit = -1) out of the shared hardening.ini into memory-unlimited.ini and mounted it only in the roles whose images it had inventoried. Its branch predates #1263, which gave FusionDirectory the php_hardening mount, so after both merged FusionDirectory ran with the image default of 128M again. A conf.d mount would not have fixed it: the nfrastack image writes php_admin_value[memory_limit] into its FPM pool (/etc/php7/pools.d/www/www-settings.conf) from PHP_MEMORY_LIMIT, and a pool php_admin_value outranks every conf.d ini. Measured on main: CLI 128M, FPM 128M. So even before #1259 the hardening ini only ever lifted the CLI limit. nfrastack/fusiondirectory:1.5 runs PHP 7.4.33 on Alpine 3.15, end of life and never patched, so the role carries the mitigation until it moves to a PHP 8 image. - Set PHP_MEMORY_LIMIT=-1 in templates/env.j2. The image seds it into php.ini and the pool, so CLI and FPM both report -1. - The root TODO lists FusionDirectory with the image and the reason it uses the variable instead of the mount; the role TODO names which setting carries which CVE mitigation. Co-Authored-By: Claude Opus 5.5 --- TODO.md | 2 +- roles/web-app-fusiondirectory/TODO.md | 2 +- roles/web-app-fusiondirectory/templates/env.j2 | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/TODO.md b/TODO.md index 273e3021a6..be8e6ea770 100644 --- a/TODO.md +++ b/TODO.md @@ -4,7 +4,7 @@ - Implement rbac administration interface - Enable IP6 for docker. - backup docker to local für ca optimieren -- Drop the `php_memory_unlimited` mount ([memory-unlimited.ini](roles/sys-svc-compose-php/files/memory-unlimited.ini), `memory_limit = -1`) from each role whose image ships PHP 8.2.35, 8.3.36, 8.4.27, 8.5.12 or later, and the file, its install task and `FILE_PHP_MEMORY_UNLIMITED_INI` once no role mounts it. It is the interim mitigation for CVE-2026-109618 and overrides the per-role `service_memory_mb` share, so a single request can exhaust the container instead of hitting a PHP limit. Still mounted by `web-app-magento` (`markoshust/magento-php:8.4-fpm`, PHP 8.4.21), `web-app-mailu` (`ghcr.io/heviat/webmail:2024.06`, the OIDC flavor, built 2025-04-04), `web-app-phpldapadmin` (`phpldapadmin/phpldapadmin:2.3.11`, built 2026-05-15) and `web-app-shopware` (`ghcr.io/shopware/docker-base:8.3`, PHP 8.3.25). `web-app-magento` keeps the 756M its `nginx.conf.sample` (`PHP_VALUE`) and `pub/.user.ini` set and keeps `post_max_size` below a quarter of that limit. +- Drop the `php_memory_unlimited` mount ([memory-unlimited.ini](roles/sys-svc-compose-php/files/memory-unlimited.ini), `memory_limit = -1`) from each role whose image ships PHP 8.2.35, 8.3.36, 8.4.27, 8.5.12 or later, and the file, its install task and `FILE_PHP_MEMORY_UNLIMITED_INI` once no role mounts it. It is the interim mitigation for CVE-2026-109618 and overrides the per-role `service_memory_mb` share, so a single request can exhaust the container instead of hitting a PHP limit. Still mounted by `web-app-magento` (`markoshust/magento-php:8.4-fpm`, PHP 8.4.21), `web-app-mailu` (`ghcr.io/heviat/webmail:2024.06`, the OIDC flavor, built 2025-04-04), `web-app-phpldapadmin` (`phpldapadmin/phpldapadmin:2.3.11`, built 2026-05-15) and `web-app-shopware` (`ghcr.io/shopware/docker-base:8.3`, PHP 8.3.25). `web-app-fusiondirectory` (`nfrastack/fusiondirectory:1.5`, PHP 7.4.33 on Alpine 3.15, end of life and never patched) carries it as `PHP_MEMORY_LIMIT=-1` in `templates/env.j2` instead (the image's pool sets `php_admin_value[memory_limit]` from it, so a conf.d ini cannot reach FPM) and drops it under the same rule. `web-app-magento` keeps the 756M its `nginx.conf.sample` (`PHP_VALUE`) and `pub/.user.ini` set and keeps `post_max_size` below a quarter of that limit. - Fork [rpardini/docker-registry-proxy](https://github.com/rpardini/docker-registry-proxy) and teach the bundled nginx to handle GCP Artifact Registry redirect URLs (`/artifacts-downloads/namespaces//repositories//downloads/`). Currently `gcr.io` blob pulls are bypassed via `NO_PROXY=…,gcr.io,storage.googleapis.com,googleusercontent.com` in [compose/registry-cache/proxy.conf](compose/registry-cache/proxy.conf), which is a workaround that disables caching for those registries. Long-term: rebase upstream's `proxy.conf` / `sub_filter` rules to recognise the new redirect shape, ship the fix in our fork, point `compose/registry-cache/Dockerfile` (or its image tag) at the fork, and drop the `NO_PROXY` bypass. ## Testing diff --git a/roles/web-app-fusiondirectory/TODO.md b/roles/web-app-fusiondirectory/TODO.md index f20419e2c3..1e47da11c4 100644 --- a/roles/web-app-fusiondirectory/TODO.md +++ b/roles/web-app-fusiondirectory/TODO.md @@ -1,4 +1,4 @@ # To-dos - Move off PHP 7.4. The pinned `nfrastack/fusiondirectory:1.5` image builds on `ghcr.io/nfrastack/container-nginx-php-fpm` with PHP 7.4 on Alpine 3.15, which is end of life and gets no fix for CVE-2026-109618 and CVE-2026-109619. - - No FusionDirectory release runs on a supported PHP yet: upstream certifies 1.4 to 1.6 for PHP 7.4 only, `nfrastack/container-fusiondirectory` builds its image only on PHP 7.4 and Alpine 3.15, and the PHP 8 port lives on upstream's unreleased `php8` branch. Until then the shared hardening ini mounted at `/etc/php7/conf.d/zz-infinito-hardening.ini` carries the mitigations for both CVEs. Options: rebuild the image on upstream's first `php8` release, or build `nfrastack/container-fusiondirectory` against a pinned `php8` commit on `container-nginx-php-fpm` 8.4 before that release. + - No FusionDirectory release runs on a supported PHP yet: upstream certifies 1.4 to 1.6 for PHP 7.4 only, `nfrastack/container-fusiondirectory` builds its image only on PHP 7.4 and Alpine 3.15, and the PHP 8 port lives on upstream's unreleased `php8` branch. Until then the shared hardening ini mounted at `/etc/php7/conf.d/zz-infinito-hardening.ini` carries the CVE-2026-109619 mitigation and `PHP_MEMORY_LIMIT=-1` in `templates/env.j2` the CVE-2026-109618 one. Options: rebuild the image on upstream's first `php8` release, or build `nfrastack/container-fusiondirectory` against a pinned `php8` commit on `container-nginx-php-fpm` 8.4 before that release. diff --git a/roles/web-app-fusiondirectory/templates/env.j2 b/roles/web-app-fusiondirectory/templates/env.j2 index a6fd6a7bf4..340112e5fe 100644 --- a/roles/web-app-fusiondirectory/templates/env.j2 +++ b/roles/web-app-fusiondirectory/templates/env.j2 @@ -8,3 +8,4 @@ LDAP01_PORT={{ lookup('config', 'svc-db-openldap', 'services.openldap.ports.loca LDAP01_ADMIN_DN={{ LDAP.DN.ADMINISTRATOR.DATA }} LDAP01_ADMIN_PASS={{ LDAP.BIND_CREDENTIAL | dotenv_quote }} LDAP01_BASE_DN={{ LDAP.DN.ROOT }} +PHP_MEMORY_LIMIT=-1 From df00ff0abfd78cfecc2114059733d30aa03071f5 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 30 Oct 2026 09:17:06 +0100 Subject: [PATCH 04/10] fix(web-app-matrix): turn off Element's forced verification in variant 1 4a56b4c19c set services.element.force_verification to false only for the SSO variants 0 and 2, on the reasoning that only the SSO specs log the same account in twice. The deploy's second pass logs the variant 1 administrator (ansible flavor, SSO off, password login) in again. PASS 1 bootstraps the account's cross-signing identity, so in PASS 2 Element stops on "Confirm your digital identity" ("Use another device" / "Can't confirm?" / "Remove this device", no skip), and the password-login spec times out at _shared.js:436 (fork run 40979252521, job 123823820263, Discourse#1 with web-app-matrix variant 1, all three retries). Variant 1 now sets force_verification to false like the other two. The services.yml default stays true, so production keeps forced verification. No variant exercises true end to end any more; the follow-up issue restores one. Co-Authored-By: Claude Opus 5.5 --- roles/web-app-matrix/meta/variants.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/web-app-matrix/meta/variants.yml b/roles/web-app-matrix/meta/variants.yml index 537076f63d..a55875e719 100644 --- a/roles/web-app-matrix/meta/variants.yml +++ b/roles/web-app-matrix/meta/variants.yml @@ -50,6 +50,8 @@ container_backup: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true + element: + force_verification: false matrix: flavor: ansible coturn: From d3f3800ed88802f8c3d7518559e203481ce28b25 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 30 Oct 2026 10:27:34 +0100 Subject: [PATCH 05/10] docs(web-app-akaunting): drop the stale PHP 8.1 to-do The role builds akaunting/docker at master. Upstream moved that Dockerfile to `FROM php:8.2.35-apache` (akaunting/docker#123, merged 2026-10-26), a release with the fixes for CVE-2026-109618 and CVE-2026-109619, so the to-do that asked to move off PHP 8.1.34 no longer describes the image the role builds. Closes #1119 Co-Authored-By: Claude Opus 5.5 --- roles/web-app-akaunting/TODO.md | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/web-app-akaunting/TODO.md b/roles/web-app-akaunting/TODO.md index 437f188b19..08cc6561fb 100644 --- a/roles/web-app-akaunting/TODO.md +++ b/roles/web-app-akaunting/TODO.md @@ -1,5 +1,4 @@ # To-dos - Enable OIDC -- Move off PHP 8.1. The role builds the image from `akaunting/docker` (`master`), whose Dockerfile starts `FROM php:8.1-apache` and runs PHP 8.1.34, which is end of life and gets no upstream fix for CVE-2026-109618 and CVE-2026-109619. Build on a PHP 8.2 or later base instead. - Add OIDC auto-provisioning so a fresh Keycloak user that lands on the akaunting tile lands on a working akaunting user account instead of a "no permission" surface. Until that is wired, BOTH the playwright biber and administrator personas are blocked via `PERSONA_BIBER_BLOCKED=true` and `PERSONA_ADMINISTRATOR_BLOCKED=true` in `templates/playwright.env.j2` (req 019 Rule 11 explicit opt-out). Once auto-provisioning works, drop both opt-out flags so the personas drive the real journey. From 69a0dfa9b924665cd65f76085b2c9d3476e222f8 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 30 Oct 2026 14:53:54 +0100 Subject: [PATCH 06/10] fix(sys-svc-webserver-core): size proxy workers from the proxy alone The shared OpenResty nginx.conf took worker_processes and worker_connections from WEBSERVER_WORKER_*, which are the min() of svc-prx-openresty's limits and the current play's application_id. The file renders once, in the scope of whichever role loads the proxy first, so that role's limits capped every vhost: on feature/store-i18n web-svc-mirror (pids_limit 64) loaded it first, every vhost got worker_connections 64, and openresty raised 555 "64 worker_connections are not enough while connecting to upstream" alerts while a single openwebui page fires ~150-170 chunk requests (CI run 40979252521, homeassistant#1). Main carries the same wiring. Add WEBSERVER_PROXY_WORKER_PROCESSES and WEBSERVER_PROXY_WORKER_CONNECTIONS, computed from svc-prx-openresty's own openresty service only (same resource_filter fallbacks, at least one worker), and render nginx.conf from them. WEBSERVER_WORKER_* keep pairing the proxy with the play for app-internal nginx such as Nextcloud's. The values can only stay equal or rise. Found by the i18n session's triage of homeassistant#1. Co-Authored-By: Claude Opus 5.5 --- group_vars/all/04_webserver.yml | 12 ++ .../templates/nginx.conf.j2 | 4 +- .../test_webserver_worker_limits.py | 146 ++++++++++++++++++ 3 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 tests/unit/python/group_vars/test_webserver_worker_limits.py diff --git a/group_vars/all/04_webserver.yml b/group_vars/all/04_webserver.yml index 5168689cb9..45f467e50c 100644 --- a/group_vars/all/04_webserver.yml +++ b/group_vars/all/04_webserver.yml @@ -25,6 +25,18 @@ WEBSERVER_WORKER_CONNECTIONS: >- ] | min }} +# Not WEBSERVER_WORKER_*: their application_id is whichever role loaded the shared proxy first, whose limits then cap every vhost. +WEBSERVER_PROXY_WORKER_PROCESSES: >- + {{ + [ + (lookup('applications') | resource_filter('svc-prx-openresty', 'cpus', '', RESOURCE_CPUS)) | float | int, + 1 + ] | max + }} + +WEBSERVER_PROXY_WORKER_CONNECTIONS: >- + {{ lookup('applications') | resource_filter('svc-prx-openresty', 'pids_limit', '', RESOURCE_PIDS_LIMIT) | int }} + # Response security headers of every vhost; an app overrides any key under `security_headers` in its meta/server.yml. WEBSERVER_SECURITY_HEADERS: nosniff: false # X-Content-Type-Options: nosniff diff --git a/roles/sys-svc-webserver-core/templates/nginx.conf.j2 b/roles/sys-svc-webserver-core/templates/nginx.conf.j2 index e8cfe19861..97b3e84dbf 100644 --- a/roles/sys-svc-webserver-core/templates/nginx.conf.j2 +++ b/roles/sys-svc-webserver-core/templates/nginx.conf.j2 @@ -1,7 +1,7 @@ -worker_processes {{ WEBSERVER_WORKER_PROCESSES }}; +worker_processes {{ WEBSERVER_PROXY_WORKER_PROCESSES }}; events { - worker_connections {{ WEBSERVER_WORKER_CONNECTIONS }}; + worker_connections {{ WEBSERVER_PROXY_WORKER_CONNECTIONS }}; } http { diff --git a/tests/unit/python/group_vars/test_webserver_worker_limits.py b/tests/unit/python/group_vars/test_webserver_worker_limits.py new file mode 100644 index 0000000000..16a9997311 --- /dev/null +++ b/tests/unit/python/group_vars/test_webserver_worker_limits.py @@ -0,0 +1,146 @@ +"""The shared proxy's nginx.conf takes its worker limits from the proxy alone. + +``nginx.conf`` of ``sys-svc-webserver-core`` renders once, in the scope of +whichever role loads ``svc-prx-openresty`` first. ``WEBSERVER_WORKER_*`` pair the +proxy with that role's ``application_id``, so a 64-pid role capped every vhost at +64 worker_connections. +""" + +from __future__ import annotations + +import unittest +from pathlib import Path +from typing import Any, ClassVar + +from ansible.parsing.dataloader import DataLoader +from ansible.template import Templar, trust_as_template + +from utils.cache import _reset_cache_for_tests +from utils.cache.carrier import ( + APPLICATIONS_RENDERED_FACT, + merged_applications_cache_key, +) +from utils.cache.files import PROJECT_ROOT, read_text +from utils.cache.yaml import load_yaml + +_WEBSERVER_VARS = Path(PROJECT_ROOT) / "group_vars" / "all" / "04_webserver.yml" +_NGINX_CONF = ( + Path(PROJECT_ROOT) + / "roles" + / "sys-svc-webserver-core" + / "templates" + / "nginx.conf.j2" +) + +_FIRST_LOADER = "web-svc-mirror" + + +def _applications(proxy_cpus: float) -> dict[str, Any]: + """Return a proxy with 256 pids and a first-loading role with 64 pids, 1 cpu. + + Args: + proxy_cpus: the proxy's ``cpus``. + """ + return { + "svc-prx-openresty": { + "services": {"openresty": {"cpus": proxy_cpus, "pids_limit": 256}} + }, + _FIRST_LOADER: {"services": {"mirror": {"cpus": 1, "pids_limit": 64}}}, + } + + +class TestWebserverWorkerLimits(unittest.TestCase): + group_vars: ClassVar[dict[str, Any]] + + @classmethod + def setUpClass(cls) -> None: + cls.group_vars = load_yaml(_WEBSERVER_VARS) + + def setUp(self) -> None: + _reset_cache_for_tests() + self.addCleanup(_reset_cache_for_tests) + + def _render( + self, expression: str, applications: dict[str, Any], **scope: Any + ) -> str: + """Render *expression* through Ansible's Templar over the group_vars. + + Args: + expression: Jinja source, e.g. ``"{{ WEBSERVER_WORKER_CONNECTIONS }}"``. + applications: payload ``lookup('applications')`` serves. + scope: play variables added on top of ``application_id``. + + Returns: + The rendered text, stripped. + """ + variables: dict[str, Any] = { + key: trust_as_template(value) if isinstance(value, str) else value + for key, value in self.group_vars.items() + } + variables.update( + application_id=_FIRST_LOADER, + RESOURCE_CPUS="0.5", + RESOURCE_PIDS_LIMIT=512, + **scope, + ) + key = merged_applications_cache_key(variables) + variables[APPLICATIONS_RENDERED_FACT] = { + "key": [key[0], list(key[1])], + "applications": applications, + } + templar = Templar(loader=DataLoader(), variables=variables) + return str(templar.template(trust_as_template(expression))).strip() + + def test_proxy_connections_follow_the_proxy_alone(self) -> None: + for scope in ({}, {"service_name": "mirror"}): + with self.subTest(**scope): + self.assertEqual( + self._render( + "{{ WEBSERVER_PROXY_WORKER_CONNECTIONS }}", + _applications(4), + **scope, + ), + "256", + ) + + def test_proxy_processes_follow_the_proxy_alone(self) -> None: + for scope in ({}, {"service_name": "mirror"}): + with self.subTest(**scope): + self.assertEqual( + self._render( + "{{ WEBSERVER_PROXY_WORKER_PROCESSES }}", + _applications(4), + **scope, + ), + "4", + ) + + def test_a_sub_cpu_proxy_still_runs_one_worker(self) -> None: + self.assertEqual( + self._render("{{ WEBSERVER_PROXY_WORKER_PROCESSES }}", _applications(0.5)), + "1", + ) + + def test_app_internal_limits_still_pair_with_the_play(self) -> None: + applications = _applications(4) + self.assertEqual( + self._render("{{ WEBSERVER_WORKER_CONNECTIONS }}", applications), "64" + ) + self.assertEqual( + self._render("{{ WEBSERVER_WORKER_PROCESSES }}", applications), "1" + ) + + def test_shared_proxy_nginx_conf_uses_the_proxy_limits(self) -> None: + worker_lines = [ + line.strip() + for line in read_text(str(_NGINX_CONF)).splitlines() + if line.strip().startswith("worker_") + ] + rendered = self._render("\n".join(worker_lines), _applications(4)) + self.assertEqual( + rendered.splitlines(), ["worker_processes 4;", "worker_connections 256;"] + ) + + +if __name__ == "__main__": + unittest.main() From f0513f32d766baf55029f9a692c24fc85f531e80 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 13:55:43 +0000 Subject: [PATCH 07/10] update(images): update Docker image versions (web-app-funkwhale) --- roles/web-app-funkwhale/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-funkwhale/meta/services.yml b/roles/web-app-funkwhale/meta/services.yml index 19774a94f1..54ca082b9f 100644 --- a/roles/web-app-funkwhale/meta/services.yml +++ b/roles/web-app-funkwhale/meta/services.yml @@ -11,7 +11,7 @@ funkwhale: # nocheck: default-placement-manager embedded typesense index; node-local in swarm placement: manager image: funkwhale/front - version: 2.0.12 + version: 2.0.13 min_storage: 224MB ports: internal: From b894db0a758a54aad20d08d8eaac226a030485f5 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 30 Oct 2026 16:51:44 +0100 Subject: [PATCH 08/10] fix(svc-bkp-volume-2-local): restore into the resolved volume mountpoint rsync 3.5.0 (Debian security update 3.5.0+ds1-0+deb13u1) refuses a destination symlink owned by neither root nor the calling user. svc-opt-ssd-hdd turns a volume's `_data` into a symlink to /mnt/hdd or /mnt/ssd. The restore drill empties the volume and runs `compose create`; docker repopulates it and lchowns the `_data` symlink to the image directory's owner (e.g. www-data). The restore then failed with "refusing to follow a symlink owned by an untrusted user" and "change_dir#1 .../_data/ failed: Too many levels of symbolic links", the drill left its subjects down, and the dependent checks timed out (CI run 40979252521, ssd-hdd#0, on feature/store-i18n; recover.py is identical on main). VolumeRecovery now resolves the mountpoint before handing it to rsync: os.path.realpath locally, `ssh realpath -e -- ` for --docker-host, so rsync writes into the real directory and never meets the symlink. `--insecure-links` was not used because older rsync builds on other distributions lack it. Two new unit tests (local symlinked mountpoint, remote resolution); both fail against the previous code. Root cause and fix by the i18n session (feature/store-i18n triage of ssd-hdd#0), ported to main. Co-Authored-By: Claude Opus 5.5 --- .../files/python/recover.py | 21 +++++-- .../recovery/test_role_recover_scripts.py | 59 +++++++++++++++++++ 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/roles/svc-bkp-volume-2-local/files/python/recover.py b/roles/svc-bkp-volume-2-local/files/python/recover.py index 5d4d0829bf..65022e973f 100755 --- a/roles/svc-bkp-volume-2-local/files/python/recover.py +++ b/roles/svc-bkp-volume-2-local/files/python/recover.py @@ -25,6 +25,7 @@ Usage: from __future__ import annotations import argparse +import os import subprocess import sys from pathlib import Path @@ -50,6 +51,7 @@ class VolumeRecovery(DirectoryRecovery): service_backup: bool = True, docker_host: str | None = None, ) -> None: + host = docker_host.split("://", 1)[-1] if docker_host else None docker = ["docker", *(["-H", docker_host] if docker_host else [])] mountpoint = subprocess.run( [*docker, "volume", "inspect", "--format", "{{.Mountpoint}}", volume], @@ -65,8 +67,8 @@ class VolumeRecovery(DirectoryRecovery): ).stdout.strip() if options not in ("", "null", "{}"): probe = ( - ["ssh", docker_host.split("://", 1)[-1], "mountpoint", "-q", mountpoint] - if docker_host + ["ssh", host, "mountpoint", "-q", mountpoint] + if host else ["mountpoint", "-q", mountpoint] ) if subprocess.run(probe, check=False).returncode != 0: @@ -77,11 +79,18 @@ class VolumeRecovery(DirectoryRecovery): "disk and be shadowed by the real backing store on the next mount. " "Start a container that holds the volume, then retry." ) - target = ( - f"{docker_host.split('://', 1)[-1]}:{mountpoint}" - if docker_host - else mountpoint + # Exception: rsync >= 3.5.0 refuses a destination symlink not owned by root, and svc-opt-ssd-hdd's _data is one. + resolved = ( + subprocess.run( + ["ssh", host, "realpath", "-e", "--", mountpoint], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if host + else os.path.realpath(mountpoint) ) + target = f"{host}:{resolved}" if host else resolved super().__init__(source_dir, target, service_backup=service_backup) diff --git a/tests/unit/python/utils/recovery/test_role_recover_scripts.py b/tests/unit/python/utils/recovery/test_role_recover_scripts.py index a2756be5e6..b86d30f748 100644 --- a/tests/unit/python/utils/recovery/test_role_recover_scripts.py +++ b/tests/unit/python/utils/recovery/test_role_recover_scripts.py @@ -69,6 +69,65 @@ class VolumeRecoverTests(unittest.TestCase): self.assertEqual(rec.target_dir, mount) self.assertFalse(rec.service_backup) + def test_restores_into_the_target_of_a_symlinked_mountpoint(self): + mod = _load("svc-bkp-volume-2-local", "volume_recover") + with tempfile.TemporaryDirectory() as td: + source = Path(td) / "source" + source.mkdir() + moved = Path(td) / "hdd" / "myvol" + moved.mkdir(parents=True) + mount = Path(td) / "volumes" / "myvol" / "_data" + mount.parent.mkdir(parents=True) + mount.symlink_to(moved) + with mock.patch.object( + mod.subprocess, + "run", + side_effect=[ + mock.Mock(stdout=f"{mount}\n"), + mock.Mock(stdout="null\n"), + ], + ): + rec = mod.VolumeRecovery(str(source), "myvol", service_backup=False) + self.assertEqual(rec.target_dir, moved.resolve()) + with mock.patch.object(mod.subprocess, "run") as run: + rec.restore() + self.assertEqual(run.call_args.args[0][-1], f"{moved.resolve()}/") + + def test_resolves_a_remote_mountpoint_on_the_remote_host(self): + mod = _load("svc-bkp-volume-2-local", "volume_recover") + with tempfile.TemporaryDirectory() as td: + source = Path(td) / "source" + source.mkdir() + with mock.patch.object( + mod.subprocess, + "run", + side_effect=[ + mock.Mock(stdout="/var/lib/docker/volumes/x/_data\n"), + mock.Mock(stdout="null\n"), + mock.Mock(stdout="/mnt/hdd/data/docker/volumes/x\n"), + ], + ) as run: + rec = mod.VolumeRecovery( + str(source), + "myvol", + service_backup=False, + docker_host="ssh://root@node-1", + ) + self.assertEqual( + run.call_args_list[2].args[0], + [ + "ssh", + "root@node-1", + "realpath", + "-e", + "--", + "/var/lib/docker/volumes/x/_data", + ], + ) + self.assertEqual( + str(rec.target_dir), "root@node-1:/mnt/hdd/data/docker/volumes/x" + ) + def test_refuses_unmounted_backing_store(self): mod = _load("svc-bkp-volume-2-local", "volume_recover") with tempfile.TemporaryDirectory() as td: From 2d80ad5f1b1cd1652cdd4dfa2c332ba7a9419970 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 30 Oct 2026 16:51:56 +0100 Subject: [PATCH 09/10] fix(web-app-openproject): wait for the swarm seeder and detect dead DB sockets Compose gates web, worker and cron on the seeder via depends_on service_completed_successfully. Swarm ignores depends_on, so nothing waited for the one-shot seeder there. In CI (store-i18n run 40979252521, OpenProject#0 swarm) an overlay flap reset the seeder's Postgres connection; Postgres rolled back RootSeeder's transaction, but the seeder never noticed and sat idle on the dead socket forever ("Executing database seed..."). No UserCustomFieldSection rows were committed, so /my/account rendered without fieldsets and the deploy still reported green. - 01_manager_ops.yml: in swarm, after the handler flush deploys the stack, poll job_state.sh on the seeder service until its newest task is Complete (fail on Shutdown, give up after 20 minutes), mirroring web-app-zammad's init-job wait. Compose keeps relying on depends_on. - env.j2: add libpq TCP keepalives (idle 60s, interval 10s, count 6) and tcp_user_timeout=120000 to DATABASE_URL, so a connection whose peer vanished errors out within about two minutes instead of hanging. The seeder then exits non-zero and swarm's on-failure restart policy reruns the seed. Co-Authored-By: Claude Opus 5.5 --- roles/web-app-openproject/tasks/01_manager_ops.yml | 12 ++++++++++++ roles/web-app-openproject/templates/env.j2 | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/roles/web-app-openproject/tasks/01_manager_ops.yml b/roles/web-app-openproject/tasks/01_manager_ops.yml index bd0331f1d7..39bb5a9a80 100644 --- a/roles/web-app-openproject/tasks/01_manager_ops.yml +++ b/roles/web-app-openproject/tasks/01_manager_ops.yml @@ -29,6 +29,18 @@ - name: 🚿 flush docker service meta: flush_handlers +- name: ⏳ Wait until the seeder has committed the seed (swarm) + ansible.builtin.script: "{{ lookup('path_absolute', 'roles/sys-svc-compose/files/shell/swarm/job_state.sh') }}" + environment: + JOB: "{{ lookup('container_service', application_id, OPENPROJECT_SEEDER_SERVICE) }}" + register: openproject_seeder_state + changed_when: false + retries: 120 + delay: 10 + until: openproject_seeder_state.rc != 1 + failed_when: openproject_seeder_state.rc != 0 + when: DEPLOYMENT_MODE == 'swarm' + - name: 🚀 Deploy Settings include_tasks: 02_settings.yml diff --git a/roles/web-app-openproject/templates/env.j2 b/roles/web-app-openproject/templates/env.j2 index cc684cdccc..bbbb560daf 100644 --- a/roles/web-app-openproject/templates/env.j2 +++ b/roles/web-app-openproject/templates/env.j2 @@ -19,7 +19,7 @@ OPENPROJECT_AUTH__SOURCE__SSO_LOGOUT__URL={{ OPENPROJECT_SSO_LOGOUT_URL }} {% endif %} POSTGRES_PASSWORD={{ lookup('database', application_id, 'password') | dotenv_quote }} -DATABASE_URL={{ lookup('database', application_id, 'url_full') }}?pool=20&encoding=unicode&reconnect=true +DATABASE_URL={{ lookup('database', application_id, 'url_full') }}?pool=20&encoding=unicode&reconnect=true&keepalives=1&keepalives_idle=60&keepalives_interval=10&keepalives_count=6&tcp_user_timeout=120000 DB_POOL={{ OPENPROJECT_DB_POOL }} RAILS_MAX_THREADS={{ OPENPROJECT_RAILS_MAX_THREADS }} RAILS_MIN_THREADS=4 From 9c1969e97c3796dfffc6236c3efe6d9ab805b6d9 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:54:01 +0000 Subject: [PATCH 10/10] update(images): update Docker image versions (web-app-funkwhale) --- roles/web-app-funkwhale/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-funkwhale/meta/services.yml b/roles/web-app-funkwhale/meta/services.yml index 54ca082b9f..e009110245 100644 --- a/roles/web-app-funkwhale/meta/services.yml +++ b/roles/web-app-funkwhale/meta/services.yml @@ -36,7 +36,7 @@ api: flavor: curl path: api/v1/instance/nodeinfo/2.0 image: funkwhale/api - version: 2.0.12 + version: 2.0.13 min_storage: 588MB ports: internal: