From 883b9c4f46b338f6fc1aefc9bf413118ac44c10f Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 6 Oct 2026 10:03:49 +0200 Subject: [PATCH 01/13] fix(web-app-prometheus): probe apps where their vhosts answer Every blackbox probe of a Tor deployment answered 404, so AppDown fired for all deployed apps although they were up. The job built its targets from canonical_domains_map(DOMAIN_PRIMARY), which knows no onion domain. In a Tor deployment the vhosts answer under . only, and the front proxy served the clearnet names from its default server. The targets now come from lookup('domains'), the map the vhosts are built from, and each target takes its scheme from the domain it probes instead of from the Prometheus vhost. Which roles are probed follows lookup('roles_with_service', 'prometheus'), the list the role's own spec expects metrics for. It leaves out the database roles, which have a domain in the map and no vhost, and the roles that declare scrape: false because their vhost carries no Prometheus block. Two roles declared more than their vhost delivers. web-svc-mirror renders a vhost of its own with no backend behind it, and on a live stack its probe answered 404 under the onion name as well, so it declares scrape: false now. web-app-peertube has a backend, but its vhost template never included the Prometheus locations. It includes them now. The spec could not see any of this in a compose deployment. dotenv_quote wraps PROMETHEUS_TARGET_ROLES_JSON in quotes there, JSON.parse returned a string, the role list came out empty and the per-role test was skipped. The value is decoded now, and the test also requires probe_success == 1 for every role in the list. Closes #792 Co-Authored-By: Claude Opus 5.5 (1M context) --- .../web-app-peertube/templates/proxy.conf.j2 | 2 + .../files/playwright/playwright.spec.js | 23 ++++- .../templates/configuration/prometheus.yml.j2 | 7 +- roles/web-svc-mirror/meta/services.yml | 2 + .../roles/web-app-prometheus/__init__.py | 0 .../test_blackbox_targets.py | 84 +++++++++++++++++++ 6 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 tests/unit/python/roles/web-app-prometheus/__init__.py create mode 100644 tests/unit/python/roles/web-app-prometheus/test_blackbox_targets.py diff --git a/roles/web-app-peertube/templates/proxy.conf.j2 b/roles/web-app-peertube/templates/proxy.conf.j2 index 1049d7bfdb..b8a8a94803 100644 --- a/roles/web-app-peertube/templates/proxy.conf.j2 +++ b/roles/web-app-peertube/templates/proxy.conf.j2 @@ -51,4 +51,6 @@ server { try_files /dev/null {{ webserver_websocket_location }}; } + + {% include 'roles/web-app-prometheus/templates/nginx/locations.conf.j2' %} } \ No newline at end of file diff --git a/roles/web-app-prometheus/files/playwright/playwright.spec.js b/roles/web-app-prometheus/files/playwright/playwright.spec.js index c3a8ecf077..f0c374918b 100644 --- a/roles/web-app-prometheus/files/playwright/playwright.spec.js +++ b/roles/web-app-prometheus/files/playwright/playwright.spec.js @@ -211,7 +211,7 @@ test("prometheus: biber is denied access after sso login", async ({ browser }) = // ----------------------------------------------------------------------------- const prometheusTargetRoles = (() => { - const raw = process.env.PROMETHEUS_TARGET_ROLES_JSON || "[]"; + const raw = decodeDotenvQuotedValue(process.env.PROMETHEUS_TARGET_ROLES_JSON || "[]"); try { const parsed = JSON.parse(raw); return Array.isArray(parsed) ? parsed : []; @@ -220,7 +220,7 @@ const prometheusTargetRoles = (() => { } })(); -test("prometheus scrape: every consumer role reports up=1", async ({ page }) => { +test("prometheus scrape: every consumer role reports up=1 and passes its probe", async ({ page }) => { test.skip(prometheusTargetRoles.length === 0, "no prometheus consumer roles in inventory"); const expectedPrometheusBaseUrl = prometheusBaseUrl.replace(/\/$/, ""); @@ -279,6 +279,25 @@ test("prometheus scrape: every consumer role reports up=1", async ({ page }) => failures, `prometheus scrape failures:\n - ${failures.join("\n - ")}` ).toEqual([]); + + const probeUrl = `${expectedPrometheusBaseUrl}/api/v1/query?query=${encodeURIComponent('probe_success{job="blackbox-healthz"}')}`; + await expect + .poll( + async () => { + const probes = (await (await page.request.get(probeUrl, { ignoreHTTPSErrors: true })).json())?.data?.result || []; + return prometheusTargetRoles + .filter((target) => { + const own = probes.filter((entry) => entry?.metric?.app === target.id); + return own.length === 0 || own.some((entry) => entry?.value?.[1] !== "1"); + }) + .map((target) => target.id); + }, + { + timeout: resolveTimeout(120_000), + message: "roles whose /healthz/ready probe is missing or failing (probe_success != 1)", + }, + ) + .toEqual([]); }); // Persona scenarios. diff --git a/roles/web-app-prometheus/templates/configuration/prometheus.yml.j2 b/roles/web-app-prometheus/templates/configuration/prometheus.yml.j2 index 35cd730a20..86ea8e71aa 100644 --- a/roles/web-app-prometheus/templates/configuration/prometheus.yml.j2 +++ b/roles/web-app-prometheus/templates/configuration/prometheus.yml.j2 @@ -41,20 +41,21 @@ scrape_configs: {% endfor %} -{% set canonical_map = lookup('applications') | canonical_domains_map(DOMAIN_PRIMARY) %} +{% set canonical_map = lookup('domains') %} +{% set probed_roles = lookup('roles_with_service', 'prometheus') | map(attribute='id') | list %} - job_name: "blackbox-healthz" metrics_path: /probe params: module: [http_ready] static_configs: {% for _application_id in CURRENT_PLAY_APPLICATIONS | list | sort %} -{% if _application_id != application_id %} +{% if _application_id in probed_roles %} {% set domains_raw = canonical_map.get(_application_id, []) %} {% set canonical_probe_domains = (domains_raw.values() | list) if (domains_raw is mapping) else domains_raw %} {% if canonical_probe_domains | length > 0 %} - targets: {% for domain in canonical_probe_domains %} - - "{{ lookup('tls', application_id, 'protocols.web') }}://{{ domain }}/healthz/ready" + - "{{ lookup('tls', domain, 'url.base') }}healthz/ready" {% endfor %} labels: app: "{{ _application_id }}" diff --git a/roles/web-svc-mirror/meta/services.yml b/roles/web-svc-mirror/meta/services.yml index 398a51e444..2540a50b14 100644 --- a/roles/web-svc-mirror/meta/services.yml +++ b/roles/web-svc-mirror/meta/services.yml @@ -22,6 +22,8 @@ prometheus: bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" + # nocheck: prometheus-scrape + scrape: false # nocheck: playwright-service-flag # nocheck: variants-coverage — sso-variant-tor requires the onion on in every variant, so the false polarity has no variant left to live in tor: diff --git a/tests/unit/python/roles/web-app-prometheus/__init__.py b/tests/unit/python/roles/web-app-prometheus/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/roles/web-app-prometheus/test_blackbox_targets.py b/tests/unit/python/roles/web-app-prometheus/test_blackbox_targets.py new file mode 100644 index 0000000000..abc914988d --- /dev/null +++ b/tests/unit/python/roles/web-app-prometheus/test_blackbox_targets.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import unittest + +from jinja2 import Environment, FileSystemLoader, StrictUndefined, select_autoescape + +from utils.cache.files import PROJECT_ROOT +from utils.cache.yaml import load_yaml_str + +TEMPLATE_DIR = PROJECT_ROOT / "roles/web-app-prometheus/templates/configuration" +ONION = "a" * 56 + ".onion" + +DOMAINS = { + "web-app-prometheus": [f"prometheus.{ONION}"], + "web-app-nextcloud": {"nextcloud": f"next.cloud.{ONION}"}, + "web-app-gitea": ["git.infinito.test"], + "web-svc-cdn": ["cdn.infinito.test"], + "svc-db-postgres": ["postgres.infinito.test"], +} +ROLES_WITH_THE_PROMETHEUS_SERVICE = ["web-app-nextcloud", "web-app-gitea"] + + +def lookup(name, *terms, **_kwargs): + if name == "domains": + return DOMAINS + if name == "roles_with_service": + return [{"id": role} for role in ROLES_WITH_THE_PROMETHEUS_SERVICE] + if name == "tls": + term, want = terms + scheme = "http" if term.endswith(".onion") else "https" + return { + "url.base": f"{scheme}://{term}/", + "protocols.web": "http", + "enabled": False, + }[want] + if name == "domain": + return DOMAINS[terms[0]][0] + if name == "scrape_target": + return f"{terms[1]}:9000" + if name == "native_metrics_apps": + return [] + raise AssertionError(f"the template calls an unexpected lookup: {name!r}") + + +def probed_targets() -> dict[str, list[str]]: + env = Environment( + loader=FileSystemLoader(str(TEMPLATE_DIR)), + undefined=StrictUndefined, + autoescape=select_autoescape(), + ) + env.filters["bool"] = bool + rendered = env.get_template("prometheus.yml.j2").render( + lookup=lookup, + application_id="web-app-prometheus", + CURRENT_PLAY_APPLICATIONS=list(DOMAINS), + DOMAIN_PRIMARY="infinito.test", + ALERT_RULES_CONFIG_DOCKER="/etc/prometheus/alert_rules.yml", + ) + jobs = {job["job_name"]: job for job in load_yaml_str(rendered)["scrape_configs"]} + return { + group["labels"]["app"]: group["targets"] + for group in jobs["blackbox-healthz"]["static_configs"] + } + + +class TestBlackboxTargets(unittest.TestCase): + def test_an_onion_app_is_probed_under_its_onion_name_over_http(self): + self.assertEqual( + probed_targets()["web-app-nextcloud"], + [f"http://next.cloud.{ONION}/healthz/ready"], + ) + + def test_a_clearnet_app_is_probed_under_its_own_scheme(self): + self.assertEqual( + probed_targets()["web-app-gitea"], + ["https://git.infinito.test/healthz/ready"], + ) + + def test_only_the_roles_the_service_lookup_names_are_probed(self): + self.assertEqual(set(probed_targets()), set(ROLES_WITH_THE_PROMETHEUS_SERVICE)) + + +if __name__ == "__main__": + unittest.main() From 2a1c5e13cef45824c9c03764a9ece999c0104153 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 6 Oct 2026 14:27:05 +0200 Subject: [PATCH 02/13] fix(web-app-prometheus): judge each probe by its own target Since the targets take their scheme from the domain they probe, two switches of the http_ready module no longer fit. Both still followed the TLS state of the Prometheus vhost. fail_if_not_ssl refused an http target, an onion vhost for example, as soon as Prometheus itself was served over TLS. It is false now. The module does not follow redirects, so the scheme of the target URL already decides the transport. insecure_skip_verify switched certificate checks off whenever Prometheus itself had no TLS, also for an https target. Checks now stay on, except in a self-signed deployment whose CA is not mounted into the Prometheus containers, where the exporter has nothing to verify against. The per-role test of the spec parsed every answer of the query API as JSON. An answer that is not OK now counts as "no probe yet", so the poll keeps waiting instead of ending on a parse error. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../files/playwright/playwright.spec.js | 3 +- .../templates/configuration/blackbox.yml.j2 | 4 +- .../test_blackbox_ready_module.py | 62 +++++++++++++++++++ 3 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 tests/unit/python/roles/web-app-prometheus/test_blackbox_ready_module.py diff --git a/roles/web-app-prometheus/files/playwright/playwright.spec.js b/roles/web-app-prometheus/files/playwright/playwright.spec.js index f0c374918b..8ed526e48c 100644 --- a/roles/web-app-prometheus/files/playwright/playwright.spec.js +++ b/roles/web-app-prometheus/files/playwright/playwright.spec.js @@ -284,7 +284,8 @@ test("prometheus scrape: every consumer role reports up=1 and passes its probe", await expect .poll( async () => { - const probes = (await (await page.request.get(probeUrl, { ignoreHTTPSErrors: true })).json())?.data?.result || []; + const response = await page.request.get(probeUrl, { ignoreHTTPSErrors: true }); + const probes = response.ok() ? (await response.json())?.data?.result || [] : []; return prometheusTargetRoles .filter((target) => { const own = probes.filter((entry) => entry?.metric?.app === target.id); diff --git a/roles/web-app-prometheus/templates/configuration/blackbox.yml.j2 b/roles/web-app-prometheus/templates/configuration/blackbox.yml.j2 index b8c78938dc..48147594e0 100644 --- a/roles/web-app-prometheus/templates/configuration/blackbox.yml.j2 +++ b/roles/web-app-prometheus/templates/configuration/blackbox.yml.j2 @@ -24,7 +24,7 @@ modules: method: GET follow_redirects: false fail_if_ssl: false - fail_if_not_ssl: {{ lookup('tls', application_id, 'enabled') | bool | lower }} + fail_if_not_ssl: false tls_config: - insecure_skip_verify: {{ not (lookup('tls', application_id, 'enabled') | bool) }} + insecure_skip_verify: {{ ((TLS_MODE | lower) == 'self_signed' and not lookup('ca_injected', application_id)) | lower }} preferred_ip_protocol: "ip4" diff --git a/tests/unit/python/roles/web-app-prometheus/test_blackbox_ready_module.py b/tests/unit/python/roles/web-app-prometheus/test_blackbox_ready_module.py new file mode 100644 index 0000000000..c49856cbc6 --- /dev/null +++ b/tests/unit/python/roles/web-app-prometheus/test_blackbox_ready_module.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import unittest + +from jinja2 import Environment, FileSystemLoader, StrictUndefined, select_autoescape + +from utils.cache.files import PROJECT_ROOT +from utils.cache.yaml import load_yaml_str + +TEMPLATE_DIR = PROJECT_ROOT / "roles/web-app-prometheus/templates/configuration" + + +def ready_module(*, tls_mode: str, prometheus_tls: bool, ca_injected: bool) -> dict: + def lookup(name, *terms): + if name == "tls" and terms[1] == "enabled": + return prometheus_tls + if name == "ca_injected": + return ca_injected + raise AssertionError(f"the template calls an unexpected lookup: {name!r}") + + env = Environment( + loader=FileSystemLoader(str(TEMPLATE_DIR)), + undefined=StrictUndefined, + autoescape=select_autoescape(), + ) + env.filters["bool"] = bool + rendered = env.get_template("blackbox.yml.j2").render( + lookup=lookup, + application_id="web-app-prometheus", + TLS_MODE=tls_mode, + ) + return load_yaml_str(rendered)["modules"]["http_ready"]["http"] + + +class TestBlackboxReadyModule(unittest.TestCase): + def test_an_http_target_passes_while_prometheus_itself_uses_tls(self): + module = ready_module( + tls_mode="letsencrypt", prometheus_tls=True, ca_injected=False + ) + self.assertIs(module["fail_if_not_ssl"], False) + + def test_certificates_are_verified_while_prometheus_itself_has_no_tls(self): + module = ready_module( + tls_mode="letsencrypt", prometheus_tls=False, ca_injected=False + ) + self.assertIs(module["tls_config"]["insecure_skip_verify"], False) + + def test_certificates_are_verified_once_the_private_ca_is_mounted(self): + module = ready_module( + tls_mode="self_signed", prometheus_tls=True, ca_injected=True + ) + self.assertIs(module["tls_config"]["insecure_skip_verify"], False) + + def test_verification_is_skipped_only_without_the_private_ca(self): + module = ready_module( + tls_mode="self_signed", prometheus_tls=False, ca_injected=False + ) + self.assertIs(module["tls_config"]["insecure_skip_verify"], True) + + +if __name__ == "__main__": + unittest.main() From a6411ebe58b85b200b19438af5130efdbc5ceeb0 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 30 Sep 2026 16:10:13 +0200 Subject: [PATCH 03/13] fix(tor): stop rounds from claiming a surface the onion cannot reach A variant that pins `services.tor.enabled: true` while enabling a role whose own `meta/services.yml` pins `tor.enabled: false` declares a combination the deployment cannot hold. The role rides no node onion, so on that round its surface lives at a clearnet host no onion browser reaches, and the dashboard's `frame-src 'self' http://*.` forbids embedding it outright. Run 36651124458 job 109692287333 shows the result: five specs fail at once, dashboard tile, matomo tracker, css injection, logout and the coturn metric label, all on hosts that are the only non-onion ones in the whole job. web-app-nextcloud variant 11 was such a round: bigbluebutton on, tor on, while bigbluebutton declares no onion. Variant 8 already answers the identical question for PeerTube (65538c40da, "clearnet, because the OIDC plugin rejects a plaintext onion issuer"), so variant 11 now does the same. With that, bigbluebutton never meets a tor node and needs no consumer gates of its own. Two roles keep a gate, for reasons that differ from each other: web-app-seaweedfs is not clearnet-only. It deploys on tor as the object store other roles reach over the internal network, and only its browser frontend is off there, which `frontend.enabled` on main already says. Its logout, dashboard and prometheus declarations claimed a surface that round does not deploy, so they take the same gate the file's own frontend and sso blocks use. web-svc-coturn belongs on a tor node and cannot serve one: STUN/TURN runs on UDP plus a 20000-port relay range that no onion descriptor can publish, and its only HTTP is a gateway answering a plaintext OK. Deploying it is right, advertising it as a prometheus consumer is not, because that scrape goes through the browser path. It is exempt from the rule itself, since it claims no visitable surface, and its `dynamic-flag` markers now carry the reasons the rule's own message asks for; one of the four was a duplicate of the line below it and is gone. The lint states the rule. It found these five pairings on its own, matching a hand count made before it existed, and reports none after the variant fix. `test_sso_variant_requires_tor` covers the complementary half, every variant of a role that has an onion must pin it on, and exempts the clearnet-only role itself; this rule covers the consumers that bind one. The suppression marker belongs on the bound role when the reason is that role's own shape, because a consumer-side marker would also silence every other clearnet role in the file. Co-Authored-By: Claude Opus 5 (1M context) --- roles/web-app-nextcloud/meta/variants.yml | 4 +- roles/web-app-seaweedfs/meta/services.yml | 12 +- roles/web-svc-coturn/meta/services.yml | 12 +- .../test_clearnet_role_forbids_variant_tor.py | 145 ++++++++++++++++++ 4 files changed, 159 insertions(+), 14 deletions(-) create mode 100644 tests/lint/ansible/roles/meta/test_clearnet_role_forbids_variant_tor.py diff --git a/roles/web-app-nextcloud/meta/variants.yml b/roles/web-app-nextcloud/meta/variants.yml index 5dad822253..60a096f8d3 100644 --- a/roles/web-app-nextcloud/meta/variants.yml +++ b/roles/web-app-nextcloud/meta/variants.yml @@ -1097,7 +1097,7 @@ enabled: false shared: false addons: *addons_static_off -# Variant 11: bigbluebutton +# Variant 11: bigbluebutton - clearnet, because bigbluebutton rides no node onion - services: container_backup: enabled: false @@ -1183,7 +1183,7 @@ enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true tor: - enabled: true + enabled: false hermes: enabled: false shared: false diff --git a/roles/web-app-seaweedfs/meta/services.yml b/roles/web-app-seaweedfs/meta/services.yml index 20906c4bb6..b839222f44 100644 --- a/roles/web-app-seaweedfs/meta/services.yml +++ b/roles/web-app-seaweedfs/meta/services.yml @@ -29,16 +29,16 @@ ldap: shared: false logout: bond: 1 - enabled: "{{ 'web-svc-logout' in group_names }}" - shared: "{{ 'web-svc-logout' in group_names }}" + enabled: "{{ ('web-svc-logout' in group_names) and ('svc-net-tor' not in group_names) }}" + shared: "{{ ('web-svc-logout' in group_names) and ('svc-net-tor' not in group_names) }}" dashboard: bond: 1 - enabled: "{{ 'web-app-dashboard' in group_names }}" - shared: "{{ 'web-app-dashboard' in group_names }}" + enabled: "{{ ('web-app-dashboard' in group_names) and ('svc-net-tor' not in group_names) }}" + shared: "{{ ('web-app-dashboard' in group_names) and ('svc-net-tor' not in group_names) }}" prometheus: bond: 1 - enabled: "{{ 'web-app-prometheus' in group_names }}" - shared: "{{ 'web-app-prometheus' in group_names }}" + enabled: "{{ ('web-app-prometheus' in group_names) and ('svc-net-tor' not in group_names) }}" + shared: "{{ ('web-app-prometheus' in group_names) and ('svc-net-tor' not in group_names) }}" # nocheck: playwright-service-flag — central object-store provider entity seaweedfs: healthcheck: diff --git a/roles/web-svc-coturn/meta/services.yml b/roles/web-svc-coturn/meta/services.yml index 46fd83e125..a27ca559cd 100644 --- a/roles/web-svc-coturn/meta/services.yml +++ b/roles/web-svc-coturn/meta/services.yml @@ -36,7 +36,7 @@ gateway: mem_reservation: 256m mem_limit: 512m pids_limit: 256 -# nocheck: dynamic-flag +# nocheck: dynamic-flag coturn runs with user_auth_enabled false, so it keeps no credential store to back; the entry stays literal until auth is switched on and no template references redis today. redis: backup: disabled: true @@ -44,13 +44,13 @@ redis: shared: "{{ 'svc-db-redis' in group_names }}" prometheus: bond: 1 - enabled: "{{ 'web-app-prometheus' in group_names }}" - shared: "{{ 'web-app-prometheus' in group_names }}" + enabled: "{{ ('web-app-prometheus' in group_names) and ('svc-net-tor' not in group_names) }}" + shared: "{{ ('web-app-prometheus' in group_names) and ('svc-net-tor' not in group_names) }}" +# nocheck: clearnet-variant-tor — STUN/TURN on UDP plus a 20000-port relay range, which no onion descriptor can publish; the only HTTP is a gateway returning a plaintext OK. No dashboard, logout, css or matomo block and no Playwright spec, so an onion round claims no visitable surface for it. Its one consumer declaration, prometheus, carries the tor gate. tor: - enabled: false # nocheck: dynamic-flag - shared: false # nocheck: dynamic-flag -# nocheck: dynamic-flag + enabled: false # nocheck: dynamic-flag no group can turn this on: see the clearnet-variant-tor reason above + shared: false # nocheck: dynamic-flag same # nocheck: playwright-service-flag, dynamic-flag infrastructure backup consumer container_backup: # nocheck: bond-baseline backup host joins outside the variant topology bond: 1 diff --git a/tests/lint/ansible/roles/meta/test_clearnet_role_forbids_variant_tor.py b/tests/lint/ansible/roles/meta/test_clearnet_role_forbids_variant_tor.py new file mode 100644 index 0000000000..31004faeb9 --- /dev/null +++ b/tests/lint/ansible/roles/meta/test_clearnet_role_forbids_variant_tor.py @@ -0,0 +1,145 @@ +"""Lint guard: a variant binding a clearnet-only role must not pin the onion on. + +A role whose ``meta/services.yml`` pins ``tor.enabled`` to a literal false rides +no node onion, so on a tor deployment its surface lives at a clearnet host that +no onion browser reaches, and the dashboard's own +``frame-src 'self' http://*.`` forbids embedding it. A variant that +enables such a role and pins ``services.tor.enabled: true`` therefore declares a +combination the deployment cannot hold: the round advertises a tile, a tracker, +a stylesheet and a scrape target for a surface that is not there. + +``test_sso_variant_requires_tor`` states the complementary half - every variant +of a role that has an onion MUST pin it on - and exempts the clearnet-only role +itself. This rule covers the consumers it does not: the variants that bind one. + +The project already applies the rule by hand. ``web-app-nextcloud``'s variants +head carries "variant 8 pairs PeerTube, which is clearnet-only since 8ffb1f567, +so that round cannot pin the onion on". + +Suppression (see ``docs/contributing/actions/testing/suppression.md``): + +* ``# nocheck: clearnet-variant-tor`` in the CLEARNET role's + ``meta/services.yml``, which exempts it everywhere: a relay with no browser + surface of its own carries no tile and no injected asset, so an onion round + may deploy it without claiming anything about it. +* the same marker in a consumer's ``meta/variants.yml`` for a case that is + sound only there. + +The marker belongs on the bound role rather than the consumer whenever the +reason is the role's own shape, because a consumer-side marker would also +silence the pairings of every other clearnet role in the same file. +""" + +from __future__ import annotations + +import unittest +from typing import TYPE_CHECKING, Any + +import yaml + +from utils.annotations.suppress import line_has_rule +from utils.cache.files import read_text +from utils.cache.yaml import load_yaml_str +from utils.roles.entity.name import get_entity_name +from utils.roles.mapping import ROLE_FILE_META_SERVICES, ROLE_FILE_META_VARIANTS + +from . import PROJECT_ROOT + +if TYPE_CHECKING: + from pathlib import Path + +ROLES_DIR = PROJECT_ROOT / "roles" +_RULE = "clearnet-variant-tor" + + +def _load_yaml(path: Path) -> Any: + if not path.is_file(): + return None + try: + text = read_text(str(path)) + except UnicodeDecodeError: + return None + if not text.strip(): + return None + try: + return load_yaml_str(text) + except yaml.YAMLError: + return None + + +def _suppressed(paths: tuple[Path, ...]) -> bool: + for path in paths: + if not path.is_file(): + continue + if any( + line_has_rule(line, _RULE) for line in read_text(str(path)).splitlines() + ): + return True + return False + + +def _flag(variant: Any, service: str) -> Any: + services = variant.get("services") if isinstance(variant, dict) else None + entry = services.get(service) if isinstance(services, dict) else None + return entry.get("enabled") if isinstance(entry, dict) else None + + +def clearnet_only_keys() -> dict[str, str]: + """Service key to role id, for every clearnet-only role the rule covers. + + Returns: + The key a consumer's variant would bind it under, mapped to the role + that declares itself clearnet-only. A role carrying the marker in its + own ``meta/services.yml`` is left out, so it is exempt everywhere. + """ + found: dict[str, str] = {} + for role_dir in sorted(p for p in ROLES_DIR.iterdir() if p.is_dir()): + services_path = role_dir / ROLE_FILE_META_SERVICES + services = _load_yaml(services_path) + tor = services.get("tor") if isinstance(services, dict) else None + if not isinstance(tor, dict) or tor.get("enabled") is not False: + continue + if _suppressed((services_path,)): + continue + key = get_entity_name(role_dir.name) + if key: + found[key] = role_dir.name + return found + + +class TestClearnetRoleForbidsVariantTor(unittest.TestCase): + def test_no_variant_pins_the_onion_while_binding_a_clearnet_role(self) -> None: + clearnet = clearnet_only_keys() + offenders: list[str] = [] + + for role_dir in sorted(p for p in ROLES_DIR.iterdir() if p.is_dir()): + variants_path = role_dir / ROLE_FILE_META_VARIANTS + variants = _load_yaml(variants_path) + if not isinstance(variants, list) or not variants: + continue + if _suppressed((variants_path,)): + continue + + for index, variant in enumerate(variants): + if _flag(variant, "tor") is not True: + continue + for key, bound_role in sorted(clearnet.items()): + if _flag(variant, key) is True: + offenders.append( + f"{role_dir.name} variant {index}: pins " + f"services.tor.enabled true while enabling {key!r} " + f"({bound_role}), which pins tor.enabled false" + ) + + self.assertEqual( + offenders, + [], + "A variant may not pin the onion on while binding a role that rides " + "none; that round advertises a surface no onion browser can reach. " + f"Turn tor off for the variant, or suppress with '# nocheck: {_RULE}' " + "naming why the pairing is sound:\n " + "\n ".join(offenders), + ) + + +if __name__ == "__main__": + unittest.main() From 50b9f60c43c2fad92fd7bfee0c132785fbeb3436 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 30 Sep 2026 12:22:28 +0200 Subject: [PATCH 04/13] fix(prometheus): resolve the native-metrics scrape target per deployment mode `native_metrics_target` was a Jinja filter, so it could not read DEPLOYMENT_MODE: filters see no Ansible variables. It returned `.name>:` unconditionally, which is `matrix-synapse:9000` for matrix. Compose serves that name; swarm does not, because `docker stack deploy` ignores `container_name` and the alias never exists. This repository says as much itself, through `lookup('compose_only', 'container_name', ...)`. The target was therefore scraped and refused, and `playwright.spec.js:274` reported `web-app-matrix: 1 matching target(s) reporting up != 1`, meaning one matching target whose `up` value is not 1. It becomes a lookup that delegates host resolution to `scrape_target`, which was already mode-aware, so exactly one plugin knows the difference between the two modes. The port has to be handed over explicitly: matrix exposes metrics on 9000 while `services.synapse.ports.internal.http` is 8008, so the existing internal-port lookup cannot serve this case on its own. `scrape_target` now refuses `port` together with an explicit `port_kind`. Before, `port` won silently and the error text still named the `ports.internal.` path that the call had never consulted. `native_metrics_apps` drops an app that pins itself to compose through `compose_mode_force` while the cluster runs swarm, because the scrape target would then name the wrong object. Only the literal case: matrix declares that override as a Jinja expression over its flavour, which this lookup cannot resolve from the scraping role's variable scope, so variant 0 on swarm stays a latent mismatch. The two comment lines claiming `container_name:port` went with the change; after it they state the opposite of what the code does. Verified by mutation: with the swarm branch of `scrape_target` disabled, the new test emits `matrix-synapse:9000`, the exact string CI reported. Co-Authored-By: Claude Opus 5 (1M context) --- plugins/filter/native_metrics_target.py | 51 -------- plugins/lookup/native_metrics_apps.py | 32 ++++- plugins/lookup/native_metrics_target.py | 68 ++++++++++ plugins/lookup/scrape_target.py | 17 ++- .../web-app-gitea/templates/prometheus.yml.j2 | 2 +- .../templates/prometheus.yml.j2 | 4 +- .../templates/prometheus.yml.j2 | 4 +- .../templates/prometheus.yml.j2 | 2 +- .../filter/test_native_metrics_target.py | 111 ---------------- .../lookup/test_native_metrics_target.py | 119 ++++++++++++++++++ 10 files changed, 234 insertions(+), 176 deletions(-) delete mode 100644 plugins/filter/native_metrics_target.py create mode 100644 plugins/lookup/native_metrics_target.py delete mode 100644 tests/unit/python/plugins/filter/test_native_metrics_target.py create mode 100644 tests/unit/python/plugins/lookup/test_native_metrics_target.py diff --git a/plugins/filter/native_metrics_target.py b/plugins/filter/native_metrics_target.py deleted file mode 100644 index cade81dc24..0000000000 --- a/plugins/filter/native_metrics_target.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import annotations - -from ansible.errors import AnsibleFilterError - -from utils.roles.entity.name import get_entity_name - - -def native_metrics_target(app_id: str, applications: dict) -> str: - """Return the container-to-container Prometheus scrape target for a native-metrics app. - - Prometheus connects to the app's Docker network (declared in prometheus's - compose.yml.j2 via the native_metrics_apps loop) and scrapes by container - name and internal port — no host port binding required, no nginx/OAuth2 in - the path. - - Usage in a per-app prometheus.yml.j2 fragment: - targets: ["{{ native_prometheus_application_id | native_metrics_target(lookup('applications')) }}"] - - When the service key differs from the app's entity name (e.g. matrix - uses service key "synapse"), set services.prometheus.native_metrics.service_key - in the app's meta/services.yml to override. - """ - app_conf = applications.get(app_id, {}) - services = app_conf.get("services", {}) if isinstance(app_conf, dict) else {} - native_metrics_conf = ( - services.get("prometheus", {}) if isinstance(services, dict) else {} - ).get("native_metrics", {}) - service_key = native_metrics_conf.get("service_key") or get_entity_name(app_id) - - container = ( - services.get(service_key, {}).get("name") - if isinstance(services, dict) - else None - ) - if not container: - raise AnsibleFilterError( - f"native_metrics_target: no services.{service_key}.name for '{app_id}'" - ) - - port = native_metrics_conf.get("port") - if port is None: - raise AnsibleFilterError( - f"native_metrics_target: no services.prometheus.native_metrics.port for '{app_id}'" - ) - - return f"{container}:{port}" - - -class FilterModule: - def filters(self): - return {"native_metrics_target": native_metrics_target} diff --git a/plugins/lookup/native_metrics_apps.py b/plugins/lookup/native_metrics_apps.py index 6a186041d3..68bc93548f 100644 --- a/plugins/lookup/native_metrics_apps.py +++ b/plugins/lookup/native_metrics_apps.py @@ -1,12 +1,35 @@ from __future__ import annotations +from collections.abc import Mapping from typing import Any from ansible.plugins.loader import lookup_loader from ansible.plugins.lookup import LookupBase from utils.cache import ROLES_DIR +from utils.cache.yaml import load_yaml_any from utils.roles.applications.config import get as get_app_conf +from utils.roles.mapping import ROLE_FILE_VARS_MAIN + + +def compose_mode_force(app_id: str) -> str: + """Return the role's own deploy-mode override. + + Args: + app_id: the role directory name. + + Returns: + The literal ``compose_mode_force`` from the role's vars, or ``""`` when + it is absent or a Jinja expression, which cannot be resolved from the + scraping role's variable scope and so leaves the override invisible here. + """ + path = ROLES_DIR / app_id / ROLE_FILE_VARS_MAIN + if not path.is_file(): + return "" + data = load_yaml_any(str(path), default_if_missing={}) + raw = data.get("compose_mode_force") if isinstance(data, Mapping) else None + value = "" if raw is None else str(raw).strip() + return "" if "{{" in value else value class LookupModule(LookupBase): @@ -16,8 +39,10 @@ class LookupModule(LookupBase): 2. a prometheus.yml.j2 template at roles//templates/ 3. reachable from this prometheus: NOT on a node-local force_bridge network under swarm, which a swarm prometheus can neither join nor - resolve. This is the single point that gates force_bridge apps out of - the native-metrics precreate + scrape. + resolve, and NOT pinned to compose by its own compose_mode_force while + the cluster runs swarm, which would make the scrape target name the + wrong object. This is the single point that gates unreachable apps out + of the native-metrics precreate + scrape. Used by web-app-prometheus/templates/configuration/prometheus.yml.j2 to auto-discover apps that expose a native /metrics endpoint without hardcoding each app name. @@ -78,6 +103,9 @@ class LookupModule(LookupBase): if is_swarm and bool(force_bridge): continue + if is_swarm and compose_mode_force(app_id) == "compose": + continue + result.append(app_id) return [result] diff --git a/plugins/lookup/native_metrics_target.py b/plugins/lookup/native_metrics_target.py new file mode 100644 index 0000000000..93140ee56e --- /dev/null +++ b/plugins/lookup/native_metrics_target.py @@ -0,0 +1,68 @@ +"""Lookup ``native_metrics_target``: scrape target for an app that exposes its +own metrics port. Reads ``services.prometheus.native_metrics`` for the service +key and port, then delegates host resolution to ``scrape_target`` so swarm and +compose stay on one implementation. + +Usage in a per-app prometheus.yml.j2 fragment: + targets: ["{{ lookup('native_metrics_target', native_prometheus_application_id) }}"] +""" + +from __future__ import annotations + +from typing import Any + +from ansible.errors import AnsibleError +from ansible.plugins.loader import lookup_loader +from ansible.plugins.lookup import LookupBase + +from utils.roles.applications.config import get +from utils.roles.entity.name import get_entity_name + + +class LookupModule(LookupBase): + def run( + self, + terms: list[Any] | None, + variables: dict[str, Any] | None = None, + **kwargs: Any, + ) -> list[str]: + if not terms: + raise AnsibleError( + "native_metrics_target lookup requires the application_id term" + ) + + application_id = str(terms[0]).strip() + if not application_id: + raise AnsibleError("native_metrics_target: application_id must be non-empty") + + vars_ = variables or getattr(self._templar, "available_variables", {}) or {} + applications = lookup_loader.get( + "applications", loader=self._loader, templar=getattr(self, "_templar", None) + ).run([], variables=vars_)[0] + + def conf(key: str) -> str: + value = get( + applications=applications, + application_id=application_id, + config_path=f"services.prometheus.native_metrics.{key}", + strict=False, + default="", + ) + return "" if value is None else str(value).strip() + + port = conf("port") + if not port: + raise AnsibleError( + "native_metrics_target: no " + f"services.prometheus.native_metrics.port for {application_id!r}" + ) + + service_key = conf("service_key") or get_entity_name(application_id) + if not service_key: + raise AnsibleError( + f"native_metrics_target: cannot derive service key for {application_id!r}" + ) + + return lookup_loader.get( + "scrape_target", loader=self._loader, templar=getattr(self, "_templar", None) + ).run([application_id, service_key], variables=vars_, port=port) diff --git a/plugins/lookup/scrape_target.py b/plugins/lookup/scrape_target.py index d99674e8d7..06190f50be 100644 --- a/plugins/lookup/scrape_target.py +++ b/plugins/lookup/scrape_target.py @@ -1,6 +1,7 @@ """Lookup ``scrape_target``: mode-aware ``:`` for a Prometheus scrape target. Swarm resolves ``tasks._`` (per-task DNS); -compose uses ``compose_host`` or the service container name.""" +compose uses ``compose_host`` or the service container name. ``port`` overrides +the ``services..ports.internal.`` lookup.""" from __future__ import annotations @@ -37,12 +38,20 @@ class LookupModule(LookupBase): if not service_key: raise AnsibleError("scrape_target: service_key must be non-empty") - port_kind = ( + explicit_port_kind = ( _as_str(terms[2]) if len(terms) > 2 else _as_str(kwargs.get("port_kind")) - ) or "http" + ) + port_kind = explicit_port_kind or "http" compose_host = ( _as_str(terms[3]) if len(terms) > 3 else _as_str(kwargs.get("compose_host")) ) + port_override = _as_str(kwargs.get("port")) + if port_override and explicit_port_kind: + raise AnsibleError( + f"scrape_target: {application_id!r} passed both port={port_override!r} " + f"and port_kind={explicit_port_kind!r}; port wins and port_kind would be " + "silently dropped, so pass only one" + ) vars_ = variables or getattr(self._templar, "available_variables", {}) or {} templar = getattr(self, "_templar", None) @@ -57,7 +66,7 @@ class LookupModule(LookupBase): "applications", loader=self._loader, templar=getattr(self, "_templar", None) ).run([], variables=vars_)[0] - port = _as_str( + port = port_override or _as_str( get( applications=applications, application_id=application_id, diff --git a/roles/web-app-gitea/templates/prometheus.yml.j2 b/roles/web-app-gitea/templates/prometheus.yml.j2 index 3583badebf..ba2fc8151c 100644 --- a/roles/web-app-gitea/templates/prometheus.yml.j2 +++ b/roles/web-app-gitea/templates/prometheus.yml.j2 @@ -9,6 +9,6 @@ bearer_token: "{{ _token }}" {% endif %} static_configs: - - targets: ["{{ native_prometheus_application_id | native_metrics_target(lookup('applications')) }}"] + - targets: ["{{ lookup('native_metrics_target', native_prometheus_application_id) }}"] labels: app: "{{ native_prometheus_application_id }}" diff --git a/roles/web-app-matrix/templates/prometheus.yml.j2 b/roles/web-app-matrix/templates/prometheus.yml.j2 index 79e65ead8f..ac5712173f 100644 --- a/roles/web-app-matrix/templates/prometheus.yml.j2 +++ b/roles/web-app-matrix/templates/prometheus.yml.j2 @@ -1,12 +1,10 @@ # Matrix Synapse — server and federation metrics. # Requires enable_metrics: true in homeserver.yaml. # Enable via native_metrics.enabled: true in inventory. - # Scraped container-to-container: prometheus joins the matrix Docker network and - # reaches synapse directly at container_name:port — no host binding, no nginx/OAuth2. - job_name: "matrix-synapse" scheme: http metrics_path: /_synapse/metrics static_configs: - - targets: ["{{ native_prometheus_application_id | native_metrics_target(lookup('applications')) }}"] + - targets: ["{{ lookup('native_metrics_target', native_prometheus_application_id) }}"] labels: app: "{{ native_prometheus_application_id }}" diff --git a/roles/web-app-mattermost/templates/prometheus.yml.j2 b/roles/web-app-mattermost/templates/prometheus.yml.j2 index a3b4a71621..67a7e7ae66 100644 --- a/roles/web-app-mattermost/templates/prometheus.yml.j2 +++ b/roles/web-app-mattermost/templates/prometheus.yml.j2 @@ -1,11 +1,9 @@ # Mattermost — application and Go runtime metrics. - # Scraped container-to-container: prometheus joins the mattermost Docker network and - # reaches the app directly at container_name:port — no host binding, no nginx/OAuth2. # Requires MM_METRICSSETTINGS_ENABLE=true and MM_METRICSSETTINGS_LISTENADDRESS (set in env.j2 when enabled). - job_name: "mattermost" scheme: http metrics_path: /metrics static_configs: - - targets: ["{{ native_prometheus_application_id | native_metrics_target(lookup('applications')) }}"] + - targets: ["{{ lookup('native_metrics_target', native_prometheus_application_id) }}"] labels: app: "{{ native_prometheus_application_id }}" diff --git a/roles/web-app-openbao/templates/prometheus.yml.j2 b/roles/web-app-openbao/templates/prometheus.yml.j2 index 3dd99470c9..9eedd90c6f 100644 --- a/roles/web-app-openbao/templates/prometheus.yml.j2 +++ b/roles/web-app-openbao/templates/prometheus.yml.j2 @@ -4,6 +4,6 @@ params: format: ["prometheus"] static_configs: - - targets: ["{{ native_prometheus_application_id | native_metrics_target(lookup('applications')) }}"] + - targets: ["{{ lookup('native_metrics_target', native_prometheus_application_id) }}"] labels: app: "{{ native_prometheus_application_id }}" diff --git a/tests/unit/python/plugins/filter/test_native_metrics_target.py b/tests/unit/python/plugins/filter/test_native_metrics_target.py deleted file mode 100644 index e2d773c2e5..0000000000 --- a/tests/unit/python/plugins/filter/test_native_metrics_target.py +++ /dev/null @@ -1,111 +0,0 @@ -from __future__ import annotations - -import unittest -from unittest.mock import patch - -from ansible.errors import AnsibleFilterError - -from plugins.filter.native_metrics_target import native_metrics_target - - -def _make_applications( - app_id: str, - container: str, - port: int, - entity_name: str, - service_key: str | None = None, -) -> dict: - native_metrics: dict = {"port": port} - if service_key is not None: - native_metrics["service_key"] = service_key - return { - app_id: { - "services": { - entity_name: {"name": container}, - "prometheus": {"native_metrics": native_metrics}, - } - } - } - - -class TestNativeMetricsTargetSuccess(unittest.TestCase): - def test_returns_container_colon_port(self): - apps = _make_applications("web-app-gitea", "gitea", 3000, "gitea") - with patch( - "plugins.filter.native_metrics_target.get_entity_name", return_value="gitea" - ): - result = native_metrics_target("web-app-gitea", apps) - self.assertEqual(result, "gitea:3000") - - def test_different_app(self): - apps = _make_applications( - "web-app-mattermost", "mattermost", 8067, "mattermost" - ) - with patch( - "plugins.filter.native_metrics_target.get_entity_name", - return_value="mattermost", - ): - result = native_metrics_target("web-app-mattermost", apps) - self.assertEqual(result, "mattermost:8067") - - def test_service_key_override(self): - apps = _make_applications( - "web-app-matrix", "matrix-synapse", 9000, "synapse", service_key="synapse" - ) - with patch( - "plugins.filter.native_metrics_target.get_entity_name", - return_value="matrix", - ): - result = native_metrics_target("web-app-matrix", apps) - self.assertEqual(result, "matrix-synapse:9000") - - -class TestNativeMetricsTargetErrors(unittest.TestCase): - def test_raises_when_container_name_missing(self): - apps = { - "web-app-gitea": { - "services": { - "prometheus": {"native_metrics": {"port": 3000}}, - } - } - } - with ( - patch( - "plugins.filter.native_metrics_target.get_entity_name", - return_value="gitea", - ), - self.assertRaises(AnsibleFilterError), - ): - native_metrics_target("web-app-gitea", apps) - - def test_raises_when_port_missing(self): - apps = { - "web-app-gitea": { - "services": { - "gitea": {"name": "gitea"}, - "prometheus": {"native_metrics": {}}, - } - } - } - with ( - patch( - "plugins.filter.native_metrics_target.get_entity_name", - return_value="gitea", - ), - self.assertRaises(AnsibleFilterError), - ): - native_metrics_target("web-app-gitea", apps) - - def test_raises_when_app_not_in_applications(self): - with ( - patch( - "plugins.filter.native_metrics_target.get_entity_name", - return_value="gitea", - ), - self.assertRaises(AnsibleFilterError), - ): - native_metrics_target("web-app-gitea", {}) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/python/plugins/lookup/test_native_metrics_target.py b/tests/unit/python/plugins/lookup/test_native_metrics_target.py new file mode 100644 index 0000000000..e3abbdb9dd --- /dev/null +++ b/tests/unit/python/plugins/lookup/test_native_metrics_target.py @@ -0,0 +1,119 @@ +"""Tests for the native_metrics_target lookup. + +The lookup delegates host resolution to scrape_target, so these run the real +scrape_target rather than a stub: the regression being guarded is that swarm +must not receive a compose container name, and only the delegation proves it. +""" + +from __future__ import annotations + +import unittest +from typing import Any +from unittest.mock import MagicMock, patch + +from ansible.errors import AnsibleError + +from plugins.lookup.native_metrics_target import LookupModule +from plugins.lookup.scrape_target import LookupModule as ScrapeTargetLookup + +ENTITIES = {"web-app-matrix": "matrix", "web-app-gitea": "gitea"} + + +class _Templar: + def __init__(self, vars_=None): + self.available_variables = vars_ or {} + + def template(self, value, **_): + return value + + +def _apps(native_metrics: dict[str, Any]) -> dict[str, Any]: + return { + "web-app-matrix": { + "services": { + "prometheus": {"native_metrics": native_metrics}, + "synapse": { + "name": "matrix-synapse", + "ports": {"internal": {"http": 8008}}, + }, + }, + }, + } + + +def _run(terms, *, mode, apps): + variables = {"DEPLOYMENT_MODE": mode} + lookup = LookupModule() + lookup._templar = _Templar(variables) + lookup._loader = MagicMock() + + def _dispatch(name, **_kwargs): + if name == "applications": + return MagicMock(run=lambda *_a, **_k: [apps]) + scrape = ScrapeTargetLookup() + scrape._templar = _Templar(variables) + scrape._loader = MagicMock() + return scrape + + with ( + patch( + "plugins.lookup.native_metrics_target.lookup_loader" + ) as native_loader_mock, + patch("plugins.lookup.scrape_target.lookup_loader") as scrape_loader_mock, + patch( + "plugins.lookup.native_metrics_target.get_entity_name", + side_effect=lambda app_id: ENTITIES.get(app_id, ""), + ), + patch( + "plugins.lookup.scrape_target.get_entity_name", + side_effect=lambda app_id: ENTITIES.get(app_id, ""), + ), + ): + native_loader_mock.get.side_effect = _dispatch + scrape_loader_mock.get.return_value = MagicMock(run=lambda *_a, **_k: [apps]) + return lookup.run(terms, variables=variables) + + +class TestNativeMetricsTargetLookup(unittest.TestCase): + def test_swarm_uses_per_task_dns_not_the_container_name(self): + apps = _apps({"port": 9000, "service_key": "synapse"}) + result = _run(["web-app-matrix"], mode="swarm", apps=apps) + self.assertEqual(result, ["tasks.matrix_synapse:9000"]) + + def test_compose_uses_the_container_name(self): + apps = _apps({"port": 9000, "service_key": "synapse"}) + result = _run(["web-app-matrix"], mode="compose", apps=apps) + self.assertEqual(result, ["matrix-synapse:9000"]) + + def test_service_key_defaults_to_the_entity_name(self): + apps = { + "web-app-gitea": { + "services": { + "prometheus": {"native_metrics": {"port": 3000}}, + "gitea": { + "name": "gitea", + "ports": {"internal": {"http": 3000}}, + }, + }, + }, + } + result = _run(["web-app-gitea"], mode="swarm", apps=apps) + self.assertEqual(result, ["tasks.gitea_gitea:3000"]) + + def test_native_metrics_port_takes_precedence_over_the_internal_port(self): + apps = _apps({"port": 9000, "service_key": "synapse"}) + result = _run(["web-app-matrix"], mode="compose", apps=apps) + self.assertNotIn("8008", result[0]) + + def test_missing_port_raises(self): + apps = _apps({"service_key": "synapse"}) + with self.assertRaises(AnsibleError): + _run(["web-app-matrix"], mode="compose", apps=apps) + + def test_missing_application_id_raises(self): + with self.assertRaises(AnsibleError): + _run([], mode="compose", apps=_apps({"port": 9000})) + + +if __name__ == "__main__": + unittest.main() From f77ccf56e88459740ac965c82decd7caab81669f Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 8 Oct 2026 08:54:03 +0200 Subject: [PATCH 05/13] fix(web-app-matrix): publish Synapse metrics in the ansible flavor The role advertised `matrix-synapse:9000` to Prometheus in both flavors. That container only exists in the compose flavor. In the ansible flavor, which is the default, Synapse runs inside the `matrix` container and had no metrics listener, so the scrape target was down on every deployment that pairs Matrix with Prometheus. The ansible flavor now enables the Synapse metrics listener on 9100 and the declaration names port and service per flavor, so the target is `matrix:9100` there and stays `matrix-synapse:9000` in the compose flavor. This is the metrics part of b3e0dce578 on `feature/store-i18n`, without the addon switch that branch introduces around it. Co-Authored-By: Claude Opus 5.5 (1M context) --- roles/web-app-matrix/meta/services.yml | 4 ++-- roles/web-app-matrix/templates/flavor/ansible/vars.yml.j2 | 4 ++++ roles/web-app-matrix/vars/main.yml | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/roles/web-app-matrix/meta/services.yml b/roles/web-app-matrix/meta/services.yml index 1fcc023f32..5a59dbcf13 100644 --- a/roles/web-app-matrix/meta/services.yml +++ b/roles/web-app-matrix/meta/services.yml @@ -104,8 +104,8 @@ prometheus: shared: "{{ 'web-app-prometheus' in group_names }}" native_metrics: enabled: true - port: 9000 - service_key: synapse + port: "{{ 9000 if lookup('config', 'web-app-matrix', 'services.matrix.flavor') == 'compose' else 9100 }}" + service_key: "{{ 'synapse' if lookup('config', 'web-app-matrix', 'services.matrix.flavor') == 'compose' else 'matrix' }}" communication: channel: true # nocheck: playwright-service-flag: shared provider entity; the app itself is covered by this role's own Playwright spec diff --git a/roles/web-app-matrix/templates/flavor/ansible/vars.yml.j2 b/roles/web-app-matrix/templates/flavor/ansible/vars.yml.j2 index 8185cfdfd2..9db5433cae 100644 --- a/roles/web-app-matrix/templates/flavor/ansible/vars.yml.j2 +++ b/roles/web-app-matrix/templates/flavor/ansible/vars.yml.j2 @@ -153,5 +153,9 @@ matrix_bridge_mautrix_twitter_enabled: {{ MATRIX_BRIDGE.mautrix_twitter | string matrix_bridge_mautrix_whatsapp_enabled: {{ MATRIX_BRIDGE.mautrix_whatsapp | string | lower }} matrix_bridge_mautrix_gitter_enabled: {{ MATRIX_BRIDGE.gitter | string | lower }} +matrix_synapse_metrics_enabled: true +matrix_synapse_container_metrics_api_host_bind_port: "0.0.0.0:{{ MATRIX_SYNAPSE_METRICS_PORT }}" +matrix_synapse_metrics_port: {{ MATRIX_SYNAPSE_METRICS_PORT }} + jitsi_enabled: {{ MATRIX_CONFERENCING_JITSI | string | lower }} matrix_rtc_enabled: {{ MATRIX_CONFERENCING_ELEMENT_CALL | string | lower }} diff --git a/roles/web-app-matrix/vars/main.yml b/roles/web-app-matrix/vars/main.yml index deea5c3e51..96f3e2daef 100644 --- a/roles/web-app-matrix/vars/main.yml +++ b/roles/web-app-matrix/vars/main.yml @@ -82,6 +82,7 @@ MATRIX_BRIDGE: "{{ lookup('config', application_id, 'services.matrix.plugins') } MATRIX_CONFERENCING_JITSI: "{{ lookup('config', application_id, 'services.matrix.conferencing.jitsi') | bool }}" MATRIX_CONFERENCING_ELEMENT_CALL: "{{ lookup('config', application_id, 'services.matrix.conferencing.element_call') | bool }}" +MATRIX_SYNAPSE_METRICS_PORT: "{{ lookup('config', application_id, 'services.prometheus.native_metrics.port') }}" MATRIX_LDAP_ENABLED: "{{ lookup('config', application_id, 'services.ldap.enabled') | bool }}" From 31d391a008f848216373d61a64ebf6202869ceba Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 8 Oct 2026 08:55:51 +0200 Subject: [PATCH 06/13] style: settle the formatter rewrite of the native metrics lookup Co-Authored-By: Claude Opus 5.5 (1M context) --- plugins/lookup/native_metrics_target.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/lookup/native_metrics_target.py b/plugins/lookup/native_metrics_target.py index 93140ee56e..201f517f32 100644 --- a/plugins/lookup/native_metrics_target.py +++ b/plugins/lookup/native_metrics_target.py @@ -33,7 +33,9 @@ class LookupModule(LookupBase): application_id = str(terms[0]).strip() if not application_id: - raise AnsibleError("native_metrics_target: application_id must be non-empty") + raise AnsibleError( + "native_metrics_target: application_id must be non-empty" + ) vars_ = variables or getattr(self._templar, "available_variables", {}) or {} applications = lookup_loader.get( @@ -64,5 +66,7 @@ class LookupModule(LookupBase): ) return lookup_loader.get( - "scrape_target", loader=self._loader, templar=getattr(self, "_templar", None) + "scrape_target", + loader=self._loader, + templar=getattr(self, "_templar", None), ).run([application_id, service_key], variables=vars_, port=port) From 428b619ceb762c158306a018f10980a778ba8a8d Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 8 Oct 2026 08:56:00 +0200 Subject: [PATCH 07/13] fix(tor): gate the consumer pins of seaweedfs and coturn in their variants The tor gate that 312d09e290 put into the services.yml of web-app-seaweedfs and web-svc-coturn never took effect on a deployed round. Both roles pin the same flags to a literal true in variant 0, the round writes those pins into the inventory, and an inventory value wins over the expression in services.yml. Measured on a compose round of web-app-matrix variant 0 with tor: the inventory carried prometheus enabled true for both roles, so both stayed in the Prometheus target list and the spec failed on the missing web-app-seaweedfs label and probe. Variant 0 now carries the gate itself for prometheus on both roles and for logout and dashboard on seaweedfs, the three flags 312d09e290 names. The bond-baseline and variants-coverage guards require a literal true in variant 0 for these blocks, so they carry the same markers the frontend block of seaweedfs already has, for the same reason. Co-Authored-By: Claude Opus 5.5 (1M context) --- roles/web-app-seaweedfs/meta/services.yml | 6 ++++++ roles/web-app-seaweedfs/meta/variants.yml | 12 ++++++------ roles/web-svc-coturn/meta/services.yml | 2 ++ roles/web-svc-coturn/meta/variants.yml | 4 ++-- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/roles/web-app-seaweedfs/meta/services.yml b/roles/web-app-seaweedfs/meta/services.yml index b839222f44..2ba8b58900 100644 --- a/roles/web-app-seaweedfs/meta/services.yml +++ b/roles/web-app-seaweedfs/meta/services.yml @@ -27,14 +27,20 @@ sso: ldap: enabled: false shared: false +# nocheck: variants-coverage — the onion gate is a hard invariant; no variant may force this consumer on a tor node. +# nocheck: bond-baseline — same reason: variant 0 must not pin this consumer on. logout: bond: 1 enabled: "{{ ('web-svc-logout' in group_names) and ('svc-net-tor' not in group_names) }}" shared: "{{ ('web-svc-logout' in group_names) and ('svc-net-tor' not in group_names) }}" +# nocheck: variants-coverage — the onion gate is a hard invariant; no variant may force this consumer on a tor node. +# nocheck: bond-baseline — same reason: variant 0 must not pin this consumer on. dashboard: bond: 1 enabled: "{{ ('web-app-dashboard' in group_names) and ('svc-net-tor' not in group_names) }}" shared: "{{ ('web-app-dashboard' in group_names) and ('svc-net-tor' not in group_names) }}" +# nocheck: variants-coverage — the onion gate is a hard invariant; no variant may force this consumer on a tor node. +# nocheck: bond-baseline — same reason: variant 0 must not pin this consumer on. prometheus: bond: 1 enabled: "{{ ('web-app-prometheus' in group_names) and ('svc-net-tor' not in group_names) }}" diff --git a/roles/web-app-seaweedfs/meta/variants.yml b/roles/web-app-seaweedfs/meta/variants.yml index 77e2c251f1..678afaac86 100644 --- a/roles/web-app-seaweedfs/meta/variants.yml +++ b/roles/web-app-seaweedfs/meta/variants.yml @@ -9,14 +9,14 @@ enabled: true shared: true logout: - enabled: true - shared: true + enabled: "{{ 'svc-net-tor' not in group_names }}" + shared: "{{ 'svc-net-tor' not in group_names }}" dashboard: - enabled: true - shared: true + enabled: "{{ 'svc-net-tor' not in group_names }}" + shared: "{{ 'svc-net-tor' not in group_names }}" prometheus: - enabled: true - shared: true + enabled: "{{ 'svc-net-tor' not in group_names }}" + shared: "{{ 'svc-net-tor' not in group_names }}" - services: container_backup: enabled: false diff --git a/roles/web-svc-coturn/meta/services.yml b/roles/web-svc-coturn/meta/services.yml index a27ca559cd..26f1a9818f 100644 --- a/roles/web-svc-coturn/meta/services.yml +++ b/roles/web-svc-coturn/meta/services.yml @@ -42,6 +42,8 @@ redis: disabled: true enabled: false shared: "{{ 'svc-db-redis' in group_names }}" +# nocheck: variants-coverage — the tor conjunct is the clearnet-variant-tor invariant below, so no variant may pin the true polarity on an onion node +# nocheck: bond-baseline — same reason: variant 0 must not pin the scrape consumer on prometheus: bond: 1 enabled: "{{ ('web-app-prometheus' in group_names) and ('svc-net-tor' not in group_names) }}" diff --git a/roles/web-svc-coturn/meta/variants.yml b/roles/web-svc-coturn/meta/variants.yml index c3cf254fc1..1297a4474f 100644 --- a/roles/web-svc-coturn/meta/variants.yml +++ b/roles/web-svc-coturn/meta/variants.yml @@ -9,8 +9,8 @@ container_backup: enabled: true prometheus: - enabled: true - shared: true + enabled: "{{ 'svc-net-tor' not in group_names }}" + shared: "{{ 'svc-net-tor' not in group_names }}" redis: shared: true tor: From 7736755bc877fb6302401031c532c05ecb7b1ca0 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:51:19 +0000 Subject: [PATCH 08/13] update(images): update Docker image versions (web-app-pixelfed) --- roles/web-app-pixelfed/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-pixelfed/meta/services.yml b/roles/web-app-pixelfed/meta/services.yml index 72dc71d410..10fa557e0a 100644 --- a/roles/web-app-pixelfed/meta/services.yml +++ b/roles/web-app-pixelfed/meta/services.yml @@ -49,7 +49,7 @@ pixelfed: shared: true enabled: true # nocheck: dynamic-flag image: zknt/pixelfed - version: "20261007" + version: "20261008" name: pixelfed min_storage: 588MB backup: From 5baccf49cdcc3fcd049e3643caea83d1d1885c42 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 8 Oct 2026 17:08:35 +0200 Subject: [PATCH 09/13] fix(dev-compose): build the stack image without host networking Buildx 0.38.0 gates host networking in bake behind an entitlement that Compose never grants, so `make compose-up` failed wherever Compose had to build the image itself. The build block asked for it without need: scripts/image/build.sh builds the same Dockerfile on the bridge network. Closes #875 Co-Authored-By: Claude Opus 5.5 (1M context) --- compose.yml | 1 - ...est_compose_build_needs_no_host_network.py | 60 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 tests/lint/repository/test_compose_build_needs_no_host_network.py diff --git a/compose.yml b/compose.yml index 5cc60f4415..3bb9f3e536 100644 --- a/compose.yml +++ b/compose.yml @@ -23,7 +23,6 @@ services: build: context: . dockerfile: Dockerfile - network: host args: INFINITO_PARENT_IMAGE: ${INFINITO_PARENT_IMAGE:?Run 'make dotenv' to generate the .env single source of truth} INFINITO_SRC_DIR: ${INFINITO_SRC_DIR:?Run 'make dotenv' to generate the .env single source of truth} diff --git a/tests/lint/repository/test_compose_build_needs_no_host_network.py b/tests/lint/repository/test_compose_build_needs_no_host_network.py new file mode 100644 index 0000000000..bb951b001a --- /dev/null +++ b/tests/lint/repository/test_compose_build_needs_no_host_network.py @@ -0,0 +1,60 @@ +"""Compose hands a build to ``docker buildx bake`` and grants it only the +``fs.read`` entitlement of the build context. Since Buildx 0.38.0 a block with +``network: host`` stops there with ``additional privileges requested``, and +declaring ``build.entitlements`` or setting ``COMPOSE_BAKE=false`` does not +change that, so such a service cannot be built through ``make compose-up``. +""" + +from __future__ import annotations + +import unittest +from pathlib import Path + +from utils.cache.files import iter_project_files +from utils.cache.yaml import load_yaml_any + +from . import PROJECT_ROOT + +STACK_FILE = str(PROJECT_ROOT / "compose.yml") +OVERRIDES = f"{PROJECT_ROOT / 'compose'}/" +STACK_BUILD = "compose.yml: infinito" + + +def _builds(path): + """Yield ``(label, build)`` for every service of a compose file that builds. + + Args: + path: compose file to read; a YAML file without ``services`` yields nothing. + """ + document = load_yaml_any(path) + if not isinstance(document, dict): + return + for name, service in (document.get("services") or {}).items(): + build = service.get("build") + if isinstance(build, dict): + yield f"{Path(path).relative_to(PROJECT_ROOT)}: {name}", build + + +class TestComposeBuildNeedsNoHostNetwork(unittest.TestCase): + def test_no_build_block_asks_for_host_networking(self) -> None: + compose_files = [ + path + for path in sorted(iter_project_files(extensions=(".yml",))) + if path == STACK_FILE or path.startswith(OVERRIDES) + ] + builds = dict(hit for path in compose_files for hit in _builds(path)) + self.assertIn(STACK_BUILD, builds, "the stack build block was not scanned") + offenders = [ + label for label, build in builds.items() if build.get("network") == "host" + ] + self.assertEqual( + [], + offenders, + "Compose cannot grant bake the network.host entitlement " + "(Buildx >= 0.38.0 refuses the build); drop 'network: host' from: " + f"{offenders}", + ) + + +if __name__ == "__main__": + unittest.main() From 3cefb01d7110508258ba883c19189f7cd1333b6e Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 8 Oct 2026 18:28:58 +0200 Subject: [PATCH 10/13] fix(web-app-suitecrm): keep sessions in Redis so replicas share them With three swarm replicas the PHP session files on the shared NFS volume were read as empty under concurrent requests of one browser, so SuiteCRM answered 403 on /api/graphql and dropped the session. Sessions now live in Redis with session locking that waits like flock did; the phpredis version is pinned in services.yml with its own update source, and the engine lookup renders the save path. Refs #879 Co-Authored-By: Claude Opus 5.5 (1M context) --- plugins/lookup/engine.py | 35 +++++++++++++ roles/web-app-suitecrm/files/Dockerfile | 5 ++ roles/web-app-suitecrm/meta/services.yml | 24 +++++++-- roles/web-app-suitecrm/meta/variants.yml | 4 ++ .../web-app-suitecrm/templates/compose.yml.j2 | 1 + roles/web-app-suitecrm/templates/env.j2 | 2 + roles/web-app-suitecrm/templates/php.ini.j2 | 6 ++- roles/web-app-suitecrm/vars/main.yml | 1 + .../lookup/test_engine_redis_session_path.py | 49 +++++++++++++++++++ 9 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 tests/unit/python/plugins/lookup/test_engine_redis_session_path.py diff --git a/plugins/lookup/engine.py b/plugins/lookup/engine.py index 5c769cc538..c428c9acdf 100644 --- a/plugins/lookup/engine.py +++ b/plugins/lookup/engine.py @@ -2,6 +2,7 @@ {{ lookup('engine', 'redis', application_id) }} # full dict {{ lookup('engine', 'redis', application_id, 'url') }} # one field + {{ lookup('engine', 'redis', application_id, 'session_path') }} # phpredis session.save_path Mirrors the ``database`` lookup but for the non-RDBMS engines registered in :data:`utils.roles.applications.services.engines.ENGINES` (redis, memcached, @@ -16,6 +17,7 @@ See ``docs/architecture/central-engines.md``. from __future__ import annotations from typing import Any +from urllib.parse import quote from ansible.errors import AnsibleError from ansible.plugins.loader import lookup_loader @@ -44,9 +46,31 @@ _FIELDS = ( "prefix", "url", "container", + "session_path", ) +def redis_session_path( + host: str, port: int, prefix: str, username: str, password: str +) -> str: + """Return the phpredis ``session.save_path`` of one consumer. + + Args: + host: redis host the consumer reaches. + port: redis port. + prefix: consumer prefix; session keys become ``:session:``. + username: ACL user, sent only together with a password. + password: redis password; empty for an embedded sidecar. + """ + path = f"tcp://{host}:{port}?prefix={quote(prefix, safe='')}:session:" + if password: + path += ( + f"&auth[user]={quote(username, safe='')}" + f"&auth[pass]={quote(password, safe='')}" + ) + return path + + def qdrant_consumer_key(applications: dict[str, Any], svc_id: str) -> str: """Return the engine-wide key a data-owning qdrant consumer presents. @@ -197,6 +221,17 @@ class LookupModule(LookupBase): "prefix": prefix, "url": url, "container": container, + "session_path": ( + redis_session_path( + host, + port, + prefix, + username if shared else "", + password if shared else "", + ) + if engine == "redis" + else "" + ), } if want == "all": return [resolved] diff --git a/roles/web-app-suitecrm/files/Dockerfile b/roles/web-app-suitecrm/files/Dockerfile index 681672774b..08c8f95e71 100644 --- a/roles/web-app-suitecrm/files/Dockerfile +++ b/roles/web-app-suitecrm/files/Dockerfile @@ -5,6 +5,7 @@ FROM ${SUITECRM_BASE_IMAGE}:${SUITECRM_BASE_VERSION} AS base ARG SUITECRM_APP_DIR ARG SUITECRM_PHP_MEMORY_LIMIT +ARG SUITECRM_PHPREDIS_VERSION RUN rm -rf /var/lib/apt/lists/* \ && apt-get -o Acquire::Retries=3 update && apt-get -o Acquire::Retries=3 install -y \ @@ -32,6 +33,10 @@ RUN rm -rf /var/lib/apt/lists/* \ ldap \ && rm -rf /var/lib/apt/lists/* +RUN : "${SUITECRM_PHPREDIS_VERSION:?}" \ + && pecl install "redis-${SUITECRM_PHPREDIS_VERSION}" \ + && docker-php-ext-enable redis + RUN : "${SUITECRM_PHP_MEMORY_LIMIT:?}" && { \ echo 'file_uploads = On'; \ echo 'upload_max_filesize = 32M'; \ diff --git a/roles/web-app-suitecrm/meta/services.yml b/roles/web-app-suitecrm/meta/services.yml index 68c73b211d..cc7a861a50 100644 --- a/roles/web-app-suitecrm/meta/services.yml +++ b/roles/web-app-suitecrm/meta/services.yml @@ -29,6 +29,16 @@ mariadb: bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-mariadb' in group_names }}" +redis: + bond: 1 + backup: + disabled: true + enabled: true # nocheck: dynamic-flag + shared: "{{ 'svc-db-redis' in group_names }}" + cpus: "0.25" + mem_reservation: 128m + mem_limit: 256m + pids_limit: 256 # nocheck: playwright-service-flag — self-provider entry; test_env_services_match.py:28-32 grants the forbidden-self-gate exemption suitecrm: healthcheck: @@ -49,12 +59,16 @@ suitecrm: version: 8.2-apache custom: true app_version: 8.9.1 + phpredis_version: 6.3.0 update: - key: app_version - type: git_tags - repository: https://github.com/salesagility/SuiteCRM-Core.git - match: "v" - strip: true + - key: app_version + type: git_tags + repository: https://github.com/salesagility/SuiteCRM-Core.git + match: "v" + strip: true + - key: phpredis_version + type: git_tags + repository: https://github.com/phpredis/phpredis.git name: suitecrm min_storage: 376MB cpus: 1.5 diff --git a/roles/web-app-suitecrm/meta/variants.yml b/roles/web-app-suitecrm/meta/variants.yml index 6353170e71..23ed53e8a9 100644 --- a/roles/web-app-suitecrm/meta/variants.yml +++ b/roles/web-app-suitecrm/meta/variants.yml @@ -24,6 +24,8 @@ shared: true mariadb: shared: true + redis: + shared: true matomo: enabled: true shared: true @@ -56,6 +58,8 @@ shared: false # nocheck: variants-static-override backup shared stays literally true mariadb: shared: false # nocheck: variants-static-override backup shared stays literally true + redis: + shared: false matomo: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true diff --git a/roles/web-app-suitecrm/templates/compose.yml.j2 b/roles/web-app-suitecrm/templates/compose.yml.j2 index 62057023e5..a4a9f433e8 100644 --- a/roles/web-app-suitecrm/templates/compose.yml.j2 +++ b/roles/web-app-suitecrm/templates/compose.yml.j2 @@ -6,6 +6,7 @@ SUITECRM_BASE_IMAGE: "{{ SUITECRM_BASE_IMAGE }}" SUITECRM_BASE_VERSION: "{{ SUITECRM_BASE_VERSION }}" SUITECRM_APP_VERSION: "{{ SUITECRM_APP_VERSION }}" + SUITECRM_PHPREDIS_VERSION: "{{ SUITECRM_PHPREDIS_VERSION }}" SUITECRM_ENTRYPOINT_SCRIPT_HOST_REL: "{{ SUITECRM_ENTRYPOINT_SCRIPT_HOST_REL }}" SUITECRM_ENTRYPOINT_SCRIPT_DOCKER: "{{ SUITECRM_ENTRYPOINT_SCRIPT_DOCKER }}" SUITECRM_LDAP_EXTENSION_DIR: "{{ SUITECRM_LDAP_EXTENSION_DIR }}" diff --git a/roles/web-app-suitecrm/templates/env.j2 b/roles/web-app-suitecrm/templates/env.j2 index 826cb31386..acc107ad15 100644 --- a/roles/web-app-suitecrm/templates/env.j2 +++ b/roles/web-app-suitecrm/templates/env.j2 @@ -21,6 +21,8 @@ SUITECRM_ADMIN_EMAIL={{ lookup('users', 'administrator').email }} SUITECRM_URL={{ SUITECRM_URL }} +SUITECRM_SESSION_SAVE_PATH={{ lookup('engine', 'redis', application_id, 'session_path') }} + {% if SUITECRM_SSO_ENABLED | bool %} AUTH_TYPE=saml diff --git a/roles/web-app-suitecrm/templates/php.ini.j2 b/roles/web-app-suitecrm/templates/php.ini.j2 index e818dbc83e..1ff9b031e4 100644 --- a/roles/web-app-suitecrm/templates/php.ini.j2 +++ b/roles/web-app-suitecrm/templates/php.ini.j2 @@ -3,5 +3,7 @@ display_startup_errors = Off log_errors = On error_reporting = E_ALL & ~E_DEPRECATED & ~E_USER_DEPRECATED -session.save_handler = files -session.save_path = "{{ SUITECRM_APP_DIR }}/tmp" +session.save_handler = redis +session.save_path = ${SUITECRM_SESSION_SAVE_PATH} +redis.session.locking_enabled = 1 +redis.session.lock_retries = -1 diff --git a/roles/web-app-suitecrm/vars/main.yml b/roles/web-app-suitecrm/vars/main.yml index 2b1708dfdd..aaf9602ea6 100644 --- a/roles/web-app-suitecrm/vars/main.yml +++ b/roles/web-app-suitecrm/vars/main.yml @@ -11,6 +11,7 @@ SUITECRM_PHP_CONFIG_HOST: "{{ [ lookup('container', application_id, 'directories SUITECRM_BASE_IMAGE: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.image') }}" SUITECRM_BASE_VERSION: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.version') }}" SUITECRM_APP_VERSION: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.app_version') }}" +SUITECRM_PHPREDIS_VERSION: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.phpredis_version') }}" SUITECRM_CONTAINER: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.name') }}" SUITECRM_CONTAINER_ADDRESS: "{{ lookup('container_address', application_id, entity_name) }}" diff --git a/tests/unit/python/plugins/lookup/test_engine_redis_session_path.py b/tests/unit/python/plugins/lookup/test_engine_redis_session_path.py new file mode 100644 index 0000000000..0b8dba098e --- /dev/null +++ b/tests/unit/python/plugins/lookup/test_engine_redis_session_path.py @@ -0,0 +1,49 @@ +"""phpredis reads ``auth[user]``/``auth[pass]`` and ``prefix`` from the query of +``session.save_path``; a password with a reserved character that is not +percent-encoded splits the query and the handler authenticates with a fragment. +""" + +from __future__ import annotations + +import importlib +import unittest +from urllib.parse import parse_qs, urlsplit + +plugin_module = importlib.import_module("plugins.lookup.engine") +redis_session_path = plugin_module.redis_session_path + + +class TestRedisSessionPath(unittest.TestCase): + def test_an_embedded_sidecar_gets_no_credentials(self) -> None: + self.assertEqual( + "tcp://redis:6379?prefix=suitecrm:session:", + redis_session_path("redis", 6379, "suitecrm", "", ""), + ) + + def test_a_shared_redis_gets_user_and_password(self) -> None: + query = parse_qs( + urlsplit( + redis_session_path("redis-central", 6379, "suitecrm", "default", "pw") + ).query + ) + self.assertEqual(["default"], query["auth[user]"]) + self.assertEqual(["pw"], query["auth[pass]"]) + + def test_a_password_with_reserved_characters_survives_the_query(self) -> None: + password = "a&b=c?d#e/f+g h" + query = parse_qs( + urlsplit( + redis_session_path("redis-central", 6379, "suitecrm", "default", password) + ).query + ) + self.assertEqual([password], query["auth[pass]"]) + self.assertEqual(["suitecrm:session:"], query["prefix"]) + + def test_a_user_without_a_password_is_not_sent(self) -> None: + self.assertNotIn( + "auth", redis_session_path("redis", 6379, "suitecrm", "default", "") + ) + + +if __name__ == "__main__": + unittest.main() From 057347df3f5894249485d9aa15077bc751539561 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:44:00 +0000 Subject: [PATCH 11/13] update(images): update Docker image versions (web-app-socialhome) --- roles/web-app-socialhome/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-socialhome/meta/services.yml b/roles/web-app-socialhome/meta/services.yml index 77098a50d5..43f68c9d7a 100644 --- a/roles/web-app-socialhome/meta/services.yml +++ b/roles/web-app-socialhome/meta/services.yml @@ -13,7 +13,7 @@ socialhome: # nocheck: default-placement-manager embedded sqlite in a node-local volume; no clustered storage placement: manager image: ghcr.io/social-home-io/socialhome - version: "2026.10.7" + version: "2026.10.8" ports: local: http: 8089 From c32984f94e3df6ff7dcafad8bb3d3538495f65a9 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 8 Oct 2026 20:11:10 +0200 Subject: [PATCH 12/13] fix(web-app-suitecrm): mark the Redis session store as having no Playwright surface The Redis service #881 added is always on and only holds sessions, so no spec can gate on it. tests/integration/roles/playwright/test_env_services_match.py failed on main and in every pull request because the service carried neither a REDIS_SERVICE_ENABLED flag nor the marker the other Redis consumers use. Co-Authored-By: Claude Opus 5.5 (1M context) --- roles/web-app-suitecrm/meta/services.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/web-app-suitecrm/meta/services.yml b/roles/web-app-suitecrm/meta/services.yml index cc7a861a50..042eb5b315 100644 --- a/roles/web-app-suitecrm/meta/services.yml +++ b/roles/web-app-suitecrm/meta/services.yml @@ -29,6 +29,7 @@ mariadb: bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-mariadb' in group_names }}" +# nocheck: playwright-service-flag — session store behind every request, no Playwright surface of its own redis: bond: 1 backup: From 64677d479022e48f28ed7063de73d4a82b47d1f7 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 8 Oct 2026 20:55:04 +0200 Subject: [PATCH 13/13] chore: regenerate role docs and apply autoformat after #881 and #873 `make quality-high` rewrote these files on every branch: the SuiteCRM README and the root role table lacked the Redis service #881 added, the session path test was not formatted, and an OpenTalk secret description from #873 was not wrapped. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 2 +- roles/web-app-opentalk/meta/secrets.yml | 3 ++- roles/web-app-suitecrm/README.md | 3 +++ .../python/plugins/lookup/test_engine_redis_session_path.py | 4 +++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ae2daf2654..87d450bbfd 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,7 @@ Every invokable role in the tested lifecycle envelope, with its upstream homepag | [Sphinx](roles/web-app-sphinx/) | beta | Documentation generator that builds HTML from a source repository and serves it through a lightweight HTTP server | [🌐](https://www.sphinx-doc.org/) [🛠️](roles/web-app-sphinx/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [Stalwart Mail Server](roles/web-app-stalwart/) | beta | All-in-one Stalwart mail server with SMTP, IMAP, JMAP, POP3, ManageSieve, CalDAV, CardDAV and built-in spam filtering | [🌐](https://stalw.art/) [🛠️](roles/web-app-stalwart/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Tor SMTP Gateway (svc-net-tor-smtp)](roles/svc-net-tor-smtp/), [Universal Logout](roles/web-svc-logout/) | | [Storage Optimizer](roles/svc-opt-ssd-hdd/) | beta | Tiering role placing Docker volumes on SSD or HDD by workload type, with symlinks keeping paths consistent | [🛠️](roles/svc-opt-ssd-hdd/README.md#quick-setup) | | -| [SuiteCRM](roles/web-app-suitecrm/) | beta | Open-source CRM platform for sales, service, cases, and campaign management with workflow automation | [🌐](https://suitecrm.com/) [🎬](https://www.youtube.com/watch?v=pOAs_C3Fp_E) [🛠️](roles/web-app-suitecrm/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [SuiteCRM](roles/web-app-suitecrm/) | beta | Open-source CRM platform for sales, service, cases, and campaign management with workflow automation | [🌐](https://suitecrm.com/) [🎬](https://www.youtube.com/watch?v=pOAs_C3Fp_E) [🛠️](roles/web-app-suitecrm/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Swapfile](roles/svc-opt-swapfile/) | beta | On-target automation for creating and activating swap space via a cloned swapfile script | [🛠️](roles/svc-opt-swapfile/README.md#quick-setup) | | | [Taiga](roles/web-app-taiga/) | beta | Open-source agile project management platform for Scrum, Kanban, and hybrid team workflows | [🌐](https://taiga.io/) [🎬](https://www.youtube.com/watch?v=TYPX8nQEU5I) [🛠️](roles/web-app-taiga/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-app-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Tor Onion Service](roles/svc-net-tor/) | beta | Tor daemon publishing the node as a v3 onion service | [🛠️](roles/svc-net-tor/README.md#quick-setup) | [Asset Mirror (Privacy Proxy)](roles/web-svc-mirror/) | diff --git a/roles/web-app-opentalk/meta/secrets.yml b/roles/web-app-opentalk/meta/secrets.yml index e0b40eb4e4..5b251109a8 100644 --- a/roles/web-app-opentalk/meta/secrets.yml +++ b/roles/web-app-opentalk/meta/secrets.yml @@ -16,7 +16,8 @@ credentials: validation: min_length: 32 recorder_api_secret: - description: API key secret the controller signs its requests to the recorder with (`roomserver.modules.recording.api_key`, `http.api_keys` of the recorder). + description: API key secret the controller signs its requests to the recorder with (`roomserver.modules.recording.api_key`, `http.api_keys` of the + recorder). algorithm: alphanumeric validation: min_length: 32 diff --git a/roles/web-app-suitecrm/README.md b/roles/web-app-suitecrm/README.md index 70fc57dce9..8091a52906 100644 --- a/roles/web-app-suitecrm/README.md +++ b/roles/web-app-suitecrm/README.md @@ -27,6 +27,7 @@ flowchart LR dep_svc_bkp_volume_2_local["svc-bkp-volume-2-local 💻"] dep_svc_db_mariadb["svc-db-mariadb 🐳🐝"] dep_svc_db_openldap["svc-db-openldap 🐳🐝"] + dep_svc_db_redis["svc-db-redis 🐳🐝"] dep_svc_net_tor["svc-net-tor 🐳🐝"] dep_web_app_dashboard["web-app-dashboard 🐳🐝"] dep_web_app_keycloak["web-app-keycloak 🐳🐝"] @@ -44,6 +45,7 @@ flowchart LR svc_matomo["matomo"] svc_email["email"] svc_mariadb["mariadb"] + svc_redis["redis"] svc_suitecrm["suitecrm"] svc_css["css"] svc_prometheus["prometheus"] @@ -53,6 +55,7 @@ flowchart LR dep_svc_bkp_volume_2_local -. "0..1" .-> svc_container_backup dep_svc_db_mariadb -. "0..1" .-> svc_mariadb dep_svc_db_openldap -. "0..1" .-> svc_ldap + dep_svc_db_redis -. "0..1" .-> svc_redis dep_svc_net_tor -. "0..1" .-> svc_tor dep_web_app_dashboard -. "0..1" .-> svc_dashboard dep_web_app_keycloak -. "0..1" .-> svc_sso diff --git a/tests/unit/python/plugins/lookup/test_engine_redis_session_path.py b/tests/unit/python/plugins/lookup/test_engine_redis_session_path.py index 0b8dba098e..d7576f2d90 100644 --- a/tests/unit/python/plugins/lookup/test_engine_redis_session_path.py +++ b/tests/unit/python/plugins/lookup/test_engine_redis_session_path.py @@ -33,7 +33,9 @@ class TestRedisSessionPath(unittest.TestCase): password = "a&b=c?d#e/f+g h" query = parse_qs( urlsplit( - redis_session_path("redis-central", 6379, "suitecrm", "default", password) + redis_session_path( + "redis-central", 6379, "suitecrm", "default", password + ) ).query ) self.assertEqual([password], query["auth[pass]"])