diff --git a/compose/swarm/playbook.yml b/compose/swarm/playbook.yml index f1d3f0431a..98738d2f04 100644 --- a/compose/swarm/playbook.yml +++ b/compose/swarm/playbook.yml @@ -27,9 +27,6 @@ lab_systemd_timeout: "{{ lookup('env', 'SWARM_PILOT_SYSTEMD_TIMEOUT') | default('180', true) }}" lab_is_mgr: "{{ inventory_hostname == lab_mgr }}" lab_is_worker: "{{ inventory_hostname in [lab_wrk1, lab_wrk2] }}" - # Exception: the NFS sidecar is a DNS node too — the backup stack lands on - # it and runs the same sys-svc-container DinD-DNS gate as the swarm nodes, - # which NXDOMAINs on the app domain without the dnsmasq wildcard. lab_dns_node: "{{ lab_is_mgr or lab_is_worker or inventory_hostname == lab_nfs }}" lab_domain_target: "{{ lookup('env', 'INFINITO_SWARM_MGR_IP') }}" lab_filesystem: "{{ lookup('env', 'INFINITO_DOCKER_FILESYSTEM') }}" diff --git a/inventories/development/default.yml b/inventories/development/default.yml index 8df4658e11..f8a0351f91 100644 --- a/inventories/development/default.yml +++ b/inventories/development/default.yml @@ -56,8 +56,6 @@ users: uid: 10001 gid: 10001 - # Exception: the mail personas exchange real messages (Playwright webmail flows, the - # Stalwart migration e2e), which a role-derived 'identity'-only account cannot receive. accounts: ["mailbox", "identity"] roles: [] tokens: {} diff --git a/inventories/development/swarm.yml b/inventories/development/swarm.yml index 009d41b266..b2aa786804 100644 --- a/inventories/development/swarm.yml +++ b/inventories/development/swarm.yml @@ -20,10 +20,11 @@ storage: SVC_STORAGE_NFS_SERVER_STATE_ALLOWED_CIDRS: - "192.168.244.0/24" # nocheck: hardcoded-subnet (act swarm-lab subnet; nodes + controller mount the fsid=0 state export from it) -# Exception: env-steerable so bleeding-edge-kernel hosts can pin "0" — their -# NFS client breaks ganesha 4.1+ sessions (first access after mount EIOs). SVC_STORAGE_NFS_SERVER_MINOR_VERSIONS: "{{ lookup('env', 'INFINITO_SWARM_NFS_MINOR_VERSIONS') | default('0, 1, 2', true) }}" # nocheck: swarm-only inventory; evaluated on the CI runner where load.sh exports it, never inside the compose DiD +MAIL_PRODUCTION_HOST: "{{ lookup('env', 'INFINITO_MAIL_PRODUCTION_HOST') | default('', true) }}" +MAIL_PRODUCTION_RESOLVER: "{{ lookup('env', 'INFINITO_MAIL_PRODUCTION_RESOLVER') | default('', true) }}" + swarm: network: encryption: true diff --git a/plugins/lookup/database.py b/plugins/lookup/database.py index 5d1f14e47f..e6649e604c 100644 --- a/plugins/lookup/database.py +++ b/plugins/lookup/database.py @@ -83,8 +83,6 @@ class LookupModule(LookupBase): database_service = get_database_service_config(applications, consumer_id) enabled = bool(database_service.get("enabled", False)) shared = bool(database_service.get("shared", False)) - # Exception: a service-level `name:` declares an app-shipped DB container (e.g. upstream - # compose bundles) — the platform does not provision it, so convention credentials are fiction. name_override = str(database_service.get("name") or "").strip() managed = bool(enabled and not shared and not name_override) diff --git a/plugins/lookup/smtp_host.py b/plugins/lookup/smtp_host.py index bdb38ecfda..31eec66989 100644 --- a/plugins/lookup/smtp_host.py +++ b/plugins/lookup/smtp_host.py @@ -80,10 +80,6 @@ class LookupModule(LookupBase): raise AnsibleError("smtp_host: the term must be the email mapping.") variables = variables or getattr(self._templar, "available_variables", {}) or {} in_container = _to_bool(kwargs.get("in_container", False), strict=False) - # Exception: these two reach a lookup as raw group_vars templates, and an unrendered - # `{{ ... }}` string is neither true nor a known mode — the rig would silently keep the - # configured relay domain (a .onion on an onion node, which msmtp cannot route) instead - # of the loopback this resolver exists to hand back. templar = getattr(self, "_templar", None) variables = dict(variables) for key in ("DOCKER_IN_CONTAINER", "DEPLOYMENT_MODE"): diff --git a/plugins/lookup/tor_ports.py b/plugins/lookup/tor_ports.py index ec24f33741..4c20823d9e 100644 --- a/plugins/lookup/tor_ports.py +++ b/plugins/lookup/tor_ports.py @@ -24,18 +24,23 @@ def _as_bool(value: object) -> bool: return bool(value) -def _collect_local_tcp_ports(port_categories: Any, into: set[int]) -> None: - """Add single-int TCP ports of the loopback-published ``ports.local`` group - only. An exposed service publishes just its local plaintext port to - 127.0.0.1, so the onion HiddenServicePort must target that; forwarding a - ``public`` TLS port (e.g. ldaps 636) would be a dead loopback target (nothing - listens there in the exposed variant) and plaintext-into-TLS.""" +def _collect_group_tcp_ports(port_categories: Any, group: str, into: set[int]) -> None: + """Add single-int TCP ports of one ``ports.`` category group. + + ``exposed: true`` collects ``local`` only: an exposed service publishes just + its local plaintext port to 127.0.0.1, so the onion HiddenServicePort must + target that; forwarding its ``public`` TLS port (e.g. ldaps 636) would be a + dead loopback target (nothing listens there in the exposed variant) and + plaintext-into-TLS. ``exposed: public`` collects ``public`` instead, for a + service whose protocol ports genuinely bind the host (the active mail + provider's 25/465/993/...) and should ride the node onion as-is. + """ if not isinstance(port_categories, dict): return - local = port_categories.get("local") - if not isinstance(local, dict): + entries = port_categories.get(group) + if not isinstance(entries, dict): return - for category, value in local.items(): + for category, value in entries.items(): if category in _UDP_ONLY_CATEGORIES: continue if isinstance(value, int): @@ -81,11 +86,14 @@ def collect_exposed_ports( ``exposed`` is an explicit per-service opt-in (default false) that makes the service reachable over the node onion: its port gets a dedicated - ``HiddenServicePort``. Because ``applications`` is the variant-merged config - (base ``meta/services.yml`` deep-merged with the round's ``meta/variants.yml`` + ``HiddenServicePort``. ``exposed: true`` forwards the service's ``ports.local`` + group; ``exposed: public`` forwards ``ports.public`` instead, for a service + whose protocol ports bind the host directly (the active mail provider). + Because ``applications`` is the variant-merged config (base + ``meta/services.yml`` deep-merged with the round's ``meta/variants.yml`` override), a service is forwarded only in the variant/config where it sets - ``exposed: true`` — which is exactly what lets the two DB variants be tested - over Tor (v0 exposed -> reachable, v1 not -> refused). + the flag — which is exactly what lets the two DB variants be tested over Tor + (v0 exposed -> reachable, v1 not -> refused). """ ports: set[int] = set() deployed = set(deployed_roles) @@ -98,9 +106,13 @@ def collect_exposed_ports( if not isinstance(services, dict): continue for entity in services.values(): - if not isinstance(entity, dict) or not _as_bool(entity.get("exposed")): + if not isinstance(entity, dict): continue - _collect_local_tcp_ports(entity.get("ports"), ports) + exposed = entity.get("exposed") + if isinstance(exposed, str) and exposed.strip().lower() == "public": + _collect_group_tcp_ports(entity.get("ports"), "public", ports) + elif _as_bool(exposed): + _collect_group_tcp_ports(entity.get("ports"), "local", ports) return sorted(ports) @@ -114,8 +126,9 @@ class LookupModule(LookupBase): '127.0.0.1:'}`` dicts, sorted by port. Two sources are unioned: * every ``ports.public`` TCP port of tor-enabled roles, and - * every port of a service that opts in with ``exposed: true`` (resolved - against the variant-merged applications view, so per-variant). + * every port of a service that opts in with ``exposed`` (resolved against + the variant-merged applications view, so per-variant): ``true`` forwards + its ``ports.local`` group, ``public`` its ``ports.public`` group. Public/exposed ports bind the host interface, so the loopback target reaches them from svc-net-tor's host-network container. Consumed by diff --git a/roles/svc-storage-nfs-server/defaults/main.yml b/roles/svc-storage-nfs-server/defaults/main.yml index daba048280..c0aff0a7be 100644 --- a/roles/svc-storage-nfs-server/defaults/main.yml +++ b/roles/svc-storage-nfs-server/defaults/main.yml @@ -1,6 +1,3 @@ --- # Param: NFSv4 minor versions ganesha offers to clients (comma-separated). -# Exception: overridable default (not vars/) — pin to "0" on hosts whose -# bleeding-edge kernel NFS client breaks ganesha 4.1+ sessions (first "ls" -# after mount returns EIO; observed with kernel 7.0 against ganesha 6.x). SVC_STORAGE_NFS_SERVER_MINOR_VERSIONS: "0, 1, 2" diff --git a/roles/sys-ca-selfsigned/files/shell/with-ca-trust.sh b/roles/sys-ca-selfsigned/files/shell/with-ca-trust.sh index f7152392f4..0b38ecaa07 100644 --- a/roles/sys-ca-selfsigned/files/shell/with-ca-trust.sh +++ b/roles/sys-ca-selfsigned/files/shell/with-ca-trust.sh @@ -6,8 +6,6 @@ set -eu VERBOSE="${VERBOSE:-1}" -# Exception: SPOT — system CA bundle candidates, in preference order -# (Debian/Ubuntu, RHEL/Fedora, Alpine/openssl default). SYS_CA_BUNDLE_CANDIDATES="/etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/cert.pem" log() { @@ -40,9 +38,6 @@ log "Sanitized trust name: $name" installed=0 -# Exception: env-based trust fallback builds a COMBINED bundle (system CAs + -# our CA) so the env vars don't break public-HTTPS validation; falls back to -# our CA only if no system bundle is found. ca_bundle="$CA_TRUST_CERT" combined="/tmp/with-ca-trust-combined.crt" # shellcheck disable=SC2086 # intentional word-splitting; paths contain no spaces @@ -57,7 +52,6 @@ done export SSL_CERT_FILE="$ca_bundle" export REQUESTS_CA_BUNDLE="$ca_bundle" export CURL_CA_BUNDLE="$ca_bundle" -# Exception: Node already ships public roots; it only needs our CA appended. export NODE_EXTRA_CA_CERTS="$CA_TRUST_CERT" if [ -n "${CA_TRUST_CERT_EXTRA:-}" ] && [ -r "${CA_TRUST_CERT_EXTRA}" ]; then diff --git a/roles/sys-ctl-alm-compose/tasks/00_core.yml b/roles/sys-ctl-alm-compose/tasks/00_core.yml index 6fb3cb88b5..693bf77fdb 100644 --- a/roles/sys-ctl-alm-compose/tasks/00_core.yml +++ b/roles/sys-ctl-alm-compose/tasks/00_core.yml @@ -39,7 +39,4 @@ state: started when: - MODE_ASSERT_LOCAL | bool - # Exception: shared-service preloading happens before the mail provider - # exposes its final TLS SMTP listener, so the notifier self-test must wait - # for the normal app phase. - not (service_loader_preloading | default(false) | bool) diff --git a/roles/sys-ctl-hlth-msmtp/tasks/00_core.yml b/roles/sys-ctl-hlth-msmtp/tasks/00_core.yml index aa8759ceb5..e7d1a3f458 100644 --- a/roles/sys-ctl-hlth-msmtp/tasks/00_core.yml +++ b/roles/sys-ctl-hlth-msmtp/tasks/00_core.yml @@ -9,15 +9,6 @@ - "MAIL_RECIPIENT={{ lookup('mail_recipient') }}" - "MAIL_TIMEOUT={{ lookup('email').timeout }}" - # Exception: force the early mail-send check — but never during - # shared-service preloading: this role deploys nested inside the mail - # provider's cert-deploy chain (web-app-stalwart 05_tls), BEFORE the - # platform cert is registered, so a strict-TLS send would hit the built-in - # bootstrap certificate and fail. The debug-mode default state - # ('restarted') would start the unit at deploy time regardless of flush - # timing — suppress both during preload. The alm-compose self-test covers - # the real send in the normal app phase (same guard, same reason), and the - # timer runs the check thereafter. system_service_state: "{{ omit if (service_loader_preloading | default(false) | bool) else SYS_SERVICE_DEFAULT_STATE }}" system_service_force_flush_instant: "{{ (MODE_DEBUG | bool) and not (service_loader_preloading | default(false) | bool) }}" system_service_force_flush_final: false diff --git a/roles/sys-ctl-mtn-cert-deploy/files/shell/script.sh b/roles/sys-ctl-mtn-cert-deploy/files/shell/script.sh index cd6db170d5..daa63f9d54 100644 --- a/roles/sys-ctl-mtn-cert-deploy/files/shell/script.sh +++ b/roles/sys-ctl-mtn-cert-deploy/files/shell/script.sh @@ -39,8 +39,6 @@ fi echo "Copying certificates from: $ssl_cert_source_dir -> $docker_compose_cert_directory" cp -RvL "${ssl_cert_source_dir}/"* "$docker_compose_cert_directory" -# Exception: Mailu expects key.pem/cert.pem — map from the LE names -# (privkey.pem/fullchain.pem) when present, else pass key/cert through. if [ -f "${ssl_cert_source_dir}/privkey.pem" ] && [ -f "${ssl_cert_source_dir}/fullchain.pem" ]; then cp -v "${ssl_cert_source_dir}/privkey.pem" "${docker_compose_cert_directory}/key.pem" cp -v "${ssl_cert_source_dir}/fullchain.pem" "${docker_compose_cert_directory}/cert.pem" @@ -57,16 +55,8 @@ chmod a+r -v "${docker_compose_cert_directory}/"* || exit 1 cd "$docker_compose_instance_directory" || exit 1 -# Exception: "--" stops wrapper arg parsing, so compose flags like -# "--services" pass through to the correct -p/-f/--env-file stack. services="$(sh -c "$compose_cmd -- ps --services")" -# Rationale: restart every service that consumes the deployed certificate -# material, i.e. has the cert directory bind-mounted. The TLS terminator -# differs per provider (Mailu: the nginx front container; Stalwart: the mail -# server itself), so selection MUST go by consumption, not by which binary a -# container ships — a service that keeps running with the old cert serves it -# until expiry. restart_services="" for service in $services; do diff --git a/roles/sys-ctl-mtn-cert-deploy/files/shell/test.sh b/roles/sys-ctl-mtn-cert-deploy/files/shell/test.sh index 28066ec062..0a2f4ad5bd 100755 --- a/roles/sys-ctl-mtn-cert-deploy/files/shell/test.sh +++ b/roles/sys-ctl-mtn-cert-deploy/files/shell/test.sh @@ -48,8 +48,6 @@ check_local() { return fi - # Exception: compare public keys, not moduli — an EC key has no modulus, so the - # conventional `openssl ... -modulus` pairing check silently passes for every EC cert. cert_pub="$(openssl x509 -in "$cert" -pubkey -noout 2>/dev/null)" key_pub="$(openssl pkey -in "$key" -pubout 2>/dev/null)" if [ -z "$key_pub" ] || [ "$cert_pub" != "$key_pub" ]; then diff --git a/roles/sys-ctl-mtn-cert-deploy/meta/services.yml b/roles/sys-ctl-mtn-cert-deploy/meta/services.yml index cc596a0325..0e19e2db9c 100644 --- a/roles/sys-ctl-mtn-cert-deploy/meta/services.yml +++ b/roles/sys-ctl-mtn-cert-deploy/meta/services.yml @@ -2,13 +2,7 @@ cert-deploy: lifecycle: beta required_by: - # Exception: only the default provider is a static requirement — Mailu - # includes the shared run-once unit only when it is the ACTIVE provider - # (MAIL_PROVIDER), and a legacy/standalone Mailu deploy (email consumer - # off, TLS_FLAVOR=notls behind the proxy) legitimately never runs it, so - # listing web-app-mailu here fails the required_by execution guard on - # exactly that deploy shape. compose: - roles: [web-app-stalwart] + roles: [web-app-stalwart, web-app-mailu] swarm: - roles: [web-app-stalwart] + roles: [web-app-stalwart, web-app-mailu] diff --git a/roles/sys-ctl-mtn-cert-deploy/tasks/00_core.yml b/roles/sys-ctl-mtn-cert-deploy/tasks/00_core.yml index 357bdcc967..5b9666ef18 100644 --- a/roles/sys-ctl-mtn-cert-deploy/tasks/00_core.yml +++ b/roles/sys-ctl-mtn-cert-deploy/tasks/00_core.yml @@ -24,9 +24,6 @@ ansible.builtin.script: cmd: "{{ [ role_path, 'files/shell/test.sh' ] | path_join }} {{ lookup('container', application_id, 'directories.instance') }}" environment: - # Exception: caller-supplied endpoints win so the mail provider can point the remote - # check at the production server it just configured; the env var stays the fallback - # for standalone use. CERT_TEST_REMOTE_ENDPOINTS: >- {{ cert_deploy_remote_endpoints | default(lookup('env', 'CERT_TEST_REMOTE_ENDPOINTS'), true) }} diff --git a/roles/sys-front-tls-selfsigned/tasks/00_core.yml b/roles/sys-front-tls-selfsigned/tasks/00_core.yml index 35e478729e..94f6ecfdab 100644 --- a/roles/sys-front-tls-selfsigned/tasks/00_core.yml +++ b/roles/sys-front-tls-selfsigned/tasks/00_core.yml @@ -106,7 +106,6 @@ - name: "🧮 TLS | (self_signed) compute wanted/existing SAN sets" ansible.builtin.set_fact: _ss_wanted_sans: "{{ _ss_san | map('lower') | list | unique | sort }}" - # Danger: \\s would put a literal 's' in the class and truncate every SAN. _ss_existing_sans: >- {{ ( diff --git a/roles/sys-svc-compose-ca/files/python/compose_ca.py b/roles/sys-svc-compose-ca/files/python/compose_ca.py index a559fc2e9e..7a0ec19938 100755 --- a/roles/sys-svc-compose-ca/files/python/compose_ca.py +++ b/roles/sys-svc-compose-ca/files/python/compose_ca.py @@ -297,12 +297,6 @@ def _gather_one_image( ["docker", "image", "inspect", image], cwd=cwd, env=env, timeout=90 ) if rc not in {0, TIMEOUT_RC} and pull_if_absent: - # Exception: this override is generated before the stack's `compose - # pull`, and `docker image inspect` does not pull. Pull an absent - # image so the /bin/sh probe (and thus the CA-trust wrapper) is not - # skipped. Never pull locally-BUILT tags (pull_if_absent=False): the - # registry would serve an unrelated base image whose CMD then gets - # pinned into the override, clobbering the built app's entrypoint. run(["docker", "pull", image], cwd=cwd, env=env, timeout=600, capture=False) rc, out, _err = run( ["docker", "image", "inspect", image], cwd=cwd, env=env, timeout=90 diff --git a/roles/sys-svc-mail-msmtp/templates/msmtprc.conf.j2 b/roles/sys-svc-mail-msmtp/templates/msmtprc.conf.j2 index 7c045f3c91..98c22ba93c 100644 --- a/roles/sys-svc-mail-msmtp/templates/msmtprc.conf.j2 +++ b/roles/sys-svc-mail-msmtp/templates/msmtprc.conf.j2 @@ -2,7 +2,6 @@ {% set ca_injected = sys_svc_mail_msmtp_in_container | bool and lookup('ca_injected', application_id) %} {% set smtp_target = lookup('smtp_host', email, in_container=sys_svc_mail_msmtp_in_container) %} defaults -# Exception: strict servers (Stalwart rejectNonFqdn) refuse a bare container hostname as EHLO. domain {{ DOMAIN_PRIMARY }} logfile ~/.msmtp.log timeout {{ email.timeout }} @@ -15,14 +14,6 @@ tls_trust_file {{ CA_TRUST.inject_cert_container }} {% endif %} tls_host_override {{ email.host }} {% if smtp_target.endswith('.onion') %} -{# Exception: gated on the host actually dialled, NOT on email.host — inside the rig - smtp_host hands back loopback even when the provider's canonical is an onion, and - routing 127.0.0.1 through Tor hangs until the send times out (rc=124). A real .onion - peer has no route off the node, so it must go through Tor's SOCKS5 listener. The - endpoint comes from lookup('tor_socks_proxy'), which owns the swarm-vs-compose host - split, so that policy is not restated here; the host render reaches the same listener - on loopback. Both lookups sit inside this branch because they raise when svc-net-tor - is absent, which is every clearnet deployment. #} {% set socks = (lookup('tor_socks_proxy') | regex_replace('^socks5h://', '')) if sys_svc_mail_msmtp_in_container | bool else lookup('tor_socks') %} proxy_host {{ socks.split(':')[0] }} proxy_port {{ socks.split(':')[1] }} diff --git a/roles/sys-svc-rdbms/tasks/dedicated.yml b/roles/sys-svc-rdbms/tasks/dedicated.yml index 0c5695ae07..4c91b600be 100644 --- a/roles/sys-svc-rdbms/tasks/dedicated.yml +++ b/roles/sys-svc-rdbms/tasks/dedicated.yml @@ -15,7 +15,6 @@ include_tasks: dedicated_postgres.yml when: _database_type == 'postgres' -# Exception: without this row baudolo silently falls back to a live-file snapshot and the restore replays a torn data dir. - name: "🔧 For '{{ application_id }}/{{ _database_consumer_id }}': Seed the backup entry" include_tasks: utils/seed_backup.yml when: diff --git a/roles/test-e2e-playwright/files/personas/utils/index.js b/roles/test-e2e-playwright/files/personas/utils/index.js index d2eb6ae9d3..f3c8c8d972 100644 --- a/roles/test-e2e-playwright/files/personas/utils/index.js +++ b/roles/test-e2e-playwright/files/personas/utils/index.js @@ -22,6 +22,7 @@ const interaction = require("./interaction"); const dotenv = require("./dotenv"); const dom = require("./dom"); const seaweedfs = require("./seaweedfs"); +const mailbox = require("./mailbox"); module.exports = { ...env, @@ -34,4 +35,5 @@ module.exports = { ...dotenv, ...dom, ...seaweedfs, + ...mailbox, }; diff --git a/roles/test-e2e-playwright/files/personas/utils/mailbox.js b/roles/test-e2e-playwright/files/personas/utils/mailbox.js new file mode 100644 index 0000000000..6c3876d1e6 --- /dev/null +++ b/roles/test-e2e-playwright/files/personas/utils/mailbox.js @@ -0,0 +1,87 @@ +/** + * Read a message back out of the recipient's mailbox, through the mail + * provider's webmail. + * + * Outbound mail configuration fails in ways the sender never notices: an + * implicit-TLS port announced as STARTTLS, AUTH offered on a listener that has + * none, or a relay the application can reach but the provider rejects. In every + * one of those the send call still returns success. The only proof that a + * setting is right is the message arriving, so these helpers exist for any role + * whose configuration decides how mail leaves it. + * + * `webmailSsoLogin(page, username, password, baseUrl)` + * Drives the provider's webmail login. The webmail redirects to Keycloak on + * its own (`oauth_login_redirect`), so this only has to click an SSO control + * when one is rendered, then fill the Keycloak form. A caller that already + * holds a Keycloak session never sees a form — the redirect answers straight + * back into the mailbox — so the helper waits for whichever of the two + * surfaces arrives instead of hanging on the form. + * + * `waitForEmailInMailbox(page, baseUrl, subject, timeout)` + * Polls INBOX and Junk for a message whose subject matches, and returns its + * row. Junk is not a fallback for a broken setup: a test domain publishes no + * mail-auth DNS, so a correctly delivered message is legitimately filed + * there. Folders are switched by `_mbox` URL, because clicking a folder link + * hangs when the row it points at is not actionable yet. + */ + +const { performKeycloakLoginForm } = require("./keycloak"); +const { gotoOnion } = require("./env"); +const { resolveTimeout } = require("../../timeouts"); + +const MAIL_UI = "#messagelist, .compose, a[href*='_action=compose'], .toolbar"; +const SSO_CONTROL = /sso|single sign.?on|login with|openid/i; + +const MAILBOXES = ["INBOX", "Junk Mail"]; + +async function webmailSsoLogin(page, username, password, baseUrl) { + await gotoOnion(page, `${baseUrl}/`); + const ssoButton = page + .getByRole("button", { name: SSO_CONTROL }) + .or(page.getByRole("link", { name: SSO_CONTROL })); + if (await ssoButton.first().isVisible().catch(() => false)) { + await ssoButton.first().click({ timeout: resolveTimeout(30_000) }); + } + + const mailUi = page.locator(MAIL_UI).first(); + const usernameField = page + .getByRole("textbox", { name: /username|email/i }) + .or(page.locator("input[name='username'], input#username")) + .first(); + + await Promise.any([ + mailUi.waitFor({ state: "visible", timeout: resolveTimeout(60_000) }), + usernameField.waitFor({ state: "visible", timeout: resolveTimeout(60_000) }), + ]); + if (await usernameField.isVisible().catch(() => false)) { + await performKeycloakLoginForm(page, username, password); + } + await mailUi.waitFor({ state: "visible", timeout: resolveTimeout(60_000) }); +} + +function messageRow(page, subject) { + return page + .locator("#messagelist tbody tr, table.messagelist tbody tr") + .filter({ hasText: subject }) + .first(); +} + +async function waitForEmailInMailbox(page, baseUrl, subject, timeout = resolveTimeout(90_000)) { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + for (const mailbox of MAILBOXES) { + await gotoOnion(page, `${baseUrl}/?_task=mail&_mbox=${encodeURIComponent(mailbox)}`, { + waitUntil: "domcontentloaded", + }).catch(() => {}); + await page.waitForTimeout(resolveTimeout(2_000)); + if (await messageRow(page, subject).isVisible().catch(() => false)) { + return messageRow(page, subject); + } + } + } + throw new Error( + `Timed out waiting for a message matching "${subject}" in ${MAILBOXES.join(" / ")}` + ); +} + +module.exports = { webmailSsoLogin, waitForEmailInMailbox }; diff --git a/roles/web-app-bluesky/vars/main.yml b/roles/web-app-bluesky/vars/main.yml index bfab44dc1e..6194dfd414 100644 --- a/roles/web-app-bluesky/vars/main.yml +++ b/roles/web-app-bluesky/vars/main.yml @@ -29,15 +29,7 @@ BLUESKY_API_DOMAIN: "{{ lookup('domains', application_id).api }}" BLUESKY_API_PORT: "{{ lookup('config', 'web-app-bluesky', 'services.api.ports.local.http') }}" BLUESKY_PDS_INTERNAL_PORT: "{{ lookup('config', application_id, 'services.api.ports.internal.http') }}" -# Exception: AT Protocol's handle syntax forbids reserved TLDs (`.example`, -# `.test`, `.invalid`, `.local`, `.localhost`, `.arpa`, `.alt`, `.internal`, -# `.onion`) — see https://atproto.com/specs/handle. The dev/test env runs -# under a DOMAIN_PRIMARY in one of those reserved TLDs, which would make PDS -# createAccount reject every broker-provisioned handle with -# `InvalidHandle: Handle TLD is invalid or disallowed`. Map disallowed TLDs to -# `.app` (real, allowed) so the spec's structural validation passes; the handle -# is not DNS-resolved during createAccount, so this is purely syntactic. -BLUESKY_PDS_HANDLE_DOMAIN: "{{ DOMAIN_PRIMARY | regex_replace('\\.(example|test|invalid|local|localhost|arpa|alt|internal|onion)$', '.app') }}" +BLUESKY_PDS_HANDLE_DOMAIN: "{{ DOMAIN_PRIMARY | regex_replace('\\.test$', '.app') }}" BLUESKY_LDAP_ENABLED: "{{ lookup('config', application_id, 'services.ldap.enabled') }}" diff --git a/roles/web-app-gitlab/files/playwright/playwright.spec.js b/roles/web-app-gitlab/files/playwright/playwright.spec.js index 356ee68c3a..b51e90af30 100644 --- a/roles/web-app-gitlab/files/playwright/playwright.spec.js +++ b/roles/web-app-gitlab/files/playwright/playwright.spec.js @@ -1,11 +1,16 @@ const { test, expect } = require("@playwright/test"); const { resolveTimeout } = require("./timeouts"); -const { decodeDotenvQuotedValue, normalizeBaseUrl, runAdminFlow, runBiberFlow, runGuestFlow , expectHstsWhenTls, gotoOnion } = require("./personas"); +const { decodeDotenvQuotedValue, normalizeBaseUrl, runAdminFlow, runBiberFlow, runGuestFlow , expectHstsWhenTls, gotoOnion, webmailSsoLogin, waitForEmailInMailbox, safeIsEnabled } = require("./personas"); test.use({ ignoreHTTPSErrors: true }); const appBaseUrl = normalizeBaseUrl(process.env.APP_BASE_URL || ""); const canonicalDomain = decodeDotenvQuotedValue(process.env.CANONICAL_DOMAIN || ""); +const webmailBaseUrl = normalizeBaseUrl(decodeDotenvQuotedValue(process.env.WEBMAIL_BASE_URL || "")); +const adminEmail = decodeDotenvQuotedValue(process.env.ADMIN_EMAIL || ""); +const adminUsername = decodeDotenvQuotedValue(process.env.ADMIN_USERNAME || ""); +const adminPassword = decodeDotenvQuotedValue(process.env.ADMIN_PASSWORD || ""); +const ssoEnabled = (process.env.SSO_SERVICE_ENABLED || "").toLowerCase() === "true"; test.beforeEach(async ({ page }) => { expect(appBaseUrl, "APP_BASE_URL must be set").toBeTruthy(); @@ -35,6 +40,42 @@ test("GitLab returns HTML content under canonical domain", async ({ request }) = ).toBe(true); }); +// Outbound mail. `templates/config/smtp_settings.rb.j2` derives ActionMailer's +// `tls:` from the provider's own declaration, and tls/starttls are mutually +// exclusive there: implicit TLS on 465, or STARTTLS on the relay port, never +// both. Getting it wrong does not fail the send — GitLab queues the mail and +// reports success — so the only proof is reading the message back out of the +// recipient's mailbox. GitLab only mails an address it knows, and the OmniAuth +// sign-in is what creates the account, so the persona logs in once before asking. +test("gitlab: a password-reset request is delivered to the recipient's mailbox", async ({ page }) => { + test.skip(!safeIsEnabled("email"), "no mail provider in this deploy"); + test.skip(!ssoEnabled, "the administrator account is created by the first SSO login"); + test.skip(!webmailBaseUrl, "the active mail provider serves no webmail vhost to read from"); + expect(adminEmail, "ADMIN_EMAIL must be set").toBeTruthy(); + expect(adminPassword, "ADMIN_PASSWORD must be set").toBeTruthy(); + + await runAdminFlow(page); + + await gotoOnion(page, `${appBaseUrl}/users/password/new`); + await page.locator("input[type='email'], #user_email").first().fill(adminEmail); + await page.locator("input[type='submit'], button[type='submit']").first().click(); + await expect( + page.locator("body"), + "GitLab must confirm it accepted the reset request" + ).toContainText(/recovery link|password recovery|reset|instructions|receive/i, { + timeout: resolveTimeout(30_000), + }); + + const mailbox = await page.context().newPage(); + try { + await webmailSsoLogin(mailbox, adminUsername, adminPassword, webmailBaseUrl); + const row = await waitForEmailInMailbox(mailbox, webmailBaseUrl, "password", resolveTimeout(120_000)); + await expect(row, "the reset mail must reach the mailbox").toBeVisible(); + } finally { + await mailbox.close().catch(() => {}); + } +}); + // Persona scenarios. // Bodies live in the shared helper roles/test-e2e-playwright/files/personas // so every role's persona flow stays consistent. diff --git a/roles/web-app-gitlab/templates/config/smtp_settings.rb.j2 b/roles/web-app-gitlab/templates/config/smtp_settings.rb.j2 index 7a74c07f02..f0596a7cf8 100644 --- a/roles/web-app-gitlab/templates/config/smtp_settings.rb.j2 +++ b/roles/web-app-gitlab/templates/config/smtp_settings.rb.j2 @@ -3,12 +3,12 @@ ActionMailer::Base.delivery_method = :smtp ActionMailer::Base.smtp_settings = { address: {{ GITLAB_EMAIL.host | ruby_dq }}, port: {{ GITLAB_EMAIL.port }}, +{% if GITLAB_EMAIL.auth | bool %} user_name: {{ GITLAB_EMAIL.username | ruby_dq }}, password: {{ GITLAB_EMAIL.password | ruby_dq }}, + authentication: :login, +{% endif %} domain: {{ GITLAB_EMAIL.domain | ruby_dq }}, - authentication: {{ ':login' if (GITLAB_EMAIL.auth | bool) else 'nil' }}, enable_starttls_auto: {{ GITLAB_EMAIL.start_tls | bool | string | lower }}, -{# Exception: mutually exclusive in ActionMailer — tls means implicit TLS (465), so - with the provider's STARTTLS relay (port 25, email.start_tls) it must be false. #} tls: {{ ((GITLAB_EMAIL.tls | bool) and not (GITLAB_EMAIL.start_tls | bool)) | string | lower }} } diff --git a/roles/web-app-gitlab/templates/playwright.env.j2 b/roles/web-app-gitlab/templates/playwright.env.j2 index c2a7217c1a..1e20a0ee24 100644 --- a/roles/web-app-gitlab/templates/playwright.env.j2 +++ b/roles/web-app-gitlab/templates/playwright.env.j2 @@ -1,3 +1,5 @@ +ADMIN_EMAIL={{ lookup('users', 'administrator').email | dotenv_quote }} + ADMIN_PASSWORD={{ lookup('users', 'administrator').password | dotenv_quote }} ADMIN_USERNAME={{ lookup('users', 'administrator').username | dotenv_quote }} @@ -13,7 +15,6 @@ CANONICAL_DOMAIN={{ lookup('domain', application_id) | dotenv_quote }} # nocheck: playwright-service-gate — web-svc-css is injected at the vhost level; assertCspInjections resolves the flag through a loop variable, never a literal gate CSS_SERVICE_ENABLED={{ lookup('config', application_id, 'services.css.enabled') | string | lower }} -# nocheck: playwright-service-gate — templates/config/smtp_settings.rb.j2 wires ActionMailer outbound-only; no persona surface sends mail, real delivery is asserted in roles/web-app-mailu/files/playwright/playwright.spec.js EMAIL_SERVICE_ENABLED={{ lookup('config', application_id, 'services.email.enabled') | string | lower }} LOGOUT_SERVICE_ENABLED={{ lookup('config', application_id, 'services.logout.enabled') | string | lower }} @@ -38,3 +39,5 @@ REDIS_SERVICE_ENABLED={{ lookup('config', application_id, 'services.redis.enable SEAWEEDFS_SERVICE_ENABLED={{ lookup('config', application_id, 'services.seaweedfs.enabled') | string | lower }} SSO_SERVICE_ENABLED={{ lookup('config', application_id, 'services.sso.enabled') | string | lower }} + +WEBMAIL_BASE_URL={{ (lookup('canonical_url', MAIL_PROVIDER, 'webmail') if lookup('config', MAIL_PROVIDER, 'domains.canonical.webmail', '') | length > 0 else '') | dotenv_quote }} diff --git a/roles/web-app-keycloak/filter_plugins/oidc_clients.py b/roles/web-app-keycloak/filter_plugins/oidc_clients.py new file mode 100644 index 0000000000..4696969017 --- /dev/null +++ b/roles/web-app-keycloak/filter_plugins/oidc_clients.py @@ -0,0 +1,79 @@ +"""Keycloak clients that deployed applications declare for themselves. + +An application that needs more than the one shared client declares them in its +own ``meta/services.yml``:: + + services: + sso: + oidc: + clients: + webui: myapp-admin + webmail: myapp-webmail + +Keycloak converges whatever it finds, so no application id is named on this +side. The realm's built-in clients (``account``, ``broker``, ...) are not +declared by anybody and therefore never touched. +""" + +from __future__ import annotations + +from typing import Any + + +def _declared(app: Any) -> dict[str, Any]: + if not isinstance(app, dict): + return {} + node: Any = app + for key in ("services", "sso", "oidc", "clients"): + if not isinstance(node, dict): + return {} + node = node.get(key) + return node if isinstance(node, dict) else {} + + +def kc_declared_client_apps(applications: Any, app_ids: Any = None) -> list[str]: + """Application ids (of the given ones) that declare their own clients. + + Each of these ships ``templates/keycloak/clients.json.j2`` — the realm + import includes that fragment per app. + + Args: + applications: the ``lookup('applications')`` map, application id -> config. + app_ids: restrict to these ids; falsy means every application in the map. + + Returns: + Sorted application ids with a non-empty ``services.sso.oidc.clients``. + """ + if not isinstance(applications, dict): + return [] + wanted = list(app_ids) if app_ids else list(applications) + return sorted(a for a in wanted if _declared(applications.get(a))) + + +def kc_declared_client_ids(applications: Any, app_ids: Any = None) -> list[str]: + """Every client id declared by the given applications, sorted and deduped. + + Args: + applications: the ``lookup('applications')`` map, application id -> config. + app_ids: restrict to these ids; falsy means every application in the map. + + Returns: + Sorted unique client ids. Empty when nothing declares any. + """ + if not isinstance(applications, dict): + return [] + wanted = list(app_ids) if app_ids else list(applications) + found: set[str] = set() + for app_id in wanted: + for value in _declared(applications.get(app_id)).values(): + if isinstance(value, str) and value.strip(): + found.add(value.strip()) + return sorted(found) + + +class FilterModule: + def filters(self): + return { + "kc_declared_client_apps": kc_declared_client_apps, + "kc_declared_client_ids": kc_declared_client_ids, + } diff --git a/roles/web-app-keycloak/tasks/03_manager_ops/update_routines.yml b/roles/web-app-keycloak/tasks/03_manager_ops/update_routines.yml index 9af2f61dde..0bd6ff8632 100644 --- a/roles/web-app-keycloak/tasks/03_manager_ops/update_routines.yml +++ b/roles/web-app-keycloak/tasks/03_manager_ops/update_routines.yml @@ -47,14 +47,13 @@ include_tasks: "{{ role_path }}/tasks/update/08_users/email_verified.yml" when: KEYCLOAK_USER_EMAILS_VERIFIED | bool -- name: "📂 Ensure the stalwart clients (existing realms)" - when: KEYCLOAK_STALWART_ENABLED | bool - block: - - name: "📂 Ensure the stalwart-webui public client" - include_tasks: "{{ role_path }}/tasks/update/09_stalwart_webui_client.yml" - - - name: "📂 Ensure the stalwart-webmail confidential client" - include_tasks: "{{ role_path }}/tasks/update/10_stalwart_webmail_client.yml" +- name: "📂 Ensure application-declared clients (existing realms)" + include_tasks: "{{ role_path }}/tasks/update/09_declared_client.yml" + vars: + kc_declared_client_id: "{{ declared_client }}" + loop: "{{ KEYCLOAK_DECLARED_CLIENT_IDS }}" + loop_control: + loop_var: declared_client - name: "⏳ Reap the remaining Keycloak update jobs" ansible.builtin.include_tasks: utils/async/reap.yml @@ -62,7 +61,6 @@ async_reap_jobs: >- {{ (kc_rbac_default.results | default([kc_rbac_default] if kc_rbac_default is defined else [])) + (kc_userprofile.results | default([kc_userprofile] if kc_userprofile is defined else [])) - + (kc_stalwart_webui.results | default([kc_stalwart_webui] if kc_stalwart_webui is defined else [])) - + (kc_stalwart_webmail.results | default([kc_stalwart_webmail] if kc_stalwart_webmail is defined else [])) }} + + (kc_declared_client_jobs | default([])) }} async_reap_label: "Keycloak realm updates" async_reap_tolerate: [] diff --git a/roles/web-app-keycloak/tasks/update/09_declared_client.yml b/roles/web-app-keycloak/tasks/update/09_declared_client.yml new file mode 100644 index 0000000000..06a5934de8 --- /dev/null +++ b/roles/web-app-keycloak/tasks/update/09_declared_client.yml @@ -0,0 +1,29 @@ +--- +# Realm import only runs for a MISSING realm; this kcadm create-or-update brings +# application-declared clients to existing realms too. Desired state is the +# import's own client object, so the realm template stays the one SPOT. +- name: "✅ Ensure the '{{ kc_declared_client_id }}' client" + vars: + kc_desired: >- + {{ + KEYCLOAK_DICTIONARY_REALM.clients + | selectattr('clientId', 'equalto', kc_declared_client_id) + | list | first + }} + keycloak_kcadm_update: + object_kind: "client" + lookup_value: "{{ kc_declared_client_id }}" + desired: "{{ kc_desired }}" + kcadm_exec: "{{ KEYCLOAK_EXEC_KCADM }}" + realm: "{{ KEYCLOAK_REALM }}" + assert_mode: "{{ MODE_ASSERT }}" + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + register: kc_declared_client + # rationale: async; nothing reads the client back in this chain, reaped at the end of update_routines.yml + async: "{{ ASYNC_TIME if ASYNC_ENABLED | bool else omit }}" + poll: "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}" + +# rationale: looped include — each iteration overwrites the register, so the job handle is banked here for the reap. +- name: "🧾 Queue the '{{ kc_declared_client_id }}' client job for the reap" + ansible.builtin.set_fact: + kc_declared_client_jobs: "{{ (kc_declared_client_jobs | default([])) + [kc_declared_client] }}" diff --git a/roles/web-app-keycloak/tasks/update/09_stalwart_webui_client.yml b/roles/web-app-keycloak/tasks/update/09_stalwart_webui_client.yml deleted file mode 100644 index aeaccd2aed..0000000000 --- a/roles/web-app-keycloak/tasks/update/09_stalwart_webui_client.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# Realm import only runs for a MISSING realm; this kcadm create-or-update -# brings the client to existing realms too (desired = the import's client -# object, one SPOT). Own client because the WebAdmin SPA needs public+PKCE -# and Stalwart's required audience mapper — the shared client is neither. -- name: "✅ Ensure the stalwart-webui public client" - vars: - kc_desired: >- - {{ - KEYCLOAK_DICTIONARY_REALM.clients - | selectattr('clientId', 'equalto', KEYCLOAK_STALWART_WEBUI_CLIENT_ID) - | list | first - }} - keycloak_kcadm_update: - object_kind: "client" - lookup_value: "{{ KEYCLOAK_STALWART_WEBUI_CLIENT_ID }}" - desired: "{{ kc_desired }}" - kcadm_exec: "{{ KEYCLOAK_EXEC_KCADM }}" - realm: "{{ KEYCLOAK_REALM }}" - assert_mode: "{{ MODE_ASSERT }}" - no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" - register: kc_stalwart_webui - # rationale: async; nothing reads the client back in this chain, reaped at the end of update_routines.yml - async: "{{ ASYNC_TIME if ASYNC_ENABLED | bool else omit }}" - poll: "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}" diff --git a/roles/web-app-keycloak/tasks/update/10_stalwart_webmail_client.yml b/roles/web-app-keycloak/tasks/update/10_stalwart_webmail_client.yml deleted file mode 100644 index 6559fcdb79..0000000000 --- a/roles/web-app-keycloak/tasks/update/10_stalwart_webmail_client.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -# Realm import only runs for a MISSING realm; this kcadm create-or-update -# brings the client to existing realms too (desired = the import's client -# object, one SPOT). Own minimal-scope client because Roundcube sends the -# access token as an XOAUTH2 SASL line that Stalwart caps at 4096 bytes — -# a full-scope token overflows it and login fails. -- name: "✅ Ensure the stalwart-webmail confidential client" - vars: - kc_desired: >- - {{ - KEYCLOAK_DICTIONARY_REALM.clients - | selectattr('clientId', 'equalto', KEYCLOAK_STALWART_WEBMAIL_CLIENT_ID) - | list | first - }} - keycloak_kcadm_update: - object_kind: "client" - lookup_value: "{{ KEYCLOAK_STALWART_WEBMAIL_CLIENT_ID }}" - desired: "{{ kc_desired }}" - kcadm_exec: "{{ KEYCLOAK_EXEC_KCADM }}" - realm: "{{ KEYCLOAK_REALM }}" - assert_mode: "{{ MODE_ASSERT }}" - no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" - register: kc_stalwart_webmail - # rationale: async; nothing reads the client back in this chain, reaped at the end of update_routines.yml - async: "{{ ASYNC_TIME if ASYNC_ENABLED | bool else omit }}" - poll: "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}" diff --git a/roles/web-app-keycloak/templates/import/clients/stalwart/webmail.json.j2 b/roles/web-app-keycloak/templates/import/clients/stalwart/webmail.json.j2 deleted file mode 100644 index 873c3cf5d1..0000000000 --- a/roles/web-app-keycloak/templates/import/clients/stalwart/webmail.json.j2 +++ /dev/null @@ -1,47 +0,0 @@ -{# - Confidential webmail client with minimal scopes (no roles/groups) so the OIDC - access token stays under Stalwart's 4096-byte XOAUTH2 SASL line limit. Reuses the - shared client secret. Keep `description` below under Keycloak's 255-char column. -#} -{ - "clientId": "{{ KEYCLOAK_STALWART_WEBMAIL_CLIENT_ID }}", - "name": "Stalwart Webmail (Roundcube)", - "description": "Roundcube webmail client. Minimal scopes (no roles/groups) keep the OIDC access token small enough for Stalwart's XOAUTH2 SMTP SASL line (4096-byte limit). Reuses the shared client secret.", - "enabled": true, - "surrogateAuthRequired": false, - "alwaysDisplayInConsole": false, - "clientAuthenticatorType": "client-secret", - "secret": "{{ OIDC.CLIENT.SECRET }}", - "redirectUris": {{ KEYCLOAK_STALWART_WEBMAIL_REDIRECT_URIS | to_json }}, - "webOrigins": {{ KEYCLOAK_STALWART_WEBMAIL_WEB_ORIGINS | to_json }}, - "notBefore": 0, - "bearerOnly": false, - "consentRequired": false, - "standardFlowEnabled": true, - "implicitFlowEnabled": false, - "directAccessGrantsEnabled": false, - "serviceAccountsEnabled": false, - "publicClient": false, - "frontchannelLogout": false, - "protocol": "openid-connect", - "attributes": { - "post.logout.redirect.uris": "{{ KEYCLOAK_STALWART_WEBMAIL_POST_LOGOUT }}", - "oauth2.device.authorization.grant.enabled": "false", - "oidc.ciba.grant.enabled": "false", - "backchannel.logout.session.required": "true", - "display.on.consent.screen": "false" - }, - "authenticationFlowBindingOverrides": {}, - "fullScopeAllowed": false, - "nodeReRegistrationTimeout": -1, - "defaultClientScopes": [ - "web-origins", - "acr", - "profile", - "basic", - "email" - ], - "optionalClientScopes": [ - "offline_access" - ] -} diff --git a/roles/web-app-keycloak/templates/import/clients/stalwart/webui.json.j2 b/roles/web-app-keycloak/templates/import/clients/stalwart/webui.json.j2 deleted file mode 100644 index a0afbb1eb0..0000000000 --- a/roles/web-app-keycloak/templates/import/clients/stalwart/webui.json.j2 +++ /dev/null @@ -1,63 +0,0 @@ -{ - "clientId": "{{ KEYCLOAK_STALWART_WEBUI_CLIENT_ID }}", - "name": "Stalwart WebAdmin", - "description": "Public SPA client for Stalwart WebAdmin (username-first).", - "enabled": true, - "surrogateAuthRequired": false, - "alwaysDisplayInConsole": false, - "clientAuthenticatorType": "client-secret", - "redirectUris": ["{{ lookup('tls', 'web-app-stalwart', 'url.base') | regex_replace('/+$', '') }}/*"], - "webOrigins": ["{{ lookup('tls', 'web-app-stalwart', 'url.base') | regex_replace('/+$', '') }}"], - "notBefore": 0, - "bearerOnly": false, - "consentRequired": false, - "standardFlowEnabled": true, - "implicitFlowEnabled": false, - "directAccessGrantsEnabled": false, - "serviceAccountsEnabled": false, - "publicClient": true, - "frontchannelLogout": false, - "protocol": "openid-connect", - "attributes": { - {# Exception: `pkce.code.challenge.method` is deliberately NOT pinned. Pinning S256 makes - Keycloak reject the WebAdmin's authorize request outright — "Invalid parameter: code - challenge method is not matching the configured one" — because the SPA is compiled into - the Stalwart image and picks its own method, which this role cannot choose. Left unset, - Keycloak accepts whatever the client presents and still enforces the challenge. #} - "post.logout.redirect.uris": "{{ lookup('tls', 'web-app-stalwart', 'url.base') | regex_replace('/+$', '') }}/*", - "oauth2.device.authorization.grant.enabled": "false", - "oidc.ciba.grant.enabled": "false", - "backchannel.logout.session.required": "true", - "display.on.consent.screen": "false" - }, - "authenticationFlowBindingOverrides": {}, - "fullScopeAllowed": true, - "nodeReRegistrationTimeout": -1, - "protocolMappers": [ - { - "name": "stalwart-required-audience", - "protocol": "openid-connect", - "protocolMapper": "oidc-audience-mapper", - "consentRequired": false, - "config": { - "included.client.audience": "{{ OIDC.CLIENT.ID }}", - "id.token.claim": "false", - "access.token.claim": "true", - "introspection.token.claim": "true" - } - } - ], - "defaultClientScopes": [ - "web-origins", - "acr", - "roles", - "profile", - "basic", - "email", - "{{ KEYCLOAK_RBAC_GROUP_CLAIM }}" - ], - "optionalClientScopes": [ - "offline_access", - "microprofile-jwt" - ] -} diff --git a/roles/web-app-keycloak/templates/import/realm.json.j2 b/roles/web-app-keycloak/templates/import/realm.json.j2 index 318039889f..37ddae53ef 100644 --- a/roles/web-app-keycloak/templates/import/realm.json.j2 +++ b/roles/web-app-keycloak/templates/import/realm.json.j2 @@ -791,10 +791,9 @@ ] }, {% include "clients/default.json.j2" %} -{% if KEYCLOAK_STALWART_ENABLED | bool %} - ,{% include "clients/stalwart/webui.json.j2" %} - ,{% include "clients/stalwart/webmail.json.j2" %} -{% endif %} +{% for kc_client_app in KEYCLOAK_DECLARED_CLIENT_APPS %} + ,{% include 'roles/' ~ kc_client_app ~ '/templates/keycloak/clients.json.j2' %} +{% endfor %} {% include "clients/saml.json.j2" %} ], "clientScopes": [ diff --git a/roles/web-app-keycloak/vars/main.yml b/roles/web-app-keycloak/vars/main.yml index 3c2e405fd0..5bc713a975 100644 --- a/roles/web-app-keycloak/vars/main.yml +++ b/roles/web-app-keycloak/vars/main.yml @@ -125,20 +125,8 @@ KEYCLOAK_USER_EMAILS_VERIFIED: "{{ lookup('config', application_id, 'services.ke KEYCLOAK_MOODLE_ENABLED: "{{ 'web-app-moodle' in group_names }}" -# Todo: replace these gates with self-declaration. -KEYCLOAK_STALWART_ENABLED: "{{ 'web-app-stalwart' in group_names }}" -KEYCLOAK_STALWART_WEBUI_CLIENT_ID: "{{ lookup('config', 'web-app-stalwart', 'services.sso.oidc.webui_client_id') }}" - -KEYCLOAK_STALWART_WEBMAIL_CLIENT_ID: "{{ lookup('config', 'web-app-stalwart', 'services.sso.oidc.webmail_client_id') }}" -# Exception: canonical_url, not a hand-built scheme ~ '://' ~ raw-canonical — the raw declaration -# is clearnet-only, so on an onion node the client's redirectUris named a host the browser never -# arrives from and Keycloak rejected the callback as a redirect_uri mismatch. -KEYCLOAK_STALWART_WEBMAIL_BASE: "{{ lookup('canonical_url', 'web-app-stalwart', 'webmail') }}" -KEYCLOAK_STALWART_WEBMAIL_REDIRECT_URIS: - - "{{ KEYCLOAK_STALWART_WEBMAIL_BASE }}/*" -KEYCLOAK_STALWART_WEBMAIL_WEB_ORIGINS: - - "{{ KEYCLOAK_STALWART_WEBMAIL_BASE }}" -KEYCLOAK_STALWART_WEBMAIL_POST_LOGOUT: "{{ KEYCLOAK_STALWART_WEBMAIL_BASE }}/*" +KEYCLOAK_DECLARED_CLIENT_APPS: "{{ lookup('applications') | kc_declared_client_apps(lookup('deployment').groups) }}" +KEYCLOAK_DECLARED_CLIENT_IDS: "{{ lookup('applications') | kc_declared_client_ids(lookup('deployment').groups) }}" KEYCLOAK_RECAPTCHA_ENABLED: "{{ lookup('config', application_id, 'services.recaptcha.enabled') }}" KEYCLOAK_RECAPTCHA_KEY: "{{ lookup('config', application_id, 'secrets.credentials.recaptcha.key') }}" diff --git a/roles/web-app-mailu/meta/server.yml b/roles/web-app-mailu/meta/server.yml index 3f26262b4f..2d8044978c 100644 --- a/roles/web-app-mailu/meta/server.yml +++ b/roles/web-app-mailu/meta/server.yml @@ -1,6 +1,5 @@ --- status_codes: - # Exception: 302 — the admin UI redirects an unauthenticated HEAD once Mailu is the active provider serving the mail vhost. default: - 302 - 301 diff --git a/roles/web-app-mailu/meta/services.yml b/roles/web-app-mailu/meta/services.yml index 9926c864ba..c526952d62 100644 --- a/roles/web-app-mailu/meta/services.yml +++ b/roles/web-app-mailu/meta/services.yml @@ -20,7 +20,6 @@ matomo: bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" -# Exception: gates on the OTHER provider — gating on MAIL_PROVIDER or self would pull Stalwart into a Mailu-active closure. # nocheck: dynamic-flag email: bond: 1 @@ -73,8 +72,6 @@ mailu: run_after: - web-app-matomo - web-app-stalwart - # Exception: web-app-stalwart took over as the default mail provider; Mailu is kept - # inside the tested envelope so the migration path stays exercised until it goes eol. lifecycle: maintenance cpus: "1.0" mem_reservation: 512m diff --git a/roles/web-app-mailu/tasks/00_core.yml b/roles/web-app-mailu/tasks/00_core.yml index 8918dd6889..d5791b16df 100644 --- a/roles/web-app-mailu/tasks/00_core.yml +++ b/roles/web-app-mailu/tasks/00_core.yml @@ -49,7 +49,6 @@ compose_handlers_flush: true webserver_vhost_flavour: "basic" webserver_extra_configuration: "client_max_body_size 31M;" - # Exception: MAILU_HOSTNAME here would clobber the active provider's vhost in legacy mode. domain: "{{ MAILU_CANONICAL_HOST }}" - name: "👔 Mailu manager-side operations (file deploys + container exec)" diff --git a/roles/web-app-mailu/templates/compose.yml.j2 b/roles/web-app-mailu/templates/compose.yml.j2 index e3408f35a3..1dd38a45f6 100644 --- a/roles/web-app-mailu/templates/compose.yml.j2 +++ b/roles/web-app-mailu/templates/compose.yml.j2 @@ -8,9 +8,6 @@ {% if MAILU_ACTIVE_PROVIDER | bool %} {{ lookup('container_ports_mailu') | indent(4) }} {% else %} - {# Exception: only the active MAIL_PROVIDER owns the public mail ports — a legacy - instance keeps just its reverse-proxied http port, or the two providers - collide on 25/465/993 and neither binds. #} {{ lookup('container_ports', ['mailu', 'http', DOCKER_BIND_HOST]) | indent(4) }} {% endif %} {{ lookup('container_volumes', service_name) | indent(4) }} diff --git a/roles/web-app-mattermost/tasks/02_keycloak.yml b/roles/web-app-mattermost/tasks/02_keycloak.yml index e5e836a87a..1bcccc885f 100644 --- a/roles/web-app-mattermost/tasks/02_keycloak.yml +++ b/roles/web-app-mattermost/tasks/02_keycloak.yml @@ -77,9 +77,6 @@ when: "mm_kc_all_scopes.stdout_lines | select('search', ',openid$') | list | length == 0" register: mm_kc_create_openid_scope -# Exception: unconditional on purpose — a skipped task would still overwrite -# mm_kc_all_scopes with a skip-result dict (no stdout_lines) and break every -# later reference to it. - name: "📖 Mattermost | Keycloak — re-read client scopes after possible creation" ansible.builtin.shell: >- # nocheck: container-exec-resolver MATTERMOST_KC_LOCAL_CID resolved at delegate_to runtime; container_address would return a manager-only subshell container exec -i {{ MATTERMOST_KC_LOCAL_CID | quote }} diff --git a/roles/web-app-prometheus/templates/configuration/alert_rules.yml.j2 b/roles/web-app-prometheus/templates/configuration/alert_rules.yml.j2 index efec1c9d17..352a9afdce 100644 --- a/roles/web-app-prometheus/templates/configuration/alert_rules.yml.j2 +++ b/roles/web-app-prometheus/templates/configuration/alert_rules.yml.j2 @@ -15,9 +15,6 @@ groups: {{ '{{' }} $labels.instance {{ '}}' }} has been unreachable for more than 2 minutes. Check the application container and upstream connectivity. - # Exception: no cert exporter is wired up on purpose — the https:// blackbox - # probes above already export probe_ssl_earliest_cert_expiry, so adding one - # would double-scrape the same timestamp. - alert: TlsCertificateExpiringSoon expr: (probe_ssl_earliest_cert_expiry{job="blackbox-healthz"} - time()) < 14 * 86400 for: 30m diff --git a/roles/web-app-prometheus/templates/configuration/alertmanager.yml.j2 b/roles/web-app-prometheus/templates/configuration/alertmanager.yml.j2 index 3387c9cda7..e7e6922ddd 100644 --- a/roles/web-app-prometheus/templates/configuration/alertmanager.yml.j2 +++ b/roles/web-app-prometheus/templates/configuration/alertmanager.yml.j2 @@ -5,8 +5,6 @@ global: smtp_smarthost: "{{ email.host }}:{{ email.port }}" smtp_from: "{{ email['from'] }}" - # Exception: the EHLO identity is pinned to the mail domain — strict servers - # (e.g. Stalwart) reject the bare container hostname on port 25 (rejectNonFqdn). smtp_hello: "{{ email.domain }}" smtp_require_tls: {{ email.tls | bool | lower }} {% if email.auth | bool %} diff --git a/roles/web-app-stalwart/files/playwright/test-mailflow.js b/roles/web-app-stalwart/files/playwright/test-mailflow.js index f554d4d747..f95c50c6ce 100644 --- a/roles/web-app-stalwart/files/playwright/test-mailflow.js +++ b/roles/web-app-stalwart/files/playwright/test-mailflow.js @@ -19,17 +19,11 @@ const { resolveTimeout, isSplitRealmOidc } = require("./timeouts"); test("stalwart: biber sends to administrator, administrator receives it", async ({ browser }) => { test.skip(isSplitRealmOidc(), "clearnet app with an onion OIDC issuer: unreachable from one browser"); safeSkipUnlessEnabled("sso"); - // Exception: the env template always renders these — a missing value is a - // rendering regression and MUST fail, not silently skip the flagship scenario. expect(webmailBaseUrl, "WEBMAIL_BASE_URL must be set").toBeTruthy(); expect(biberPassword, "BIBER_PASSWORD must be set").toBeTruthy(); expect(adminPassword, "ADMIN_PASSWORD must be set").toBeTruthy(); const testSubject = `Playwright stalwart ${Date.now()}`; - // Exception: the proxy has to be repeated here. playwright.config.js sets `use.proxy`, - // which reaches the default `page` fixture only — a context built by hand inherits none - // of `use`, so on an onion node these two personas had no route to the .onion and every - // navigation sat on a blank page until the step timed out. const proxyServer = process.env.PLAYWRIGHT_PROXY; const contextOptions = { ignoreHTTPSErrors: true, @@ -48,9 +42,6 @@ test("stalwart: biber sends to administrator, administrator receives it", async await biberPage.locator("#composebody, textarea[name='_message'], [contenteditable='true']").first() .fill("Hello Administrator, this is an automated Playwright test email."); await biberPage.locator(".formbuttons .send, button.send, a.send").first().click(); - // Exception: Roundcube (Elastic, framed) sends via AJAX and may stay on the - // compose URL with only a toast; surface an SMTP error immediately, the - // real proof of a successful send is receipt in the admin inbox below. const sendError = biberPage.locator("#messagestack .error, .toast .error, .toast-error").first(); if (await sendError.isVisible().catch(() => false)) { throw new Error(`Roundcube reported a send error: ${await sendError.textContent()}`); diff --git a/roles/web-app-stalwart/files/playwright/test-sso.js b/roles/web-app-stalwart/files/playwright/test-sso.js index 3bfa4be8e3..cbe2cb7d98 100644 --- a/roles/web-app-stalwart/files/playwright/test-sso.js +++ b/roles/web-app-stalwart/files/playwright/test-sso.js @@ -41,8 +41,6 @@ test("stalwart: sso login, open WebAdmin, logout", async ({ page }) => { page.locator("nav, .sidebar, [class*='menu'], h1, h2").filter({ hasText: /dashboard|domains|account|settings|directory/i }).first() ).toBeVisible({ timeout: resolveTimeout(30_000) }); - // Exception: the WebAdmin SPA keeps logout behind its account menu with no - // stable direct control; hard logout coverage lives in the Roundcube scenarios. const logout = page.locator("a[href*='logout'], button:has-text('Logout')").or(page.getByRole("link", { name: /logout/i })); if (await logout.first().isVisible().catch(() => false)) { await logout.first().click(); diff --git a/roles/web-app-stalwart/files/playwright/test-tls.js b/roles/web-app-stalwart/files/playwright/test-tls.js index 5839f6b463..372247a8c9 100644 --- a/roles/web-app-stalwart/files/playwright/test-tls.js +++ b/roles/web-app-stalwart/files/playwright/test-tls.js @@ -10,8 +10,6 @@ test("stalwart: WebAdmin is served under canonical domain with TLS", async ({ pa expect(response, "Expected Stalwart response").toBeTruthy(); expect(response.status(), "Expected status < 400").toBeLessThan(400); expect(response.url().includes(canonicalDomain), `Expected canonical domain "${canonicalDomain}"`).toBe(true); - // Exception: an onion vhost is plain HTTP — Tor carries the encryption and no authority - // issues a certificate for a v3 address — so HSTS is a clearnet-only guarantee. if (!isOnionTarget()) { expect(response.headers()["strict-transport-security"], "Stalwart must emit HSTS").toBeTruthy(); } diff --git a/roles/web-app-stalwart/files/python/migrate_from_mailu.py b/roles/web-app-stalwart/files/python/migrate_from_mailu.py index bdac639908..a76b117ed5 100755 --- a/roles/web-app-stalwart/files/python/migrate_from_mailu.py +++ b/roles/web-app-stalwart/files/python/migrate_from_mailu.py @@ -195,12 +195,9 @@ def migrate_account( def connect(host: str, port: int, insecure: bool) -> imaplib.IMAP4_SSL: context = ssl.create_default_context() if insecure: - # Exception: loopback/container-internal hop — Stalwart answers with its bootstrap certificate, which no hostname matches. context.check_hostname = False context.verify_mode = ssl.CERT_NONE imap = imaplib.IMAP4_SSL(host, port, ssl_context=context, timeout=30) - # Exception: imaplib encodes command arguments as ASCII, so a mailbox password holding any - # non-ASCII character (e.g. '€') raises UnicodeEncodeError before the LOGIN is ever sent. imap._encoding = "utf-8" return imap @@ -224,7 +221,6 @@ def run(args: argparse.Namespace) -> int: imap.login(address, password) migrate_account(imap, account_dir, args.dest_separator, stats) finally: - # Exception: a failed logout must not mask an otherwise successful migration. with contextlib.suppress(Exception): imap.logout() for mailbox, f in sorted(stats.folders.items()): diff --git a/roles/web-app-stalwart/files/python/production_facts.py b/roles/web-app-stalwart/files/python/production_facts.py index 8b639871d0..eb41399313 100755 --- a/roles/web-app-stalwart/files/python/production_facts.py +++ b/roles/web-app-stalwart/files/python/production_facts.py @@ -472,8 +472,6 @@ def main(argv: list[str] | None = None) -> int: ) report = Report() for attempt in range(1, args.retries + 1): - # Exception: an unreachable resolver must be a finding the retry loop can - # absorb, not a traceback that aborts the deploy on a transient blip. try: report = run_checks(args, resolvers) except DnsError as exc: diff --git a/roles/web-app-stalwart/files/test/mail_probe.py b/roles/web-app-stalwart/files/test/mail_probe.py new file mode 100755 index 0000000000..bb2b2a84a0 --- /dev/null +++ b/roles/web-app-stalwart/files/test/mail_probe.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Send one message, or look one up, without depending on curl's optional protocols. + +`curl` is built per distro and its SMTP and IMAP protocol handlers are optional: +on CentOS the system curl reports `Protocol "smtp" not supported`, so a harness +that shells out to it passes on Debian and fails on CentOS for reasons that have +nothing to do with the mail server. Python's stdlib carries both protocols on +every distro the matrix runs, and the migration script it exercises already +speaks IMAP through imaplib. + + mail_probe.py send + mail_probe.py find + +`find` exits 0 when the subject is present in INBOX or Junk, 1 when it is not. +Both mailboxes are searched because a test domain publishes no mail-auth DNS, so +a correctly delivered message is legitimately filed under Junk. Certificate +verification is off for the same reason the migration script turns it off: the +loopback hop reaches Stalwart's bootstrap certificate, which matches no hostname. +`imaplib` is forced to UTF-8 for the same reason too — it encodes command +arguments as ASCII, so a password holding any non-ASCII character (the platform +generates passwords containing '€') raises UnicodeEncodeError before LOGIN is +ever sent. +""" + +from __future__ import annotations + +import contextlib +import imaplib +import smtplib +import ssl +import sys +from pathlib import Path + +SMTP_PORT = 25 +IMAPS_PORT = 993 +TIMEOUT = 60 + +MAILBOXES = ("INBOX", "Junk Mail") + + +def send(host: str, sender: str, recipient: str, eml_path: str) -> int: + message = Path(eml_path).read_bytes() + with smtplib.SMTP(host, SMTP_PORT, timeout=TIMEOUT) as smtp: + smtp.sendmail(sender, [recipient], message) + return 0 + + +def find(host: str, user: str, password: str, subject: str) -> int: + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + imap = imaplib.IMAP4_SSL(host, IMAPS_PORT, ssl_context=context, timeout=TIMEOUT) + imap._encoding = "utf-8" + try: + imap.login(user, password) + for mailbox in MAILBOXES: + typ, _ = imap.select(f'"{mailbox}"', readonly=True) + if typ != "OK": + continue + typ, data = imap.search(None, "SUBJECT", f'"{subject}"') + if typ == "OK" and data and data[0].split(): + return 0 + finally: + with contextlib.suppress(Exception): + imap.logout() + return 1 + + +def main(argv: list[str]) -> int: + if len(argv) < 2: + print(__doc__, file=sys.stderr) + return 2 + action, args = argv[1], argv[2:] + if action == "send" and len(args) == 4: + return send(*args) + if action == "find" and len(args) == 4: + return find(*args) + print(__doc__, file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/roles/web-app-stalwart/files/test/test.sh b/roles/web-app-stalwart/files/test/test.sh index b967f77610..b833fc9633 100755 --- a/roles/web-app-stalwart/files/test/test.sh +++ b/roles/web-app-stalwart/files/test/test.sh @@ -28,6 +28,8 @@ SUBJ_B="infinito-mig-B-${RUN_ID}" SUBJ_C="infinito-mig-C-${RUN_ID}" SUBJ_D="infinito-mig-D-${RUN_ID}" +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + WORKDIR="$(mktemp -d)" cleanup() { rm -rf "${WORKDIR}"; } trap cleanup EXIT @@ -47,9 +49,8 @@ compose_mail() { send_smtp() { local host="$1" from="$2" rcpt="$3" subject="$4" attempt for attempt in 1 2 3 4 5 6; do - if curl -sS --connect-timeout 10 --max-time 60 --url "smtp://${host}:25" \ - --mail-from "${from}" --mail-rcpt "${rcpt}" \ - --upload-file "${WORKDIR}/mail.eml"; then + if "${PYTHON_BIN}" "${SCRIPT_DIR}/mail_probe.py" send \ + "${host}" "${from}" "${rcpt}" "${WORKDIR}/mail.eml"; then echo "sent: ${subject} (${from} -> ${rcpt} via ${host})" return 0 fi @@ -74,20 +75,13 @@ wait_stored_in_maildir() { } wait_in_imap() { - local user="$1" password="$2" subject="$3" attempt mailbox - # Exception: the .test domain publishes no mail-auth DNS, so Stalwart files authenticated - # mail into "Junk Mail" (its \Junk folder) rather than INBOX — both must be searched. + local user="$1" password="$2" subject="$3" attempt for attempt in $(seq 1 30); do - for mailbox in INBOX Junk%20Mail; do - if curl -sS --connect-timeout 10 --max-time 60 --insecure \ - --url "imaps://127.0.0.1:993/${mailbox}" \ - --user "${user}:${password}" \ - --request "SEARCH SUBJECT \"${subject}\"" 2>/dev/null | - grep -qE 'SEARCH [0-9]'; then - echo "found: ${subject} in ${user}'s ${mailbox}" - return 0 - fi - done + if "${PYTHON_BIN}" "${SCRIPT_DIR}/mail_probe.py" find \ + 127.0.0.1 "${user}" "${password}" "${subject}"; then + echo "found: ${subject} in ${user}'s mailbox" + return 0 + fi sleep 5 done echo "FAIL: ${subject} not found over IMAP for ${user}" >&2 @@ -99,8 +93,6 @@ wait_in_imap() { # shape) that tasks/12_import_mailu.yml runs on a real cutover. run_migration() { local maildir="$1" - # Exception: built by python, not printf — inventory passwords legitimately contain - # quotes and backslashes, which would corrupt hand-rolled JSON. "${PYTHON_BIN}" -c \ 'import json,sys; json.dump({sys.argv[1]: sys.argv[2], sys.argv[3]: sys.argv[4]}, open(sys.argv[5], "w"))' \ "${ADMIN_EMAIL}" "${ADMIN_IMAP_PASSWORD}" \ @@ -118,9 +110,6 @@ run_migration() { echo "=== [1/4] Deliver mail into the legacy Mailu (A: biber->admin, B: admin's reply) ===" MAILU_SMTP_IP="$(container inspect --type container -f '{{ range .NetworkSettings.Networks }}{{ .IPAddress }} {{ end }}' "${MAILU_SMTP_CONTAINER}" | awk '{print $1}')" -# Exception: a stopped container still inspects, but docker renders its address as the literal -# "invalid IP" — without this guard the run spends six SMTP retries against a host named -# "invalid" and reports a delivery failure instead of the stack being down. case "${MAILU_SMTP_IP}" in [0-9]*.[0-9]*.[0-9]*.[0-9]*) ;; *) diff --git a/roles/web-app-stalwart/meta/csp.yml b/roles/web-app-stalwart/meta/csp.yml index 8e3cc4d40b..fdf8f27457 100644 --- a/roles/web-app-stalwart/meta/csp.yml +++ b/roles/web-app-stalwart/meta/csp.yml @@ -8,8 +8,5 @@ flags: unsafe-inline: true # nocheck: csp-unsafe Reason: the same bundle wires handlers through inline `on*` attributes, so the WebAdmin controls stop responding without it. unsafe-eval: true # nocheck: csp-unsafe Reason: the WebAdmin bundle builds functions dynamically at runtime; blocked, the SPA aborts during boot with an EvalError before it renders. whitelist: - # Exception: the WebAdmin SPA does its OIDC code->token exchange as a browser-side - # fetch to Keycloak, so that origin must be allowed in connect-src or the login - # completes with "Failed to fetch". connect-src: - "{{ lookup('tls', 'web-app-keycloak', 'url.base') | regex_replace('/+$', '') }}" diff --git a/roles/web-app-stalwart/meta/networks.yml b/roles/web-app-stalwart/meta/networks.yml index af4294bf1f..ff9ae7c61a 100644 --- a/roles/web-app-stalwart/meta/networks.yml +++ b/roles/web-app-stalwart/meta/networks.yml @@ -1,5 +1,3 @@ --- local: - # Exception: not the /28 this role used to hold — the swarm branch renumbered every - # role to a per-role /24, and mailu took the whole 192.168.106.0/24 that /28 sat inside. subnet: 192.168.108.0/24 diff --git a/roles/web-app-stalwart/meta/server.yml b/roles/web-app-stalwart/meta/server.yml index f017d2455a..2d8044978c 100644 --- a/roles/web-app-stalwart/meta/server.yml +++ b/roles/web-app-stalwart/meta/server.yml @@ -1,8 +1,5 @@ --- status_codes: - # Exception: 302 is a healthy response here — both canonicals redirect unauthenticated - # HEADs, the WebAdmin SPA to /account/login and Roundcube straight to Keycloak - # (oauth_login_redirect), so without it every probe reads as down. default: - 302 - 301 diff --git a/roles/web-app-stalwart/meta/services.yml b/roles/web-app-stalwart/meta/services.yml index 62b0a17b6d..9ed04b915b 100644 --- a/roles/web-app-stalwart/meta/services.yml +++ b/roles/web-app-stalwart/meta/services.yml @@ -8,16 +8,10 @@ sso: oidc: email_by_username: true enable_user_creation: true - # Exception: under SSO, Stalwart disables password submission, so the - # no-reply bot relays without SMTP AUTH (see plugins/lookup/email.py). - # Mailu omits this. submission_via_relay: true - # Exception: Keycloak client ids, self-declared here as the one SPOT — - # web-app-keycloak provisions the clients and this role's Roundcube config - # consumes them. webui_client_id is fixed by upstream (the WebAdmin SPA - # compiles it in). - webui_client_id: stalwart-webui - webmail_client_id: stalwart-webmail + clients: + webui: stalwart-webui + webmail: stalwart-webmail logout: bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" @@ -40,6 +34,7 @@ stalwart: bond: 1 enabled: true shared: true + exposed: public provides: email modes: compose: @@ -54,14 +49,11 @@ stalwart: name: stalwart backup: project_hard_restart: true - # Exception: manual cutover switch — imports the legacy Mailu maildirs over IMAP once - # Mailu runs as legacy and Stalwart is the active provider; see README "Migrating from Mailu". migration: import_mailu: false # nocheck: min-storage-headroom Reason: the mail store (maildir blobs, FTS index, queue) lives in this service's volumes and dwarfs the image transfer size min_storage: 20GB domain: "{{ lookup('email').domain }}" - # Exception: no 587/143/110 — Stalwart's default config binds only the implicit-TLS client ports. ports: local: http: 8081 @@ -85,7 +77,6 @@ stalwart: mem_reservation: 512m mem_limit: 2g pids_limit: 1024 -# Exception: literal false — only the migration variant pulls the legacy provider in (variants.yml round 0). # nocheck: dynamic-flag # nocheck: playwright-service-flag — legacy-provider pull for the migration scenario, no UI surface mailu: @@ -97,9 +88,7 @@ mailu: postgres: bond: 1 enabled: true # nocheck: dynamic-flag - # Exception: embedded in swarm — mailu pins its embedded redis for the same manager-VIP reason. shared: "{{ ('svc-db-postgres' in group_names) and (DEPLOYMENT_MODE == 'compose') }}" - # Exception: plain postgres, not the postgis default — baudolo's substring detection needs it for sql-dump backups. image: postgres version: "17" cpus: "0.8" @@ -113,9 +102,6 @@ webmail: enabled: true shared: true image: roundcube/roundcubemail - # Exception: the image ships no HEALTHCHECK, and curl is the only probe tool it carries (no - # wget, no nc) -- so this flavor and no other. `-f` passes the 302 that oauth_login_redirect - # returns from `/` under SSO while still failing a 5xx. healthcheck: flavor: curl version: "1.7.2-apache" @@ -138,7 +124,6 @@ clamav: ports: internal: milter: 7357 - # Exception: elevated memory — clamd holds the full signature DB in memory. cpus: "1.0" mem_reservation: 1g mem_limit: 3g diff --git a/roles/web-app-stalwart/meta/tests.yml b/roles/web-app-stalwart/meta/tests.yml index e729a3f933..272c1e148d 100644 --- a/roles/web-app-stalwart/meta/tests.yml +++ b/roles/web-app-stalwart/meta/tests.yml @@ -1,7 +1,3 @@ --- cli: - # Exception: the migration e2e polls SMTP submission and IMAP delivery for four - # messages, each with its own retry budget (curl --max-time 60 × 30 attempts × two - # mailboxes). The pathological sum exceeds the default 3600s even though a healthy - # run finishes in minutes. timeout: 7200 diff --git a/roles/web-app-stalwart/meta/variants.yml b/roles/web-app-stalwart/meta/variants.yml index 1990f8c079..a7f48e67ed 100644 --- a/roles/web-app-stalwart/meta/variants.yml +++ b/roles/web-app-stalwart/meta/variants.yml @@ -2,15 +2,14 @@ # Matrix-deploy variant overrides for web-app-stalwart. # See docs/contributing/artefact/files/role/variants.md for the file format. -# nocheck: sso-variant-tor — the onion is pinned OFF, and pinned rather than left dynamic. The -# blocker is plugins/lookup/tor_ports.py: it gates HiddenServicePort on the RAW services.yml -# `tor.enabled`, an unrendered Jinja string for every role, so the hidden service forwards no mail -# port and an onion-primary provider hands every consumer an `email.host` nothing answers on. -# (The transport half is done: msmtprc.conf.j2 dials a `.onion` target through Tor's SOCKS -# listener.) A second reason to pin rather than leave dynamic: an unrendered flag reads truthy to -# the domain-map injector while the role's own lookup still resolves clearnet, publishing onion -# vhost names nothing serves. Pinning it ON also skips 05_tls.yml (an onion vhost has no TLS), and -# with it sys-ctl-mtn-cert-deploy, whose `required_by: [web-app-stalwart]` then fails the deploy. +# nocheck: sso-variant-tor — the onion is pinned OFF, and pinned rather than left dynamic. Mail +# reachability over Tor does not need this flag: the stalwart service opts in with `exposed: public` +# (meta/services.yml), which forwards its mail ports on the node onion whenever svc-net-tor is +# deployed. The flag itself moves the role's WEB vhosts onto the onion, and that breaks two ways: +# an onion vhost has no TLS, so 05_tls.yml (and with it sys-ctl-mtn-cert-deploy, whose +# `required_by: [web-app-stalwart]` guard then fails the deploy) is skipped; and left dynamic, the +# unrendered flag reads truthy to the domain-map injector while the role's own lookup still +# resolves clearnet, publishing onion vhost names nothing serves. # web-app-mailu can pin it on only because it is never the active provider. # Two variants pin every dynamic services.yml flag on then off, so the matrix @@ -56,8 +55,6 @@ matomo: enabled: false shared: false - # Exception: the migration imports over IMAP with per-account passwords, so it rides the - # round where SSO is off — an OIDC directory replaces password auth entirely (06_oidc.yml). mailu: enabled: true # nocheck: variants-static-override shared: true # nocheck: variants-static-override diff --git a/roles/web-app-stalwart/tasks/00_core.yml b/roles/web-app-stalwart/tasks/00_core.yml index 000d8a6dab..a6dd7b16d8 100644 --- a/roles/web-app-stalwart/tasks/00_core.yml +++ b/roles/web-app-stalwart/tasks/00_core.yml @@ -1,7 +1,6 @@ --- - include_tasks: utils/once/flag.yml -# Exception: Stalwart's compiled-in validation rejects RFC 6761 reserved TLDs — fail before any state change. - name: "✅ Assert the mail domain is provisionable by Stalwart" ansible.builtin.assert: that: @@ -21,7 +20,6 @@ state: directory mode: "0755" -# Exception: swarm does not auto-create a missing bind source; sys-ctl-mtn-cert-deploy fills the directory later (05_tls). - name: ✅ Ensure certs bind-source directory exists (host) file: path: "{{ [ STALWART_VOLUMES_DIR, 'certs' ] | path_join }}" @@ -87,7 +85,6 @@ - name: "🚿 Flush Docker Compose handlers" meta: flush_handlers -# Exception: without the explicit service key the swarm upstream resolver picks the primary entity and the webmail vhost serves the mail server. - name: "🌐 Proxy the Roundcube webmail at {{ STALWART_WEBMAIL_DOMAIN }}" include_role: name: sys-stk-front-proxy diff --git a/roles/web-app-stalwart/tasks/02_webadmin.yml b/roles/web-app-stalwart/tasks/02_webadmin.yml index 82387145c6..fb66d174df 100644 --- a/roles/web-app-stalwart/tasks/02_webadmin.yml +++ b/roles/web-app-stalwart/tasks/02_webadmin.yml @@ -35,7 +35,6 @@ {'update': {stalwart_webui_app_id: {'resourceUrl': STALWART_WEBUI_LOCAL_URL}}}, 'c0']]} | to_json }} -# Exception: the /api/update/webadmin endpoint is gone; UpdateApps re-unpacks from resourceUrl. - name: "📦 Unpack the WebUI bundle from the local file" include_tasks: jmap_call.yml vars: diff --git a/roles/web-app-stalwart/tasks/04_manage_user.yml b/roles/web-app-stalwart/tasks/04_manage_user.yml index abdfb87212..6460c4352a 100644 --- a/roles/web-app-stalwart/tasks/04_manage_user.yml +++ b/roles/web-app-stalwart/tasks/04_manage_user.yml @@ -8,9 +8,6 @@ {{ stalwart_user_password | hash('sha256') if 'bot' in stalwart_user_roles else stalwart_user_password }} -# Exception: the password is a Password credential keyed by the literal index "0", and a -# re-run answers HTTP 200 with notCreated rather than an error — so neither the shape nor -# a non-2xx check can be used to tell "created" from "already there". - name: "📁 Create Stalwart account '{{ stalwart_user_name }}@{{ STALWART_DOMAIN }}'" include_tasks: jmap_call.yml vars: @@ -24,16 +21,12 @@ 'credentials': {'0': {'@type': 'Password', 'secret': stalwart_user_secret}} }}}, 'c0']]} | to_json }} -# Exception: the id of an already-existing account is only reachable through the -# notCreated(primaryKeyViolation) payload — there is no lookup-by-name method. - name: "🔍 Resolve pre-existing account id for '{{ stalwart_user_name }}'" ansible.builtin.set_fact: stalwart_existing_account_id: >- {{ ((stalwart_jmap.methodResponses[0][1].notCreated | default({})) .get('a', {}).get('objectId', {}) or {}).get('id', '') }} -# Exception: the password must be converged on re-runs — a create-only flow leaves the DB -# on stale credentials whenever the inventory secret differs, and msmtp then fails with 535. - name: "🔄 Update password for existing account '{{ stalwart_user_name }}'" include_tasks: jmap_call.yml vars: @@ -44,8 +37,6 @@ }}}, 'c0']]} | to_json }} when: stalwart_existing_account_id | length > 0 -# Exception: the token stored here is the account's effective secret itself, not a -# separate credential — a bot's secret is the sha256 of its password (see the fact above). - name: "💾 Persist Stalwart submission credential for bot '{{ stalwart_user_name }}'" ansible.builtin.include_role: name: sys-token-store diff --git a/roles/web-app-stalwart/tasks/05_tls.yml b/roles/web-app-stalwart/tasks/05_tls.yml index a6de20201b..aee3fa8745 100644 --- a/roles/web-app-stalwart/tasks/05_tls.yml +++ b/roles/web-app-stalwart/tasks/05_tls.yml @@ -6,8 +6,6 @@ include_role: name: sys-ctl-mtn-cert-deploy vars: - # Exception: MAIL_PRODUCTION_HOST is one knob for both halves of the production - # check — forwarding it here is what makes the remote TLS probe fire at all. cert_deploy_remote_endpoints: >- {{ (STALWART_PRODUCTION_HOST ~ ':465 ' ~ STALWART_PRODUCTION_HOST ~ ':993') if STALWART_PRODUCTION_HOST | length > 0 else '' }} @@ -23,8 +21,6 @@ ansible.builtin.set_fact: stalwart_cert_missing: "{{ (stalwart_jmap.methodResponses[0][1].list | length) == 0 }}" -# Exception: restart-to-activate — see 05_tls/activate.yml for why the -# freshly registered certificate needs a (re)start. - name: "🔧 Register and activate the platform TLS certificate" include_tasks: 05_tls/activate.yml when: stalwart_cert_missing | bool diff --git a/roles/web-app-stalwart/tasks/05_tls/activate.yml b/roles/web-app-stalwart/tasks/05_tls/activate.yml index cc65fee954..09e419d72d 100644 --- a/roles/web-app-stalwart/tasks/05_tls/activate.yml +++ b/roles/web-app-stalwart/tasks/05_tls/activate.yml @@ -1,9 +1,5 @@ --- # include-gated: when: stalwart_cert_missing | bool -# Exception: a freshly registered certificate is served only after a -# (re)start; without the restart the listeners keep the built-in rcgen -# certificate for the rest of the play (nothing else restarts Stalwart in -# non-SSO mode) and strict TLS verification in consumers (msmtp) fails. - name: "🔧 Register the platform TLS certificate with Stalwart" include_tasks: "{{ role_path }}/tasks/jmap_call.yml" vars: diff --git a/roles/web-app-stalwart/tasks/06_oidc.yml b/roles/web-app-stalwart/tasks/06_oidc.yml index 5408975aad..6869dd30a3 100644 --- a/roles/web-app-stalwart/tasks/06_oidc.yml +++ b/roles/web-app-stalwart/tasks/06_oidc.yml @@ -28,7 +28,6 @@ stalwart_oidc_dir_id: "{{ stalwart_jmap.methodResponses[0][1].created.kc.id }}" when: stalwart_oidc_dir_id | length == 0 -# Exception: Keycloak tokens carry aud=account — pinning requireAudience would reject them as InvalidAudience. - name: "🧹 Clear the OIDC directory's required audience" include_tasks: jmap_call.yml vars: @@ -52,8 +51,6 @@ that: stalwart_role_user | length > 0 fail_msg: "Stalwart built-in 'User' role not found; cannot grant IMAP/SMTP mailbox permissions." -# Exception: a self-created Authentication singleton lands with EMPTY defaultUserRoleIds — User accounts -# then fail IMAP login with a silent EOF; seed the built-in roles like Stalwart's own bootstrap does. - name: "🔧 Assemble default role-id maps for the authentication singleton" ansible.builtin.set_fact: stalwart_auth_defaults: @@ -84,7 +81,6 @@ stalwart_jmap_body: "{{ lookup('template', 'jmap/authentication/update.json.j2') }}" when: stalwart_auth_exists | bool -# Exception: the no-reply account has no password under OIDC, so trusted internal nets must relay without SMTP AUTH. - name: "⚙️ Build the trusted-network relay expression" ansible.builtin.set_fact: stalwart_relay_expr: >- diff --git a/roles/web-app-stalwart/tasks/09_dns-records.yml b/roles/web-app-stalwart/tasks/09_dns-records.yml index 05270d9bfd..562aa6088e 100644 --- a/roles/web-app-stalwart/tasks/09_dns-records.yml +++ b/roles/web-app-stalwart/tasks/09_dns-records.yml @@ -27,15 +27,11 @@ name: "_dmarc.{{ STALWART_DOMAIN_DNS_ZONE }}" solo: true value: '"v=DMARC1; p=reject; ruf=mailto:{{ STALWART_DMARC_RUF }}; adkim=s; aspf=s"' - # Exception: Stalwart-managed DKIM — selector + public key are read in - # 08_read_dkim.yml - # (stalwart_dkim_selector / stalwart_dkim_public_key). - type: TXT zone: "{{ STALWART_DOMAIN_DNS_ZONE }}" name: "{{ stalwart_dkim_selector }}._domainkey.{{ STALWART_DOMAIN_DNS_ZONE }}" solo: true value: '"{{ stalwart_dkim_public_key }}"' - # Exception: SRV only for the implicit-TLS ports the service binds (no 587/143/110). - type: SRV zone: "{{ STALWART_DOMAIN_DNS_ZONE }}" name: "{{ STALWART_DOMAIN }}" diff --git a/roles/web-app-stalwart/tasks/12_import_mailu.yml b/roles/web-app-stalwart/tasks/12_import_mailu.yml index e4457ba098..c8df79bc93 100644 --- a/roles/web-app-stalwart/tasks/12_import_mailu.yml +++ b/roles/web-app-stalwart/tasks/12_import_mailu.yml @@ -1,8 +1,6 @@ --- # include-gated: when: STALWART_IMPORT_MAILU and 'web-app-mailu' in group_names # and STALWART_ACTIVE_PROVIDER and DEPLOYMENT_MODE == 'compose' -# Exception: the volume couples to web-app-mailu by design — this task imports that one -# legacy provider's Dovecot maildirs and dies with it (lifecycle: maintenance → eol). - name: "🔍 Resolve the legacy Mailu maildir volume mountpoint" ansible.builtin.command: >- container volume inspect --format "{{ '{{' }} .Mountpoint {{ '}}' }}" @@ -28,8 +26,6 @@ mode: "0600" no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" -# Exception: --imap-insecure is loopback-only — the published IMAPS port on this host; the -# dev/CI cert is self-signed and a hostname match is meaningless on 127.0.0.1. - name: "📥 Import the Mailu maildirs into Stalwart over IMAP" ansible.builtin.script: cmd: >- diff --git a/roles/web-app-stalwart/tasks/jmap_call.yml b/roles/web-app-stalwart/tasks/jmap_call.yml index 27bcf54bc1..37cab47ac7 100644 --- a/roles/web-app-stalwart/tasks/jmap_call.yml +++ b/roles/web-app-stalwart/tasks/jmap_call.yml @@ -10,8 +10,6 @@ {{ STALWART_JMAP_URL }} register: stalwart_jmap_raw changed_when: false - # Rationale: retried — the container may be mid-(re)start (e.g. - # force-recreate or a swarm task reschedule) during provisioning. retries: 12 delay: 5 until: stalwart_jmap_raw.rc == 0 diff --git a/roles/web-app-stalwart/tasks/main.yml b/roles/web-app-stalwart/tasks/main.yml index 0558e54944..83ba771fa2 100644 --- a/roles/web-app-stalwart/tasks/main.yml +++ b/roles/web-app-stalwart/tasks/main.yml @@ -4,10 +4,6 @@ vars: application_id: "web-app-stalwart" -# Exception: re-entry from the service-loader post-load flush (queued in -# 00_core when the OIDC issuer was not yet deployed — Keycloak loads after the -# mail provider). Stack-host only: the exec/restart targets live on the -# manager (the service is manager-pinned). - include_tasks: 11_oidc_restart.yml when: - run_once_web_app_stalwart is defined diff --git a/roles/web-app-stalwart/tasks/manager_ops.yml b/roles/web-app-stalwart/tasks/manager_ops.yml index 102b8497dd..9fe721de29 100644 --- a/roles/web-app-stalwart/tasks/manager_ops.yml +++ b/roles/web-app-stalwart/tasks/manager_ops.yml @@ -3,7 +3,6 @@ # Manager-side operations: every task here execs into or restarts containers # of the manager-pinned stack, so only the stack host may run them. -# Exception: compose only — under swarm the config-object rotation already recreates the task. - name: "🔄 Restart clamav to apply milter config change (compose)" ansible.builtin.command: "container restart {{ STALWART_CONTAINER }}_clamav" changed_when: true @@ -13,7 +12,6 @@ - stalwart_clamav_conf is defined - stalwart_clamav_conf is changed -# Exception: compose only — under swarm the config-object rotation already recreates the task. - name: "🔄 Restart webmail to apply Roundcube config change (compose)" ansible.builtin.command: "container restart {{ STALWART_CONTAINER }}_webmail" changed_when: true diff --git a/roles/web-app-stalwart/templates/clamav-milter.conf.j2 b/roles/web-app-stalwart/templates/clamav-milter.conf.j2 index cac1729f03..5a352d593d 100644 --- a/roles/web-app-stalwart/templates/clamav-milter.conf.j2 +++ b/roles/web-app-stalwart/templates/clamav-milter.conf.j2 @@ -5,9 +5,6 @@ MilterSocketMode 660 ClamdSocket unix:/tmp/clamd.sock OnInfected Reject OnClean Accept -# Exception: this fails OPEN — while clamd is unavailable or still warming up -# (freshclam), mail is accepted unscanned rather than deferred, because a starting -# scanner must not block delivery. OnFail Accept AddHeader Replace LogSyslog no diff --git a/roles/web-app-stalwart/templates/compose.yml.j2 b/roles/web-app-stalwart/templates/compose.yml.j2 index 165e5212b9..b3e80ff202 100644 --- a/roles/web-app-stalwart/templates/compose.yml.j2 +++ b/roles/web-app-stalwart/templates/compose.yml.j2 @@ -16,13 +16,9 @@ ['stalwart', 'imaps', STALWART_IP4_PUBLIC], ['stalwart', 'sieve', STALWART_IP4_PUBLIC]) | indent(4) }} {% else %} - {# Exception: only the active MAIL_PROVIDER owns the public mail ports; a - legacy instance keeps just its reverse-proxied http port. #} {{ lookup('container_ports', ['stalwart', 'http', DOCKER_BIND_HOST]) | indent(4) }} {% endif %} environment: - # Exception: pins the break-glass admin (instead of a random one) — - # API provisioning depends on knowing these credentials. STALWART_RECOVERY_ADMIN: "{{ STALWART_ADMIN_USER }}:{{ STALWART_ADMIN_PASSWORD }}" {{ lookup('container_volumes', service_name) | indent(4) }} {{ lookup('container_depends_on', application_id) | indent(4) }} @@ -39,14 +35,10 @@ environment: ROUNDCUBEMAIL_DEFAULT_HOST: "ssl://stalwart" ROUNDCUBEMAIL_DEFAULT_PORT: "993" - # Exception: 465 — Stalwart's default submission listener is implicit - # TLS; it does not bind 587. ROUNDCUBEMAIL_SMTP_SERVER: "ssl://stalwart" ROUNDCUBEMAIL_SMTP_PORT: "465" ROUNDCUBEMAIL_DB_TYPE: "sqlite" {% if STALWART_SSO_ENABLED | bool %} - # Exception: env-injected so the secret stays out of the world-readable - # oauth.inc.php include. ROUNDCUBEMAIL_OAUTH_CLIENT_SECRET: "{{ OIDC.CLIENT.SECRET }}" {% endif %} {{ lookup('container_volumes', service_name) | indent(4) }} diff --git a/roles/web-app-stalwart/templates/keycloak/clients.json.j2 b/roles/web-app-stalwart/templates/keycloak/clients.json.j2 new file mode 100644 index 0000000000..1a6a5fce47 --- /dev/null +++ b/roles/web-app-stalwart/templates/keycloak/clients.json.j2 @@ -0,0 +1,119 @@ +{# + Keycloak clients this application ships for itself. web-app-keycloak discovers + and includes this fragment (comma-joined client objects, no leading comma) for + every deployed app that declares `services.sso.oidc.clients`; it is rendered in + Keycloak's realm-import context, so realm-owned values (OIDC.CLIENT.*, + KEYCLOAK_RBAC_GROUP_CLAIM) are in scope. + + webui: public SPA client for the WebAdmin (username-first). Its + `pkce.code.challenge.method` is deliberately not pinned — the SPA is compiled + into the Stalwart image and picks its own method; pinning one makes Keycloak + reject the authorize request outright. + + webmail: confidential Roundcube client with minimal scopes (no roles/groups) + so the OIDC access token stays under Stalwart's 4096-byte XOAUTH2 SASL line + limit. Reuses the shared client secret. Keep `description` under Keycloak's + 255-char column. +#} +{% set stalwart_kc_webmail_base = lookup('canonical_url', 'web-app-stalwart', 'webmail') %} +{% set stalwart_kc_webui_base = lookup('tls', 'web-app-stalwart', 'url.base') | regex_replace('/+$', '') %} +{ + "clientId": "{{ lookup('config', 'web-app-stalwart', 'services.sso.oidc.clients.webui') }}", + "name": "Stalwart WebAdmin", + "description": "Public SPA client for Stalwart WebAdmin (username-first).", + "enabled": true, + "surrogateAuthRequired": false, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": ["{{ stalwart_kc_webui_base }}/*"], + "webOrigins": ["{{ stalwart_kc_webui_base }}"], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "post.logout.redirect.uris": "{{ stalwart_kc_webui_base }}/*", + "oauth2.device.authorization.grant.enabled": "false", + "oidc.ciba.grant.enabled": "false", + "backchannel.logout.session.required": "true", + "display.on.consent.screen": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "protocolMappers": [ + { + "name": "stalwart-required-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.client.audience": "{{ OIDC.CLIENT.ID }}", + "id.token.claim": "false", + "access.token.claim": "true", + "introspection.token.claim": "true" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "basic", + "email", + "{{ KEYCLOAK_RBAC_GROUP_CLAIM }}" + ], + "optionalClientScopes": [ + "offline_access", + "microprofile-jwt" + ] +}, +{ + "clientId": "{{ lookup('config', 'web-app-stalwart', 'services.sso.oidc.clients.webmail') }}", + "name": "Stalwart Webmail (Roundcube)", + "description": "Roundcube webmail client. Minimal scopes (no roles/groups) keep the OIDC access token small enough for Stalwart's XOAUTH2 SMTP SASL line (4096-byte limit). Reuses the shared client secret.", + "enabled": true, + "surrogateAuthRequired": false, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "{{ OIDC.CLIENT.SECRET }}", + "redirectUris": ["{{ stalwart_kc_webmail_base }}/*"], + "webOrigins": ["{{ stalwart_kc_webmail_base }}"], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "post.logout.redirect.uris": "{{ stalwart_kc_webmail_base }}/*", + "oauth2.device.authorization.grant.enabled": "false", + "oidc.ciba.grant.enabled": "false", + "backchannel.logout.session.required": "true", + "display.on.consent.screen": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": -1, + "defaultClientScopes": [ + "web-origins", + "acr", + "profile", + "basic", + "email" + ], + "optionalClientScopes": [ + "offline_access" + ] +} diff --git a/roles/web-app-stalwart/templates/roundcube-oauth.inc.php.j2 b/roles/web-app-stalwart/templates/roundcube-oauth.inc.php.j2 index 759e713015..6a956b6ade 100644 --- a/roles/web-app-stalwart/templates/roundcube-oauth.inc.php.j2 +++ b/roles/web-app-stalwart/templates/roundcube-oauth.inc.php.j2 @@ -6,12 +6,6 @@ // Rendered even without SSO (the bind source must exist); the body self-gates. {% if STALWART_SSO_ENABLED | bool %} -// Exception: forced only when the vhost really is https. The proxy speaks HTTP to this -// container, so without this Roundcube builds the OAuth redirect_uri as http:// and Keycloak -// rejects it -- but an onion vhost IS plain HTTP (Tor carries the encryption, and no authority -// issues a certificate for a v3 address), where forcing it sends an https:// redirect_uri that -// was never registered: "Invalid parameter: redirect_uri". Derived from the same canonical the -// Keycloak client's redirectUris are built from, so the two cannot drift apart. {% if STALWART_WEBMAIL_BASE_URL.startswith('https://') %} $_SERVER['HTTPS'] = 'on'; {% endif %} @@ -19,7 +13,7 @@ $_SERVER['HTTPS'] = 'on'; $config['oauth_provider'] = 'generic'; $config['oauth_provider_name'] = '{{ OIDC.BUTTON_TEXT }}'; // Minimal-scope client so the access token stays under Stalwart's 4096-byte -// XOAUTH2 SASL line limit. See roles/web-app-keycloak clients/stalwart_webmail.json.j2. +// XOAUTH2 SASL line limit. See templates/keycloak/clients.json.j2. $config['oauth_client_id'] = '{{ STALWART_WEBMAIL_OIDC_CLIENT_ID }}'; $config['oauth_auth_uri'] = '{{ OIDC.CLIENT.AUTHORIZE_URL }}'; $config['oauth_token_uri'] = '{{ OIDC.CLIENT.TOKEN_URL }}'; @@ -27,12 +21,6 @@ $config['oauth_identity_uri'] = '{{ OIDC.CLIENT.USER_INFO_URL }}'; $config['oauth_logout_uri'] = '{{ OIDC.CLIENT.LOGOUT_URL }}'; $config['oauth_scope'] = 'openid email profile'; {% if STALWART_OIDC_SOCKS_PROXY %} -// Exception: the OAuth back-channel (code->token, JWKS, userinfo) is a server-side cURL call, -// and cURL hard-refuses to resolve a .onion name (RFC 7686) whatever /etc/hosts says -- the -// extra_hosts SSO pin fixes the callers that honour /etc/hosts, but cannot help this one. Only -// socks5h gets it through, by handing the name to Tor to resolve instead of resolving locally. -// Merged into every Guzzle client (rcube::get_http_client), and the OAuth call sets no `proxy` -// of its own, so it survives the union. Absent on clearnet, where the gate renders empty. $config['http_client'] = ['proxy' => '{{ STALWART_OIDC_SOCKS_PROXY }}']; {% endif %} $config['oauth_auth_parameters'] = []; diff --git a/roles/web-app-stalwart/templates/test.env.j2 b/roles/web-app-stalwart/templates/test.env.j2 index 3d7efe639b..e74416f36a 100644 --- a/roles/web-app-stalwart/templates/test.env.j2 +++ b/roles/web-app-stalwart/templates/test.env.j2 @@ -1,5 +1,3 @@ -{# Exception: this file is sourced by bash, so values take the shell `quote` filter — - `dotenv_quote` passes values through unquoted in swarm mode and breaks on secrets. #} ADMIN_EMAIL={{ lookup('users', 'administrator').email | quote }} ADMIN_IMAP_PASSWORD={{ lookup('users', 'administrator').password | quote }} @@ -14,15 +12,8 @@ MAILU_SMTP_CONTAINER={{ lookup('config', 'web-app-mailu', 'services.smtp.name') MAIL_DOMAIN={{ STALWART_DOMAIN | quote }} -{# Exception: addressed, not staged — test-e2e-cli copies only files/test/, and the - migration script is role code shared with tasks/12_import_mailu.yml. #} MIGRATION_SCRIPT={{ lookup('path_absolute', 'roles', application_id, 'files/python/migrate_from_mailu.py') | quote }} -{# Exception: the migration script imports nothing outside the stdlib, but the bare - ansible_python_interpreter may be older than the project floor — prefer the venv. #} PYTHON_BIN={{ (lookup('env', 'PYTHON') or '/opt/venvs/infinito/bin/python') | quote }} -{# Exception: the harness stores mail in the legacy provider and migrates it in place — - only when this role is the deploy target (not a pulled-in mail provider), on the stack - host, in compose mode (the import is compose-only; see tasks/manager_ops.yml). #} STALWART_MIGRATION_E2E={{ ((application_id in PRIMARY_APPS) and ('web-app-mailu' in group_names) and DEPLOYMENT_MODE == 'compose' and IS_STACK_HOST | bool) | string | lower }} diff --git a/roles/web-app-stalwart/vars/main.yml b/roles/web-app-stalwart/vars/main.yml index dd13fc3de6..079f4d9c19 100644 --- a/roles/web-app-stalwart/vars/main.yml +++ b/roles/web-app-stalwart/vars/main.yml @@ -11,26 +11,16 @@ STALWART_CONFIG_FILE: "{{ [ STALWART_CONFIG_DIR, 'config.json' ] | path_join }}" container_port: "{{ lookup('config', application_id, 'services.stalwart.ports.internal.http') }}" STALWART_LOCAL_BASE_URL: "http://localhost:{{ container_port }}" STALWART_JMAP_URL: "{{ STALWART_LOCAL_BASE_URL }}/jmap/" -# Exception: must stay unquoted in shell tasks — resolves to $(resolve-container-id …) in swarm. STALWART_CONTAINER_ADDRESS: "{{ lookup('container_address', application_id, 'stalwart') }}" STALWART_DOMAIN: "{{ lookup('config', application_id, 'services.stalwart.domain') }}" STALWART_DOMAIN_DNS_ZONE: "{{ STALWART_DOMAIN | to_zone }}" STALWART_EMAIL: "{{ lookup('email', application_id) }}" STALWART_HOSTNAME: "{{ STALWART_EMAIL.host }}" -# Exception: STALWART_HOSTNAME is the ACTIVE provider's mail host (MX/SPF point there); the web -# vhost must follow this role's own canonical instead, or a legacy Stalwart claims the active -# provider's domain and never serves its own. STALWART_WEB_DOMAIN: "{{ lookup('domain', application_id) }}" STALWART_HOSTNAME_DNS_ZONE: "{{ STALWART_HOSTNAME | to_zone }}" -# Exception: the container's own hostname is NOT STALWART_HOSTNAME — an onion canonical -# (`mail.<56 chars>.onion`) overflows sethostname(2)'s 64-byte cap and docker refuses to -# create the container; the lookup falls back to the short entity name in that case. container_hostname: "{{ lookup('container_hostname', application_id) }}" -# Exception: curl refuses to resolve a .onion name at all (RFC 7686), so the extra_hosts pin -# that fixes the other back-channel callers cannot help it — an onion issuer must be dialled -# through Tor's SOCKS listener instead. Empty on clearnet, where `--proxy ''` is a curl no-op. STALWART_OIDC_ONION_ISSUER: "{{ lookup('tls', 'web-app-keycloak', 'domain').endswith('.onion') }}" STALWART_OIDC_SOCKS_PROXY: "{{ lookup('tor_socks_proxy') if STALWART_OIDC_ONION_ISSUER | bool else '' }}" @@ -43,16 +33,12 @@ STALWART_CLAMAV_MILTER_TIMEOUT_CONNECT_MS: 5000 STALWART_CLAMAV_MILTER_TIMEOUT_COMMAND_MS: 5000 STALWART_CLAMAV_MILTER_TIMEOUT_DATA_MS: 15000 STALWART_CLAMAV_MILTER_CONF: "{{ [ STALWART_CONFIG_DIR, 'clamav-milter.conf' ] | path_join }}" -# Exception: resolved through canonical_url, NOT `config domains.canonical.webmail` — the raw -# declaration renders `webmail.` and never sees the onion injection, so on an -# onion node the vhost was created under a clearnet name while the domain map (and every probe) -# expected `webmail.`, which then answered 404. STALWART_WEBMAIL_BASE_URL: "{{ lookup('canonical_url', application_id, 'webmail') }}" STALWART_WEBMAIL_DOMAIN: "{{ STALWART_WEBMAIL_BASE_URL | regex_replace('^https?://', '') }}" STALWART_WEBMAIL_HTTP_PORT: "{{ lookup('config', application_id, 'services.webmail.ports.local.http') }}" STALWART_WEBMAIL_CONFIG_DIR: "{{ [ STALWART_VOLUMES_DIR, 'webmail' ] | path_join }}" STALWART_WEBMAIL_OAUTH_CONFIG: "{{ [ STALWART_WEBMAIL_CONFIG_DIR, 'oauth.inc.php' ] | path_join }}" -STALWART_WEBMAIL_OIDC_CLIENT_ID: "{{ lookup('config', application_id, 'services.sso.oidc.webmail_client_id') }}" +STALWART_WEBMAIL_OIDC_CLIENT_ID: "{{ lookup('config', application_id, 'services.sso.oidc.clients.webmail') }}" STALWART_WEBUI_VERSION: "{{ lookup('config', application_id, 'services.stalwart.webui_version') }}" STALWART_WEBUI_RESOURCE_URL: "https://github.com/stalwartlabs/webui/releases/download/{{ STALWART_WEBUI_VERSION }}/webui.zip" diff --git a/roles/web-app-zammad/vars/main.yml b/roles/web-app-zammad/vars/main.yml index fcc928397e..fd45f93894 100644 --- a/roles/web-app-zammad/vars/main.yml +++ b/roles/web-app-zammad/vars/main.yml @@ -54,5 +54,4 @@ ZAMMAD_LDAP_ENABLED: "{{ lookup('config', application_id, 'services.ldap.enabled ZAMMAD_MAIL: "{{ lookup('email', application_id) if ZAMMAD_EMAIL_ENABLED else {} }}" ZAMMAD_HELPDESK_USER: "{{ lookup('users', 'helpdesk') if ZAMMAD_EMAIL_ENABLED else {} }}" ZAMMAD_HELPDESK_EMAIL: "{{ ZAMMAD_HELPDESK_USER.email | default('') }}" -# Exception: the user password rejects SMTP AUTH (535, change_pw_next_login) — only the per-app token works. ZAMMAD_HELPDESK_PASS: "{{ (ZAMMAD_HELPDESK_USER.tokens | default({})).get(MAIL_PROVIDER, '') }}" diff --git a/scripts/system/network/dns/common.sh b/scripts/system/network/dns/common.sh index 4cb6f205a9..d946a4dcb6 100755 --- a/scripts/system/network/dns/common.sh +++ b/scripts/system/network/dns/common.sh @@ -100,7 +100,6 @@ dns_read_hosts_fallback_entries() { dns_rewrite_hosts_file() { local tmp="$1" - # Exception: replacing the inode breaks a bind-mounted hosts file (/etc/hosts in CI containers) — write in place. if [[ -e "${DNS_HOSTS_FILE}" ]]; then dns_write_file_in_place "${tmp}" "${DNS_HOSTS_FILE}" else @@ -163,7 +162,6 @@ dns_test_resolution() { echo ">>> Testing resolution" if [[ "${DNS_HOSTS_FILE}" == "/etc/hosts" ]]; then while IFS= read -r host; do - # Exception: retried — a freshly restarted resolver may need a moment before it answers. for attempt in 1 2 3; do if getent hosts "${host}"; then break diff --git a/scripts/system/network/dns/setup/linux.sh b/scripts/system/network/dns/setup/linux.sh index fa324c00c8..c7e16c53c4 100755 --- a/scripts/system/network/dns/setup/linux.sh +++ b/scripts/system/network/dns/setup/linux.sh @@ -92,9 +92,6 @@ EOF sudo systemctl enable dnsmasq --now sudo systemctl restart dnsmasq - # Exception: `resolvectl dns lo ...` is NOT used here — systemd-resolved rejects - # per-link DNS on loopback, so the zone is routed to dnsmasq via a global drop-in - # instead; the per-link form fails silently and leaves the zone unresolved. if systemctl is-active --quiet systemd-resolved 2>/dev/null; then echo ">>> Routing ${DNS_DOMAIN} to dnsmasq via systemd-resolved drop-in: ${DNS_RESOLVED_DROPIN}" sudo mkdir -p "$(dirname "${DNS_RESOLVED_DROPIN}")" diff --git a/scripts/tests/deploy/local/deploy/apps/initialize/selection.sh b/scripts/tests/deploy/local/deploy/apps/initialize/selection.sh index 1f64ad6f7a..4ab6f98f6f 100755 --- a/scripts/tests/deploy/local/deploy/apps/initialize/selection.sh +++ b/scripts/tests/deploy/local/deploy/apps/initialize/selection.sh @@ -108,13 +108,6 @@ else fi echo ">>> Creating inventory for app '${apps}'" -# Exception: RUNTIME MUST be pinned to `dev` here — the host process running this -# script lives OUTSIDE the development compose stack, so `detect_runtime()` falls back -# to "host". Without the override the matrix-init step bakes `RUNTIME=host` into -# host_vars, the Playwright E2E gate (RUNTIME in [dev, act, github]) never fires, and -# kept-deploys silently skip the test stage. Mirrors apps/reinstall/selection.sh. -# INFINITO_INVENTORY_EXTRA_VARS is merged over these defaults — the migration scenario -# pins {"MAIL_PROVIDER": "web-app-mailu"} for its legacy leg. INVENTORY_VARS='{"ASYNC_ENABLED": false, "RUNTIME": "dev"}' if [[ -n "${INFINITO_INVENTORY_EXTRA_VARS:-}" ]]; then INVENTORY_VARS="$("${PYTHON}" -c \ diff --git a/scripts/tests/workspace/utils/common.sh b/scripts/tests/workspace/utils/common.sh index ff912d9f0b..a1b241d173 100755 --- a/scripts/tests/workspace/utils/common.sh +++ b/scripts/tests/workspace/utils/common.sh @@ -53,8 +53,6 @@ load_repo_env() { load_repo_env ensure_git_safe_directory -# Exception: this re-reads the env files by hand because bare CI containers lack -# python3, so load_repo_env above may have silently done nothing. if [[ -z "${INFINITO_DOMAIN:-}" ]]; then for _env_file in "${REPO_ROOT}/.env" "${REPO_ROOT}/default.env"; do if [[ -f "${_env_file}" ]]; then @@ -67,9 +65,6 @@ fi DASHBOARD_URL="https://dashboard.${INFINITO_DOMAIN:?Missing INFINITO_DOMAIN in .env and default.env}" MATOMO_URL="https://matomo.${INFINITO_DOMAIN}" -# Exception: this expansion looks dead but is the interface contract — the constants are -# consumed by sibling scripts that source this file, and `: "${VAR}"` under `set -u` -# fails here rather than in whichever script forgot to define one. : "${DASHBOARD_APP}" "${MATOMO_APP}" "${MARIADB_APP}" "${POSTGRES_APP}" "${DASHBOARD_URL}" "${MATOMO_URL}" # Print the generated inventory and host_vars for debugging and verification. diff --git a/tests/integration/infrastructure/services/test_frontend_spot.py b/tests/integration/infrastructure/services/test_frontend_spot.py index 64675da59f..7617c3f3ca 100644 --- a/tests/integration/infrastructure/services/test_frontend_spot.py +++ b/tests/integration/infrastructure/services/test_frontend_spot.py @@ -46,8 +46,6 @@ class TestFrontendServiceSpot(unittest.TestCase): def test_keycloak_respects_run_after_dependencies(self): self.assertLess(self._index("web-app-matomo"), self._index("web-app-keycloak")) - # Exception: both providers are asserted, not just the default one — either may - # be the active MAIL_PROVIDER, so ordering has to hold for both. self.assertLess( self._index("web-app-stalwart"), self._index("web-app-keycloak") ) diff --git a/tests/integration/meta/workflows/test_diff_affected_roles.py b/tests/integration/meta/workflows/test_diff_affected_roles.py index 67e1b76572..56473a568c 100644 --- a/tests/integration/meta/workflows/test_diff_affected_roles.py +++ b/tests/integration/meta/workflows/test_diff_affected_roles.py @@ -45,8 +45,6 @@ class TestDiffAffectedRoles(unittest.TestCase): check=True, ) subprocess.run(["git", "config", "user.name", "ci"], cwd=repo, check=True) - # Exception: hermetic fixture — an operator's global commit.gpgsign=true - # would make these scratch commits launch pinentry and fail off-terminal. subprocess.run( ["git", "config", "commit.gpgsign", "false"], cwd=repo, check=True ) diff --git a/tests/unit/python/plugins/lookup/test_tor_ports.py b/tests/unit/python/plugins/lookup/test_tor_ports.py index 5be68ffe0a..6efaa52539 100644 --- a/tests/unit/python/plugins/lookup/test_tor_ports.py +++ b/tests/unit/python/plugins/lookup/test_tor_ports.py @@ -137,6 +137,63 @@ class TestTorPortsLookup(unittest.TestCase): ) self.assertEqual(ports, [389, 5432]) + def test_exposed_public_forwards_the_public_group(self) -> None: + """The active mail provider's protocol ports ride the node onion.""" + apps = { + "web-app-stalwart": { + "services": { + "stalwart": { + "exposed": "public", + "ports": { + "local": {"http": 8081}, + "public": {"smtp": 25, "smtps": 465, "imaps": 993}, + }, + } + } + } + } + self.assertEqual( + collect_exposed_ports(apps, ["web-app-stalwart"]), [25, 465, 993] + ) + + def test_exposed_public_does_not_leak_the_local_group(self) -> None: + """`public` replaces `local` — the reverse-proxied http port stays off the onion port list.""" + apps = { + "a": { + "services": { + "s": { + "exposed": "public", + "ports": {"local": {"http": 8081}, "public": {"smtps": 465}}, + } + } + } + } + self.assertEqual(collect_exposed_ports(apps, ["a"]), [465]) + + def test_exposed_true_still_never_forwards_public_ports(self) -> None: + """openldap's ldaps 636 must stay a non-forward: nothing listens on it in the exposed variant.""" + apps = { + "svc-db-openldap": { + "services": { + "openldap": { + "exposed": True, + "ports": {"local": {"ldap": 389}, "public": {"ldaps": 636}}, + } + } + } + } + self.assertEqual(collect_exposed_ports(apps, ["svc-db-openldap"]), [389]) + + def test_exposed_unknown_string_forwards_nothing(self) -> None: + apps = { + "a": { + "services": { + "s": {"exposed": "everything", "ports": {"public": {"smtp": 25}}} + } + } + } + self.assertEqual(collect_exposed_ports(apps, ["a"]), []) + def test_collect_exposed_ports_ignores_non_mappings(self) -> None: self.assertEqual(collect_exposed_ports({}, ["a"]), []) self.assertEqual(collect_exposed_ports(None, ["a"]), []) diff --git a/tests/unit/python/roles/web-app-gitlab/__init__.py b/tests/unit/python/roles/web-app-gitlab/__init__.py new file mode 100644 index 0000000000..8ee16d7567 --- /dev/null +++ b/tests/unit/python/roles/web-app-gitlab/__init__.py @@ -0,0 +1,3 @@ +from pathlib import Path + +PROJECT_ROOT: Path = Path(__file__).resolve().parents[5] diff --git a/tests/unit/python/roles/web-app-gitlab/test_smtp_settings.py b/tests/unit/python/roles/web-app-gitlab/test_smtp_settings.py new file mode 100644 index 0000000000..fec943d68e --- /dev/null +++ b/tests/unit/python/roles/web-app-gitlab/test_smtp_settings.py @@ -0,0 +1,96 @@ +"""Whether GitLab announces SMTP AUTH, and on which listener. + +Ruby's ``net/smtp`` authenticates whenever a ``user_name`` or ``password`` is +present, falling back to its default auth type — ``authentication: nil`` does +NOT switch it off. So on the provider's SSO relay, where port 25 offers no +AUTH at all, leaving the credentials in the hash makes every send die with +``503 5.5.1 AUTH not allowed`` while GitLab still reports the mail as queued. +The credentials therefore have to be omitted, not merely unaccompanied by an +auth type. + +``tls`` and ``enable_starttls_auto`` are the other half: they are mutually +exclusive in ActionMailer, implicit TLS on 465 or STARTTLS on the relay. +""" + +from __future__ import annotations + +import re +import unittest + +from jinja2 import Environment, StrictUndefined + +from utils.cache.files import read_text + +from . import PROJECT_ROOT + +TEMPLATE = ( + PROJECT_ROOT + / "roles" + / "web-app-gitlab" + / "templates" + / "config" + / "smtp_settings.rb.j2" +) + +RELAY = { + "host": "mail.infinito.test", + "port": 25, + "username": "no-reply@infinito.test", + "password": "s3cr3t", + "domain": "infinito.test", + "auth": False, + "start_tls": True, + "tls": False, +} +SUBMISSION = dict(RELAY, port=465, auth=True, start_tls=False, tls=True) + + +def _render(email: dict) -> str: + env = Environment( + trim_blocks=True, + lstrip_blocks=True, + undefined=StrictUndefined, + autoescape=False, # noqa: S701 - a Ruby config file, and Ansible's templar does not escape either + ) + env.filters["bool"] = bool + env.filters["ruby_dq"] = lambda value: '"' + str(value).replace('"', '\\"') + '"' + return env.from_string(read_text(TEMPLATE)).render(GITLAB_EMAIL=email) + + +class TestGitlabSmtpSettings(unittest.TestCase): + def test_the_relay_carries_no_credentials_at_all(self): + rendered = _render(RELAY) + self.assertNotIn("user_name", rendered) + self.assertNotIn("password", rendered) + self.assertNotIn("s3cr3t", rendered) + + def test_the_relay_announces_no_auth_type(self): + self.assertNotIn("authentication", _render(RELAY)) + + def test_authenticated_submission_still_sends_credentials(self): + rendered = _render(SUBMISSION) + self.assertIn('user_name: "no-reply@infinito.test"', rendered) + self.assertIn("authentication: :login", rendered) + + def test_tls_and_starttls_are_never_both_on(self): + for email in (RELAY, SUBMISSION): + rendered = _render(email) + tls = re.search(r"^\s*tls:\s*(\w+)", rendered, re.MULTILINE).group(1) + starttls = re.search( + r"^\s*enable_starttls_auto:\s*(\w+)", rendered, re.MULTILINE + ).group(1) + self.assertNotEqual((tls, starttls), ("true", "true")) + + def test_the_relay_uses_starttls_and_the_submission_port_implicit_tls(self): + self.assertIn("enable_starttls_auto: true", _render(RELAY)) + self.assertIn("tls: false", _render(RELAY)) + self.assertIn("tls: true", _render(SUBMISSION)) + + def test_the_hash_stays_syntactically_valid_without_credentials(self): + rendered = _render(RELAY).strip() + self.assertTrue(rendered.endswith("}")) + self.assertNotIn(",\n}", rendered) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/roles/web-app-keycloak/filter_plugins/test_oidc_clients.py b/tests/unit/python/roles/web-app-keycloak/filter_plugins/test_oidc_clients.py new file mode 100644 index 0000000000..81e1a58329 --- /dev/null +++ b/tests/unit/python/roles/web-app-keycloak/filter_plugins/test_oidc_clients.py @@ -0,0 +1,109 @@ +"""Which Keycloak clients get converged, and — more importantly — which do not. + +The realm import also carries Keycloak's own built-in clients (``account``, +``broker``, ``realm-management`` ...). Converging those would be wrong, so the +set is taken from what applications declare about themselves rather than from +the realm document. An application that declares nothing contributes nothing, +which is what keeps this a no-op for every app that needs only the one shared +client. +""" + +from __future__ import annotations + +import importlib.util +import unittest +from pathlib import Path + +PLUGIN = ( + Path(__file__).parent + / "../../../../../../roles/web-app-keycloak/filter_plugins/oidc_clients.py" +).resolve() + + +def _load(): + spec = importlib.util.spec_from_file_location("oidc_clients", PLUGIN) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _app(clients): + return {"services": {"sso": {"oidc": {"clients": clients}}}} + + +class TestDeclaredClientApps(unittest.TestCase): + def setUp(self): + self.apps = _load().kc_declared_client_apps + + def test_only_declaring_apps_are_listed(self): + apps = { + "web-app-stalwart": _app({"webui": "stalwart-webui"}), + "web-app-nextcloud": {"services": {"sso": {"oidc": {}}}}, + } + self.assertEqual( + self.apps(apps, ["web-app-stalwart", "web-app-nextcloud"]), + ["web-app-stalwart"], + ) + + def test_an_undeployed_app_is_not_listed(self): + apps = {"web-app-stalwart": _app({"webui": "stalwart-webui"})} + self.assertEqual(self.apps(apps, ["web-app-other"]), []) + + def test_a_malformed_map_is_survivable(self): + self.assertEqual(self.apps(None, ["a"]), []) + + +class TestDeclaredClientIds(unittest.TestCase): + def setUp(self): + self.ids = _load().kc_declared_client_ids + + def test_an_application_contributes_the_clients_it_declares(self): + apps = { + "web-app-stalwart": _app( + {"webui": "stalwart-webui", "webmail": "stalwart-webmail"} + ) + } + self.assertEqual( + self.ids(apps, ["web-app-stalwart"]), + ["stalwart-webmail", "stalwart-webui"], + ) + + def test_an_application_that_declares_nothing_contributes_nothing(self): + apps = {"web-app-nextcloud": {"services": {"sso": {"oidc": {}}}}} + self.assertEqual(self.ids(apps, ["web-app-nextcloud"]), []) + + def test_keycloak_builtins_are_never_included(self): + """They live in the realm document, not in any application's declaration.""" + apps = {"web-app-stalwart": _app({"webui": "stalwart-webui"})} + self.assertNotIn("account", self.ids(apps, ["web-app-stalwart"])) + self.assertNotIn("realm-management", self.ids(apps, ["web-app-stalwart"])) + + def test_an_undeployed_application_is_not_consulted(self): + apps = { + "web-app-stalwart": _app({"webui": "stalwart-webui"}), + "web-app-other": _app({"main": "other-client"}), + } + self.assertEqual(self.ids(apps, ["web-app-other"]), ["other-client"]) + + def test_duplicate_declarations_collapse(self): + apps = { + "a": _app({"one": "shared-client"}), + "b": _app({"two": "shared-client"}), + } + self.assertEqual(self.ids(apps, ["a", "b"]), ["shared-client"]) + + def test_blank_and_non_string_declarations_are_ignored(self): + apps = {"a": _app({"ok": "real", "blank": " ", "wrong": 7, "none": None})} + self.assertEqual(self.ids(apps, ["a"]), ["real"]) + + def test_a_malformed_application_map_is_survivable(self): + self.assertEqual(self.ids(None, ["a"]), []) + self.assertEqual(self.ids({"a": "not-a-dict"}, ["a"]), []) + + def test_no_app_ids_means_every_application_in_the_map(self): + apps = {"a": _app({"one": "client-a"}), "b": _app({"one": "client-b"})} + self.assertEqual(self.ids(apps), ["client-a", "client-b"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/roles/web-app-keycloak/templates/test_realm_import_json.py b/tests/unit/python/roles/web-app-keycloak/templates/test_realm_import_json.py index 2b1aafb5ba..84bbd98d06 100644 --- a/tests/unit/python/roles/web-app-keycloak/templates/test_realm_import_json.py +++ b/tests/unit/python/roles/web-app-keycloak/templates/test_realm_import_json.py @@ -50,12 +50,6 @@ CONTEXT = { "KEYCLOAK_LDAP_SYNC_CHANGES_PERIOD": "86400", "KEYCLOAK_LDAP_SYNC_FULL_PERIOD": "604800", "KEYCLOAK_RBAC_GROUP_NAME": "groups", - "KEYCLOAK_STALWART_ENABLED": True, - "KEYCLOAK_STALWART_WEBUI_CLIENT_ID": "stalwart-webui", - "KEYCLOAK_STALWART_WEBMAIL_CLIENT_ID": "stalwart-webmail", - "KEYCLOAK_STALWART_WEBMAIL_REDIRECT_URIS": ["https://webmail.example.com/*"], - "KEYCLOAK_STALWART_WEBMAIL_WEB_ORIGINS": ["https://webmail.example.com"], - "KEYCLOAK_STALWART_WEBMAIL_POST_LOGOUT": "+", "LDAP_ONLY": False, "OIDC": { "CLIENT": {"SECRET": "s3cret", "ID": "example.com"}, @@ -97,12 +91,14 @@ def fake_lookup(name, *_args, **_kwargs): """ if name == "email": return {"enabled": False} + if name == "config": + return str(_args[-1]).rsplit(".", 1)[-1] return APP_URL -def render(saml_apps: list[str]) -> str: +def render(saml_apps: list[str], client_apps: list[str] | None = None) -> str: env = Environment( - loader=FileSystemLoader(str(IMPORT_DIR)), + loader=FileSystemLoader([str(IMPORT_DIR), str(PROJECT_ROOT)]), trim_blocks=True, keep_trailing_newline=True, undefined=StrictUndefined, @@ -117,6 +113,7 @@ def render(saml_apps: list[str]) -> str: env.filters.update(role_filters()) return env.get_template("realm.json.j2").render( KEYCLOAK_SAML_APPS=saml_apps, + KEYCLOAK_DECLARED_CLIENT_APPS=client_apps or [], lookup=fake_lookup, application_id="web-app-keycloak", DOMAIN_PRIMARY="example.com", @@ -125,6 +122,18 @@ def render(saml_apps: list[str]) -> str: class TestRealmImportJson(unittest.TestCase): + def test_app_declared_clients_join_the_realm(self) -> None: + """An app in KEYCLOAK_DECLARED_CLIENT_APPS ships its own client fragment.""" + realm = json.loads(render([], client_apps=["web-app-stalwart"])) + client_ids = [c["clientId"] for c in realm["clients"]] + self.assertIn("webui", client_ids) + self.assertIn("webmail", client_ids) + + def test_no_declaring_apps_means_no_extra_clients(self) -> None: + with_apps = json.loads(render([], client_apps=["web-app-stalwart"])) + without = json.loads(render([])) + self.assertEqual(len(with_apps["clients"]) - len(without["clients"]), 2) + def test_realm_is_valid_json_without_saml_apps(self) -> None: realm = json.loads(render([])) client_ids = [c["clientId"] for c in realm["clients"]] diff --git a/tests/unit/python/roles/web-app-stalwart/files/test_migrate_from_mailu.py b/tests/unit/python/roles/web-app-stalwart/files/test_migrate_from_mailu.py index baea71166e..09f0f40888 100644 --- a/tests/unit/python/roles/web-app-stalwart/files/test_migrate_from_mailu.py +++ b/tests/unit/python/roles/web-app-stalwart/files/test_migrate_from_mailu.py @@ -12,7 +12,6 @@ def _load_module(): spec = importlib.util.spec_from_file_location("migrate_from_mailu", str(path)) mod = importlib.util.module_from_spec(spec) assert spec and spec.loader - # Exception: dataclasses resolve their module via sys.modules — register before exec. sys.modules["migrate_from_mailu"] = mod spec.loader.exec_module(mod) return mod diff --git a/utils/cache/files.py b/utils/cache/files.py index 576cd81246..f871884b9b 100644 --- a/utils/cache/files.py +++ b/utils/cache/files.py @@ -92,8 +92,6 @@ def _all_project_files() -> tuple[str, ...]: skip_dirs = _DEFAULT_SKIP_DIRS | _local_exclude_dirs() paths: list[str] = [] for dirpath, dirnames, filenames in os.walk(root_str, topdown=True): - # Exception: dirnames must be mutated IN PLACE — that is what stops os.walk - # descending into the skipped dirs; rebinding the name would walk them anyway. dirnames[:] = [d for d in dirnames if d not in skip_dirs] paths.extend(str(Path(dirpath) / fn) for fn in filenames) return tuple(paths)