diff --git a/plugins/lookup/email.py b/plugins/lookup/email.py index 56aaacb80e..d527682538 100644 --- a/plugins/lookup/email.py +++ b/plugins/lookup/email.py @@ -9,6 +9,7 @@ from ansible.plugins.lookup import LookupBase from plugins.lookup.applications import LookupModule as ApplicationsLookup from plugins.lookup.domain import LookupModule as DomainLookup from plugins.lookup.users import LookupModule as UsersLookup +from utils.roles.entity.name import get_entity_name SYSTEM_EMAIL_PREFIX = "SYSTEM_EMAIL_" @@ -166,7 +167,17 @@ class LookupModule(LookupBase): return 25 if not external: return 25 - return 465 if _as_bool(resolved.get("tls")) else 587 + ports = self._provider_ports(variables) + if _as_bool(resolved.get("tls")): + return ports.get("smtps", 465) + # Plaintext submission is the onion/no-TLS path, but a provider is + # only reachable there if it binds one. Stalwart's default config + # binds the implicit-TLS client ports plus the MX, so it advertises + # no `submission`; fall back to the MX port rather than a listener + # nothing answers on (`auth` drops with it -- an MX takes local + # recipients without SMTP AUTH, the same property the SSO relay + # above relies on). + return ports.get("submission") or ports.get("smtp", 25) if short_key == "host": env = resolved.get("environment") # `environment` folds TLS_ENABLED into its "external" base, so a @@ -187,6 +198,14 @@ class LookupModule(LookupBase): return False if self._provider_uses_sso_relay(variables): return False + # The MX listener offers no SMTP AUTH: when `port` fell back to it + # because the provider binds no plaintext submission, announcing + # AUTH aborts msmtp with EX_UNAVAILABLE before it ever sends. + ports = self._provider_ports(variables) + if not ports.get("submission") and resolved.get("port") == ports.get( + "smtp", 25 + ): + return False return _as_bool(resolved.get("external")) if short_key == "auth_mechanism": if not _as_bool(resolved.get("auth")): @@ -222,12 +241,10 @@ class LookupModule(LookupBase): value = variables.get("MAIL_PROVIDER") return str(value).strip() if value else DEFAULT_MAIL_PROVIDER - def _provider_uses_sso_relay(self, variables: dict[str, Any]) -> bool: - """True when the active provider self-declares SSO relay (via - ``services.sso.oidc.submission_via_relay``) and Keycloak is deployed.""" - group_names = variables.get("group_names") or [] - if "web-app-keycloak" not in group_names: - return False + def _provider_services(self, variables: dict[str, Any]) -> dict[str, Any]: + """The active provider's ``services`` block, or ``{}`` when it cannot be + resolved. Both the relay probe and the port probe read the provider's + own self-declaration, so the applications lookup is done once here.""" apps = ApplicationsLookup() apps._templar = getattr(self, "_templar", None) forwarded = { @@ -240,10 +257,43 @@ class LookupModule(LookupBase): **forwarded, )[0] except AnsibleError: - return False + return {} if not isinstance(entry, dict): - return False + return {} services = entry.get("services") or {} + return services if isinstance(services, dict) else {} + + def _provider_ports(self, variables: dict[str, Any]) -> dict[str, int]: + """The SMTP ports the active provider publishes, keyed by protocol. + + Read from the provider's own ``services..ports.public`` rather + than assumed, so a provider that binds no plaintext ``submission`` + (Stalwart) resolves differently from one that does (Mailu) without + either being named here. + """ + services = self._provider_services(variables) + entity = get_entity_name(self._mail_provider(variables)) + entry = services.get(entity) or {} + if not isinstance(entry, dict): + return {} + ports = (entry.get("ports") or {}).get("public") or {} + if not isinstance(ports, dict): + return {} + out: dict[str, int] = {} + for name, value in ports.items(): + try: + out[str(name)] = int(value) + except (TypeError, ValueError): + continue + return out + + def _provider_uses_sso_relay(self, variables: dict[str, Any]) -> bool: + """True when the active provider self-declares SSO relay (via + ``services.sso.oidc.submission_via_relay``) and Keycloak is deployed.""" + group_names = variables.get("group_names") or [] + if "web-app-keycloak" not in group_names: + return False + services = self._provider_services(variables) sso = services.get("sso") or {} if isinstance(services, dict) else {} oidc = sso.get("oidc") or {} if isinstance(sso, dict) else {} if not isinstance(oidc, dict): diff --git a/plugins/lookup/smtp_host.py b/plugins/lookup/smtp_host.py index b3ed83d117..bdb38ecfda 100644 --- a/plugins/lookup/smtp_host.py +++ b/plugins/lookup/smtp_host.py @@ -6,6 +6,8 @@ from ansible.errors import AnsibleError from ansible.module_utils.parsing.convert_bool import boolean as _to_bool from ansible.plugins.lookup import LookupBase +from utils.networks.lookup_context import resolve_var + LOOPBACK = "127.0.0.1" CONTAINER_HOST_ALIAS = "host.docker.internal" NODE_GROUP = "svc-swarm-node" @@ -78,4 +80,13 @@ class LookupModule(LookupBase): raise AnsibleError("smtp_host: the term must be the email mapping.") variables = variables or getattr(self._templar, "available_variables", {}) or {} in_container = _to_bool(kwargs.get("in_container", False), strict=False) + # Exception: these two reach a lookup as raw group_vars templates, and an unrendered + # `{{ ... }}` string is neither true nor a known mode — the rig would silently keep the + # configured relay domain (a .onion on an onion node, which msmtp cannot route) instead + # of the loopback this resolver exists to hand back. + templar = getattr(self, "_templar", None) + variables = dict(variables) + for key in ("DOCKER_IN_CONTAINER", "DEPLOYMENT_MODE"): + if key in variables: + variables[key] = resolve_var(templar, variables[key]) return [resolve_smtp_host(variables, email, in_container=in_container)] diff --git a/roles/sys-svc-mail-msmtp/templates/msmtprc.conf.j2 b/roles/sys-svc-mail-msmtp/templates/msmtprc.conf.j2 index 996f376267..7c045f3c91 100644 --- a/roles/sys-svc-mail-msmtp/templates/msmtprc.conf.j2 +++ b/roles/sys-svc-mail-msmtp/templates/msmtprc.conf.j2 @@ -1,5 +1,6 @@ {% set email = lookup('email') %} {% set ca_injected = sys_svc_mail_msmtp_in_container | bool and lookup('ca_injected', application_id) %} +{% set smtp_target = lookup('smtp_host', email, in_container=sys_svc_mail_msmtp_in_container) %} defaults # Exception: strict servers (Stalwart rejectNonFqdn) refuse a bare container hostname as EHLO. domain {{ DOMAIN_PRIMARY }} @@ -13,8 +14,21 @@ tls {{ email.tls | on_off }} tls_trust_file {{ CA_TRUST.inject_cert_container }} {% endif %} tls_host_override {{ email.host }} +{% if smtp_target.endswith('.onion') %} +{# Exception: gated on the host actually dialled, NOT on email.host — inside the rig + smtp_host hands back loopback even when the provider's canonical is an onion, and + routing 127.0.0.1 through Tor hangs until the send times out (rc=124). A real .onion + peer has no route off the node, so it must go through Tor's SOCKS5 listener. The + endpoint comes from lookup('tor_socks_proxy'), which owns the swarm-vs-compose host + split, so that policy is not restated here; the host render reaches the same listener + on loopback. Both lookups sit inside this branch because they raise when svc-net-tor + is absent, which is every clearnet deployment. #} +{% set socks = (lookup('tor_socks_proxy') | regex_replace('^socks5h://', '')) if sys_svc_mail_msmtp_in_container | bool else lookup('tor_socks') %} +proxy_host {{ socks.split(':')[0] }} +proxy_port {{ socks.split(':')[1] }} +{% endif %} account system_email_no_reply -host {{ lookup('smtp_host', email, in_container=sys_svc_mail_msmtp_in_container) }} +host {{ smtp_target }} port {{ email.port }} from {{ email['from'] }} {% if (email.auth | bool) %} diff --git a/roles/web-app-keycloak/templates/import/clients/stalwart/webui.json.j2 b/roles/web-app-keycloak/templates/import/clients/stalwart/webui.json.j2 index dcfe8b333e..a0afbb1eb0 100644 --- a/roles/web-app-keycloak/templates/import/clients/stalwart/webui.json.j2 +++ b/roles/web-app-keycloak/templates/import/clients/stalwart/webui.json.j2 @@ -1,7 +1,7 @@ { "clientId": "{{ KEYCLOAK_STALWART_WEBUI_CLIENT_ID }}", "name": "Stalwart WebAdmin", - "description": "Public SPA client for Stalwart WebAdmin (username-first, PKCE S256).", + "description": "Public SPA client for Stalwart WebAdmin (username-first).", "enabled": true, "surrogateAuthRequired": false, "alwaysDisplayInConsole": false, @@ -19,7 +19,11 @@ "frontchannelLogout": false, "protocol": "openid-connect", "attributes": { - "pkce.code.challenge.method": "S256", + {# Exception: `pkce.code.challenge.method` is deliberately NOT pinned. Pinning S256 makes + Keycloak reject the WebAdmin's authorize request outright — "Invalid parameter: code + challenge method is not matching the configured one" — because the SPA is compiled into + the Stalwart image and picks its own method, which this role cannot choose. Left unset, + Keycloak accepts whatever the client presents and still enforces the challenge. #} "post.logout.redirect.uris": "{{ lookup('tls', 'web-app-stalwart', 'url.base') | regex_replace('/+$', '') }}/*", "oauth2.device.authorization.grant.enabled": "false", "oidc.ciba.grant.enabled": "false", diff --git a/roles/web-app-keycloak/vars/main.yml b/roles/web-app-keycloak/vars/main.yml index b12780d452..3c2e405fd0 100644 --- a/roles/web-app-keycloak/vars/main.yml +++ b/roles/web-app-keycloak/vars/main.yml @@ -130,7 +130,10 @@ KEYCLOAK_STALWART_ENABLED: "{{ 'web-app-stalwart' in group_names }}" KEYCLOAK_STALWART_WEBUI_CLIENT_ID: "{{ lookup('config', 'web-app-stalwart', 'services.sso.oidc.webui_client_id') }}" KEYCLOAK_STALWART_WEBMAIL_CLIENT_ID: "{{ lookup('config', 'web-app-stalwart', 'services.sso.oidc.webmail_client_id') }}" -KEYCLOAK_STALWART_WEBMAIL_BASE: "{{ lookup('tls', 'web-app-stalwart', 'protocols.web') ~ '://' ~ lookup('config', 'web-app-stalwart', 'domains.canonical.webmail') }}" +# Exception: canonical_url, not a hand-built scheme ~ '://' ~ raw-canonical — the raw declaration +# is clearnet-only, so on an onion node the client's redirectUris named a host the browser never +# arrives from and Keycloak rejected the callback as a redirect_uri mismatch. +KEYCLOAK_STALWART_WEBMAIL_BASE: "{{ lookup('canonical_url', 'web-app-stalwart', 'webmail') }}" KEYCLOAK_STALWART_WEBMAIL_REDIRECT_URIS: - "{{ KEYCLOAK_STALWART_WEBMAIL_BASE }}/*" KEYCLOAK_STALWART_WEBMAIL_WEB_ORIGINS: diff --git a/roles/web-app-stalwart/files/playwright/test-mailflow.js b/roles/web-app-stalwart/files/playwright/test-mailflow.js index 4b6805e209..f554d4d747 100644 --- a/roles/web-app-stalwart/files/playwright/test-mailflow.js +++ b/roles/web-app-stalwart/files/playwright/test-mailflow.js @@ -10,13 +10,14 @@ const { biberUsername, biberPassword, } = require("./env"); -const { resolveTimeout } = require("./timeouts"); +const { resolveTimeout, isSplitRealmOidc } = require("./timeouts"); // biber -> administrator send/receive through the Roundcube webmail UI. // Login is via Keycloak SSO (Roundcube XOAUTH2 -> Stalwart), mirroring // web-app-mailu. biber and the administrator are separate people: isolated // browser contexts. test("stalwart: biber sends to administrator, administrator receives it", async ({ browser }) => { + test.skip(isSplitRealmOidc(), "clearnet app with an onion OIDC issuer: unreachable from one browser"); safeSkipUnlessEnabled("sso"); // Exception: the env template always renders these — a missing value is a // rendering regression and MUST fail, not silently skip the flagship scenario. @@ -25,8 +26,17 @@ test("stalwart: biber sends to administrator, administrator receives it", async expect(adminPassword, "ADMIN_PASSWORD must be set").toBeTruthy(); const testSubject = `Playwright stalwart ${Date.now()}`; - const biberContext = await browser.newContext({ ignoreHTTPSErrors: true }); - const adminContext = await browser.newContext({ ignoreHTTPSErrors: true }); + // Exception: the proxy has to be repeated here. playwright.config.js sets `use.proxy`, + // which reaches the default `page` fixture only — a context built by hand inherits none + // of `use`, so on an onion node these two personas had no route to the .onion and every + // navigation sat on a blank page until the step timed out. + const proxyServer = process.env.PLAYWRIGHT_PROXY; + const contextOptions = { + ignoreHTTPSErrors: true, + ...(proxyServer ? { proxy: { server: proxyServer } } : {}), + }; + const biberContext = await browser.newContext(contextOptions); + const adminContext = await browser.newContext(contextOptions); try { const biberPage = await biberContext.newPage(); diff --git a/roles/web-app-stalwart/files/playwright/test-sso.js b/roles/web-app-stalwart/files/playwright/test-sso.js index 844213564f..3bfa4be8e3 100644 --- a/roles/web-app-stalwart/files/playwright/test-sso.js +++ b/roles/web-app-stalwart/files/playwright/test-sso.js @@ -1,6 +1,7 @@ const { test, expect } = require("@playwright/test"); const { performKeycloakLoginForm, safeSkipUnlessEnabled, gotoOnion } = require("./personas"); +const { isSplitRealmOidc } = require("./timeouts"); const { appBaseUrl, canonicalDomain, @@ -14,6 +15,7 @@ const { resolveTimeout } = require("./timeouts"); // WebAdmin SSO: a username-first OAuth flow — enter the email, and (the mail // domain being bound to our OIDC directory) the webui redirects to Keycloak. test("stalwart: sso login, open WebAdmin, logout", async ({ page }) => { + test.skip(isSplitRealmOidc(), "clearnet app with an onion OIDC issuer: unreachable from one browser"); safeSkipUnlessEnabled("sso"); await gotoOnion(page, `${appBaseUrl}/`); diff --git a/roles/web-app-stalwart/files/playwright/test-tls.js b/roles/web-app-stalwart/files/playwright/test-tls.js index c05724f4f8..5839f6b463 100644 --- a/roles/web-app-stalwart/files/playwright/test-tls.js +++ b/roles/web-app-stalwart/files/playwright/test-tls.js @@ -2,7 +2,7 @@ const { test, expect } = require("@playwright/test"); const { appBaseUrl, canonicalDomain } = require("./env"); const { gotoOnion } = require("./personas"); -const { resolveTimeout } = require("./timeouts"); +const { resolveTimeout, isOnionTarget } = require("./timeouts"); // Baseline reachability: WebAdmin TLS + DAV auto-discovery through the proxy. test("stalwart: WebAdmin is served under canonical domain with TLS", async ({ page }) => { @@ -10,7 +10,11 @@ test("stalwart: WebAdmin is served under canonical domain with TLS", async ({ pa expect(response, "Expected Stalwart response").toBeTruthy(); expect(response.status(), "Expected status < 400").toBeLessThan(400); expect(response.url().includes(canonicalDomain), `Expected canonical domain "${canonicalDomain}"`).toBe(true); - expect(response.headers()["strict-transport-security"], "Stalwart must emit HSTS").toBeTruthy(); + // Exception: an onion vhost is plain HTTP — Tor carries the encryption and no authority + // issues a certificate for a v3 address — so HSTS is a clearnet-only guarantee. + if (!isOnionTarget()) { + expect(response.headers()["strict-transport-security"], "Stalwart must emit HSTS").toBeTruthy(); + } }); test("stalwart: CalDAV/CardDAV discovery is reachable", async ({ request }) => { diff --git a/roles/web-app-stalwart/meta/services.yml b/roles/web-app-stalwart/meta/services.yml index 9d19bc5c43..afb9aca4b0 100644 --- a/roles/web-app-stalwart/meta/services.yml +++ b/roles/web-app-stalwart/meta/services.yml @@ -148,7 +148,7 @@ prometheus: shared: "{{ 'web-app-prometheus' in group_names }}" # nocheck: playwright-service-flag # nocheck: variants-coverage — the onion is never pinned on for this role; see the sso-variant-tor note in meta/variants.yml -# nocheck: bond-baseline — same reason: an onion-primary mail provider hands every consumer a .onion `email.host` that MX cannot name and msmtp cannot route, so no variant may pin it on +# nocheck: bond-baseline — same reason: while the hidden service forwards no mail port, an onion-primary mail provider hands every consumer an `email.host` nothing answers on tor: bond: 0.5 enabled: "{{ 'svc-net-tor' in group_names }}" diff --git a/roles/web-app-stalwart/meta/variants.yml b/roles/web-app-stalwart/meta/variants.yml index 0e1072225c..1990f8c079 100644 --- a/roles/web-app-stalwart/meta/variants.yml +++ b/roles/web-app-stalwart/meta/variants.yml @@ -2,16 +2,16 @@ # Matrix-deploy variant overrides for web-app-stalwart. # See docs/contributing/artefact/files/role/variants.md for the file format. -# nocheck: sso-variant-tor — the onion is pinned OFF here, not on. `email.host` resolves to the -# ACTIVE provider's primary canonical, so an onion-primary Stalwart hands every consumer -# `mail.<56 chars>.onion`: MX records cannot name a .onion, the role publishes 25/465 (never -# 587, which the non-TLS branch of the port logic selects), and -# roles/sys-svc-mail-msmtp/templates/msmtprc.conf.j2 configures no SOCKS proxy, so the system -# relay cannot route there at all — hlth-msmtp then fails the deploy with EX_TEMPFAIL. The flag -# is pinned rather than left dynamic because an unrendered `{{ ... in group_names }}` reads -# truthy to the domain-map injector while the role's own lookup still resolves clearnet, which -# publishes onion vhost names nothing serves (hlth-webserver: 404). web-app-mailu can pin it on -# only because it is not the active provider, so its onion never becomes `email.host`. +# nocheck: sso-variant-tor — the onion is pinned OFF, and pinned rather than left dynamic. The +# blocker is plugins/lookup/tor_ports.py: it gates HiddenServicePort on the RAW services.yml +# `tor.enabled`, an unrendered Jinja string for every role, so the hidden service forwards no mail +# port and an onion-primary provider hands every consumer an `email.host` nothing answers on. +# (The transport half is done: msmtprc.conf.j2 dials a `.onion` target through Tor's SOCKS +# listener.) A second reason to pin rather than leave dynamic: an unrendered flag reads truthy to +# the domain-map injector while the role's own lookup still resolves clearnet, publishing onion +# vhost names nothing serves. Pinning it ON also skips 05_tls.yml (an onion vhost has no TLS), and +# with it sys-ctl-mtn-cert-deploy, whose `required_by: [web-app-stalwart]` then fails the deploy. +# web-app-mailu can pin it on only because it is never the active provider. # Two variants pin every dynamic services.yml flag on then off, so the matrix # exercises both the in-group_names and not-in-group_names branches (required by diff --git a/roles/web-app-stalwart/tasks/11_oidc_restart.yml b/roles/web-app-stalwart/tasks/11_oidc_restart.yml index 50e4352971..daa96fd4b6 100644 --- a/roles/web-app-stalwart/tasks/11_oidc_restart.yml +++ b/roles/web-app-stalwart/tasks/11_oidc_restart.yml @@ -5,7 +5,8 @@ - name: "⏳ Wait for the OIDC issuer discovery endpoint" ansible.builtin.shell: >- container exec -i {{ STALWART_CONTAINER_ADDRESS }} - {{ 30 | curl }} -ksS -o /dev/null -w "%{http_code}" {{ OIDC.CLIENT.DISCOVERY_DOCUMENT }} + {{ 30 | curl }} -ksS -o /dev/null -w "%{http_code}" + --proxy {{ STALWART_OIDC_SOCKS_PROXY | quote }} {{ OIDC.CLIENT.DISCOVERY_DOCUMENT }} register: stalwart_oidc_issuer_probe changed_when: false retries: 30 diff --git a/roles/web-app-stalwart/templates/compose.yml.j2 b/roles/web-app-stalwart/templates/compose.yml.j2 index 2b77a4487e..1ebe192595 100644 --- a/roles/web-app-stalwart/templates/compose.yml.j2 +++ b/roles/web-app-stalwart/templates/compose.yml.j2 @@ -6,6 +6,7 @@ {{ lookup('container_image', application_id, service_name) }} hostname: {{ container_hostname }} {% include 'roles/sys-svc-container/templates/base.yml.j2' %} + {{ lookup('container_extra_hosts') | indent(4) }} {% if STALWART_ACTIVE_PROVIDER | bool %} {{ lookup('container_ports', ['stalwart', 'http', DOCKER_BIND_HOST], @@ -32,6 +33,7 @@ {{ lookup('compose_only', 'container_name', STALWART_CONTAINER ~ '_webmail') }} {{ lookup('container_image', application_id, service_name) }} {% include 'roles/sys-svc-container/templates/base.yml.j2' %} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_ports', ['webmail', 'http', DOCKER_BIND_HOST]) | indent(4) }} environment: ROUNDCUBEMAIL_DEFAULT_HOST: "ssl://stalwart" diff --git a/roles/web-app-stalwart/templates/roundcube-oauth.inc.php.j2 b/roles/web-app-stalwart/templates/roundcube-oauth.inc.php.j2 index 99419ef057..759e713015 100644 --- a/roles/web-app-stalwart/templates/roundcube-oauth.inc.php.j2 +++ b/roles/web-app-stalwart/templates/roundcube-oauth.inc.php.j2 @@ -6,10 +6,15 @@ // Rendered even without SSO (the bind source must exist); the body self-gates. {% if STALWART_SSO_ENABLED | bool %} -// Reached only over TLS via the reverse proxy (which speaks HTTP to this -// container); force https so generated URLs (incl. the OAuth redirect_uri, which -// Keycloak validates against https:// only) don't come out as http://. +// Exception: forced only when the vhost really is https. The proxy speaks HTTP to this +// container, so without this Roundcube builds the OAuth redirect_uri as http:// and Keycloak +// rejects it -- but an onion vhost IS plain HTTP (Tor carries the encryption, and no authority +// issues a certificate for a v3 address), where forcing it sends an https:// redirect_uri that +// was never registered: "Invalid parameter: redirect_uri". Derived from the same canonical the +// Keycloak client's redirectUris are built from, so the two cannot drift apart. +{% if STALWART_WEBMAIL_BASE_URL.startswith('https://') %} $_SERVER['HTTPS'] = 'on'; +{% endif %} $config['oauth_provider'] = 'generic'; $config['oauth_provider_name'] = '{{ OIDC.BUTTON_TEXT }}'; @@ -21,6 +26,15 @@ $config['oauth_token_uri'] = '{{ OIDC.CLIENT.TOKEN_URL }}'; $config['oauth_identity_uri'] = '{{ OIDC.CLIENT.USER_INFO_URL }}'; $config['oauth_logout_uri'] = '{{ OIDC.CLIENT.LOGOUT_URL }}'; $config['oauth_scope'] = 'openid email profile'; +{% if STALWART_OIDC_SOCKS_PROXY %} +// Exception: the OAuth back-channel (code->token, JWKS, userinfo) is a server-side cURL call, +// and cURL hard-refuses to resolve a .onion name (RFC 7686) whatever /etc/hosts says -- the +// extra_hosts SSO pin fixes the callers that honour /etc/hosts, but cannot help this one. Only +// socks5h gets it through, by handing the name to Tor to resolve instead of resolving locally. +// Merged into every Guzzle client (rcube::get_http_client), and the OAuth call sets no `proxy` +// of its own, so it survives the union. Absent on clearnet, where the gate renders empty. +$config['http_client'] = ['proxy' => '{{ STALWART_OIDC_SOCKS_PROXY }}']; +{% endif %} $config['oauth_auth_parameters'] = []; $config['oauth_identity_fields'] = ['email', '{{ OIDC.ATTRIBUTES.USERNAME }}']; // Auto-redirect the login form straight to Keycloak (single sign-on). diff --git a/roles/web-app-stalwart/vars/main.yml b/roles/web-app-stalwart/vars/main.yml index a1272fa60b..dd13fc3de6 100644 --- a/roles/web-app-stalwart/vars/main.yml +++ b/roles/web-app-stalwart/vars/main.yml @@ -28,6 +28,12 @@ STALWART_HOSTNAME_DNS_ZONE: "{{ STALWART_HOSTNAME | to_zone }}" # create the container; the lookup falls back to the short entity name in that case. container_hostname: "{{ lookup('container_hostname', application_id) }}" +# Exception: curl refuses to resolve a .onion name at all (RFC 7686), so the extra_hosts pin +# that fixes the other back-channel callers cannot help it — an onion issuer must be dialled +# through Tor's SOCKS listener instead. Empty on clearnet, where `--proxy ''` is a curl no-op. +STALWART_OIDC_ONION_ISSUER: "{{ lookup('tls', 'web-app-keycloak', 'domain').endswith('.onion') }}" +STALWART_OIDC_SOCKS_PROXY: "{{ lookup('tor_socks_proxy') if STALWART_OIDC_ONION_ISSUER | bool else '' }}" + STALWART_CONTAINER: "{{ lookup('config', application_id, 'services.stalwart.name') }}" STALWART_ANTIVIRUS_ENABLED: "{{ lookup('config', application_id, 'services.clamav.enabled') | bool }}" @@ -37,7 +43,12 @@ STALWART_CLAMAV_MILTER_TIMEOUT_CONNECT_MS: 5000 STALWART_CLAMAV_MILTER_TIMEOUT_COMMAND_MS: 5000 STALWART_CLAMAV_MILTER_TIMEOUT_DATA_MS: 15000 STALWART_CLAMAV_MILTER_CONF: "{{ [ STALWART_CONFIG_DIR, 'clamav-milter.conf' ] | path_join }}" -STALWART_WEBMAIL_DOMAIN: "{{ lookup('config', application_id, 'domains.canonical.webmail') }}" +# Exception: resolved through canonical_url, NOT `config domains.canonical.webmail` — the raw +# declaration renders `webmail.` and never sees the onion injection, so on an +# onion node the vhost was created under a clearnet name while the domain map (and every probe) +# expected `webmail.`, which then answered 404. +STALWART_WEBMAIL_BASE_URL: "{{ lookup('canonical_url', application_id, 'webmail') }}" +STALWART_WEBMAIL_DOMAIN: "{{ STALWART_WEBMAIL_BASE_URL | regex_replace('^https?://', '') }}" STALWART_WEBMAIL_HTTP_PORT: "{{ lookup('config', application_id, 'services.webmail.ports.local.http') }}" STALWART_WEBMAIL_CONFIG_DIR: "{{ [ STALWART_VOLUMES_DIR, 'webmail' ] | path_join }}" STALWART_WEBMAIL_OAUTH_CONFIG: "{{ [ STALWART_WEBMAIL_CONFIG_DIR, 'oauth.inc.php' ] | path_join }}" diff --git a/tests/unit/python/plugins/lookup/test_email.py b/tests/unit/python/plugins/lookup/test_email.py index ffaa052526..85e8ad6194 100644 --- a/tests/unit/python/plugins/lookup/test_email.py +++ b/tests/unit/python/plugins/lookup/test_email.py @@ -309,6 +309,68 @@ class TestEmailLookup(unittest.TestCase): self.assertTrue(result["auth"]) self.assertFalse(result["start_tls"]) + def test_a_provider_without_plaintext_submission_falls_back_to_its_mx( + self, + ) -> None: + """Stalwart's default config binds the implicit-TLS client ports and the + MX, never 587, so the onion path (which drops TLS) has no submission + listener to reach. The port follows what the provider declares, and AUTH + drops with it: an MX offers none, and announcing it aborts msmtp with + EX_UNAVAILABLE before a message is ever sent.""" + _write_role_config( + self._tmp, + "web-app-mailprov", + {"mailprov": {"ports": {"public": {"smtp": 25, "smtps": 465}}}}, + ) + variables = { + "MAIL_PROVIDER": "web-app-mailprov", + "group_names": ["web-app-mailprov"], + "groups": {"web-app-mailprov": ["host1"]}, + "SYSTEM_EMAIL_HOST": "mail.x.onion", + "TLS_ENABLED": True, + "DOMAIN_PRIMARY": "x.onion", + "inventory_hostname": "host1", + } + result = self.lookup.run( + [], variables=variables, roles_dir=str(self._tmp / "roles") + )[0] + self.assertFalse(result["tls"]) + self.assertEqual(result["port"], 25) + self.assertFalse(result["auth"]) + self.assertEqual(result["auth_mechanism"], "off") + + def test_a_provider_declaring_submission_keeps_the_authenticated_path( + self, + ) -> None: + """Mailu binds 587, so the same onion target stays on plaintext + submission with AUTH PLAIN. The MX fallback above must not reach a + provider that declares a submission listener.""" + _write_role_config( + self._tmp, + "web-app-mailprov", + { + "mailprov": { + "ports": {"public": {"smtp": 25, "smtps": 465, "submission": 587}} + } + }, + ) + variables = { + "MAIL_PROVIDER": "web-app-mailprov", + "group_names": ["web-app-mailprov"], + "groups": {"web-app-mailprov": ["host1"]}, + "SYSTEM_EMAIL_HOST": "mail.x.onion", + "TLS_ENABLED": True, + "DOMAIN_PRIMARY": "x.onion", + "inventory_hostname": "host1", + } + result = self.lookup.run( + [], variables=variables, roles_dir=str(self._tmp / "roles") + )[0] + self.assertFalse(result["tls"]) + self.assertEqual(result["port"], 587) + self.assertTrue(result["auth"]) + self.assertEqual(result["auth_mechanism"], "plain") + def test_sso_relay_active_with_untemplated_enabled_gate(self) -> None: # The role default gates sso.enabled on group_names (raw Jinja here); # the guard must not treat the untemplated string as false. diff --git a/tests/unit/python/plugins/lookup/test_smtp_host.py b/tests/unit/python/plugins/lookup/test_smtp_host.py index c765f22bc4..ad79655a11 100644 --- a/tests/unit/python/plugins/lookup/test_smtp_host.py +++ b/tests/unit/python/plugins/lookup/test_smtp_host.py @@ -37,6 +37,21 @@ class _DummyTemplar: return value +class _RenderingTemplar: + """A templar that actually resolves expressions, like the real one. + + ``_DummyTemplar`` returns its input unchanged, so it cannot expose a value + that only misbehaves while still an unrendered template. + """ + + def __init__(self, available_variables=None, rendered=None): + self.available_variables = available_variables or {} + self._rendered = rendered or {} + + def template(self, value): + return self._rendered.get(value, value) + + def _email(external=True, host=RELAY): return {"external": external, "host": host} @@ -67,6 +82,18 @@ class TestSmtpHostLookup(unittest.TestCase): vars_ = {"DOCKER_IN_CONTAINER": True, "DEPLOYMENT_MODE": "compose"} self.assertEqual(self._resolve(vars_), "127.0.0.1") + def test_an_unrendered_group_var_still_reaches_the_rig_loopback(self): + """DOCKER_IN_CONTAINER is a group_vars expression, and a lookup receives it + raw. Read unrendered it is neither true nor false, so the rig kept the + configured relay instead of loopback — and on an onion node that relay is a + .onion, which msmtp cannot route (EX_UNAVAILABLE, 'proxy failure').""" + raw = "{{ (ansible_facts['env']['container'] | default('')) != '' }}" + vars_ = {"DOCKER_IN_CONTAINER": raw, "DEPLOYMENT_MODE": "compose"} + lm = self.LookupModule() + lm._templar = _RenderingTemplar(vars_, {raw: True}) + lm._loader = None + self.assertEqual(lm.run([_email()], variables=vars_)[0], "127.0.0.1") + def test_swarm_node_uses_loopback(self): vars_ = { "DOCKER_IN_CONTAINER": True, diff --git a/tests/unit/python/roles/sys-svc-mail-msmtp/test_msmtprc_ca_trust.py b/tests/unit/python/roles/sys-svc-mail-msmtp/test_msmtprc_ca_trust.py index ca80530a3c..d749f92c8a 100644 --- a/tests/unit/python/roles/sys-svc-mail-msmtp/test_msmtprc_ca_trust.py +++ b/tests/unit/python/roles/sys-svc-mail-msmtp/test_msmtprc_ca_trust.py @@ -12,6 +12,7 @@ filter shim, since ``bool`` is an Ansible filter. from __future__ import annotations +import re import unittest from jinja2 import Environment, StrictUndefined @@ -45,17 +46,38 @@ def _env() -> Environment: ) env.filters["bool"] = bool env.filters["on_off"] = lambda value: "on" if value else "off" + env.filters["regex_replace"] = lambda value, pattern, repl="": re.sub( + pattern, repl, str(value) + ) return env -def _render(*, in_container: bool, ca_injected: bool, **extra) -> str: +def _render( + *, in_container: bool, ca_injected: bool, email=None, smtp_host=None, **extra +) -> str: + """Render msmtprc. + + Args: + in_container: whether the rendered file is mounted into a container. + ca_injected: what the ca_injected lookup reports. + email: the email mapping the lookup returns. + smtp_host: the host smtp_host resolves to. Defaults to the configured + relay, which is what the resolver returns outside the rig; pass + loopback to exercise the in-rig path. + """ + email_config = email if email is not None else EMAIL + def lookup(kind, *args, **kwargs): if kind == "email": - return EMAIL + return email_config if kind == "smtp_host": - return "relay" + return smtp_host if smtp_host is not None else email_config["host"] if kind == "ca_injected": return ca_injected + if kind == "tor_socks_proxy": + return "socks5h://tor:9050" + if kind == "tor_socks": + return "127.0.0.1:9050" raise AssertionError(f"unexpected lookup: {kind}") template = _env().from_string(read_text(str(TEMPLATE))) @@ -101,5 +123,60 @@ class TestMsmtprcCaTrust(unittest.TestCase): self.assertIn(trust, (None, CA_CONTAINER), f"host path leaked: {trust}") +class TestMsmtprcOnionProxy(unittest.TestCase): + """Reaching a .onion relay at all. + + A .onion peer has no route off the node, so msmtp has to hand the connection + to Tor's SOCKS5 listener. Without it every send fails "cannot connect ... + Connection refused" and the hlth-msmtp unit takes the deploy down with it. + """ + + ONION_EMAIL = dict(EMAIL, host="mail.abc123.onion", tls=False, port=25) + + def _proxy(self, rendered: str) -> dict[str, str]: + found = {} + for line in rendered.splitlines(): + parts = line.split() + if len(parts) == 2 and parts[0] in ("proxy_host", "proxy_port"): + found[parts[0]] = parts[1] + return found + + def test_a_clearnet_relay_names_no_proxy(self): + rendered = _render(in_container=False, ca_injected=False) + self.assertEqual(self._proxy(rendered), {}) + + def test_an_onion_relay_is_reached_through_the_node_socks_listener(self): + rendered = _render( + in_container=False, ca_injected=False, email=self.ONION_EMAIL + ) + self.assertEqual( + self._proxy(rendered), {"proxy_host": "127.0.0.1", "proxy_port": "9050"} + ) + + def test_a_rig_loopback_target_is_never_routed_through_tor(self): + """Inside the rig smtp_host returns loopback even when the provider's canonical + is an onion. Gating the proxy on email.host instead of the dialled host sent + 127.0.0.1 through Tor, and every send hung until the timeout killed it + (rc=124).""" + rendered = _render( + in_container=False, + ca_injected=False, + email=self.ONION_EMAIL, + smtp_host="127.0.0.1", + ) + self.assertEqual(self._proxy(rendered), {}) + + def test_a_container_render_reaches_tor_over_the_shared_endpoint(self): + rendered = _render( + in_container=True, + ca_injected=False, + email=self.ONION_EMAIL, + application_id="web-app-moodle", + ) + self.assertEqual( + self._proxy(rendered), {"proxy_host": "tor", "proxy_port": "9050"} + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/python/roles/web-app-stalwart/test_roundcube_oauth_scheme.py b/tests/unit/python/roles/web-app-stalwart/test_roundcube_oauth_scheme.py new file mode 100644 index 0000000000..d128cc3f2a --- /dev/null +++ b/tests/unit/python/roles/web-app-stalwart/test_roundcube_oauth_scheme.py @@ -0,0 +1,92 @@ +"""Whether Roundcube forces https on its generated URLs, per canonical scheme. + +Roundcube derives the OAuth ``redirect_uri`` from the request scheme, and the +reverse proxy speaks plain HTTP to the container. Forcing https is therefore +right on a clearnet vhost and wrong on an onion one, which is served over HTTP +because Tor carries the encryption. Getting it wrong is not cosmetic: Keycloak +compares the ``redirect_uri`` against the client's registered list verbatim and +answers "Invalid parameter: redirect_uri" on a scheme mismatch, so the webmail +login dies before the form is ever drawn. + +Both sides are built from ``canonical_url``, so the assertion here is that the +template follows that one source rather than a hard-coded scheme. +""" + +from __future__ import annotations + +import unittest + +from jinja2 import Environment, StrictUndefined + +from utils.cache.files import read_text + +from . import PROJECT_ROOT + +TEMPLATE = ( + PROJECT_ROOT / "roles" / "web-app-stalwart" / "templates" / "roundcube-oauth.inc.php.j2" +) +FORCE_HTTPS = "$_SERVER['HTTPS'] = 'on';" +ONION_BASE = "http://webmail.4dl6yijmuksgh5bfaf3csqoelji7zptyuo6wy5g5prjpnk6vdbzfwnad.onion" +CLEARNET_BASE = "https://webmail.infinito.example" + + +def _render(base_url: str, *, sso_enabled: bool = True, socks_proxy: str = "") -> str: + env = Environment( + trim_blocks=True, + lstrip_blocks=True, + undefined=StrictUndefined, + autoescape=False, # noqa: S701 - a PHP config file, and Ansible's templar does not escape either + ) + env.filters["bool"] = bool + return env.from_string(read_text(TEMPLATE)).render( + STALWART_SSO_ENABLED=sso_enabled, + STALWART_WEBMAIL_BASE_URL=base_url, + STALWART_OIDC_SOCKS_PROXY=socks_proxy, + STALWART_WEBMAIL_OIDC_CLIENT_ID="roundcube", + OIDC={ + "BUTTON_TEXT": "Sign in", + "ATTRIBUTES": {"USERNAME": "preferred_username"}, + "CLIENT": { + "AUTHORIZE_URL": "https://auth.infinito.example/auth", + "TOKEN_URL": "https://auth.infinito.example/token", + "USER_INFO_URL": "https://auth.infinito.example/userinfo", + "LOGOUT_URL": "https://auth.infinito.example/logout", + }, + }, + ) + + +class TestRoundcubeOauthScheme(unittest.TestCase): + def test_a_clearnet_vhost_still_forces_https(self): + self.assertIn(FORCE_HTTPS, _render(CLEARNET_BASE)) + + def test_an_onion_vhost_is_left_on_http(self): + self.assertNotIn(FORCE_HTTPS, _render(ONION_BASE)) + + def test_the_scheme_is_never_decided_by_the_onion_suffix_alone(self): + self.assertIn(FORCE_HTTPS, _render("https://webmail.example.onion")) + + def test_an_onion_issuer_routes_the_back_channel_through_tor(self): + rendered = _render(ONION_BASE, socks_proxy="socks5h://host.docker.internal:9050") + self.assertIn( + "$config['http_client'] = ['proxy' => 'socks5h://host.docker.internal:9050'];", + rendered, + ) + + def test_the_proxy_must_hand_the_name_to_tor_to_resolve(self): + """socks5:// resolves locally, which is what cURL refuses to do for .onion.""" + rendered = _render(ONION_BASE, socks_proxy="socks5h://host.docker.internal:9050") + self.assertNotIn("'proxy' => 'socks5://", rendered) + + def test_a_clearnet_issuer_gets_no_proxy_at_all(self): + self.assertNotIn("http_client", _render(CLEARNET_BASE)) + + def test_nothing_is_emitted_without_sso(self): + rendered = _render(CLEARNET_BASE, sso_enabled=False, socks_proxy="socks5h://tor:9050") + self.assertNotIn(FORCE_HTTPS, rendered) + self.assertNotIn("oauth_client_id", rendered) + self.assertNotIn("http_client", rendered) + + +if __name__ == "__main__": + unittest.main()