diff --git a/plugins/lookup/container_mail_marker.py b/plugins/lookup/container_mail_marker.py new file mode 100644 index 0000000000..392813ef23 --- /dev/null +++ b/plugins/lookup/container_mail_marker.py @@ -0,0 +1,31 @@ +"""Lookup `container_mail_marker`: path of the msmtp_curl delivery marker. + +Single SPOT for the file the ``msmtp_curl`` healthcheck touches after msmtp +exits 0. A deploy-time assertion that the relay accepted the startup mail has +to name the same path the probe writes, and a second literal would drift the +moment the probe moves it. + +Usage: + {{ lookup('container_mail_marker') }} -> /tmp/email_sent +""" + +from __future__ import annotations + +from typing import Any + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase + +from utils.docker.healthcheck import MAIL_MARKER + + +class LookupModule(LookupBase): + def run( + self, + terms: list[Any] | None, + variables: dict[str, Any] | None = None, + **kwargs: Any, + ) -> list[str]: + if terms: + raise AnsibleError("lookup('container_mail_marker') expects no terms.") + return [MAIL_MARKER] diff --git a/roles/web-app-friendica/tasks/01_manager_ops.yml b/roles/web-app-friendica/tasks/01_manager_ops.yml index c2f9e34fb5..d942b5ccaa 100644 --- a/roles/web-app-friendica/tasks/01_manager_ops.yml +++ b/roles/web-app-friendica/tasks/01_manager_ops.yml @@ -51,3 +51,27 @@ - name: "🎬 Bootstrap Friendica admin user (only when no LDAP provider provisions them)" include_tasks: 05_user_bootstrap.yml when: not (FRIENDICA_LDAP_ENABLED | bool) + +- name: "📧 Verify the relay accepted Friendica's startup mail" + when: lookup('config', application_id, 'services.email.enabled', false) | bool + block: + - name: "⏳ Wait for the delivery marker the healthcheck writes" + shell: > # nocheck: container-exec-resolver FRIENDICA_LOCAL_CID resolved at delegate_to runtime + container exec {{ FRIENDICA_LOCAL_CID | quote }} test -f {{ lookup('container_mail_marker') }} + delegate_to: "{{ FRIENDICA_HOST_NODE }}" + register: friendica_smoke_mail + changed_when: false + failed_when: false + retries: 30 + delay: 5 + until: friendica_smoke_mail.rc == 0 + + - name: "🛑 Report a relay that refused every attempt" + ansible.builtin.assert: + that: friendica_smoke_mail.rc == 0 + fail_msg: >- + Friendica never delivered its startup mail. The healthcheck writes + {{ lookup('container_mail_marker') }} only after msmtp exits 0, so + the relay refused every attempt. Its reason is in the container's + State.Health.Log; liveness itself is unaffected, which is why the + stack still converged. diff --git a/tests/unit/utils/docker/test_healthcheck.py b/tests/unit/utils/docker/test_healthcheck.py index a4f5f3f7a8..c3bdc317e7 100644 --- a/tests/unit/utils/docker/test_healthcheck.py +++ b/tests/unit/utils/docker/test_healthcheck.py @@ -1,13 +1,49 @@ +import os +import subprocess +import tempfile import unittest +from pathlib import Path from utils.cache.yaml import load_yaml_str from utils.docker.healthcheck import PROBES, Curl, HttpStatus, MsmtpCurl, Nc, Tcp, build +_MARKER = "/tmp/email_sent" + def probe(flavor, **context): return PROBES[flavor](**context).test() +def run_probe(argv, *, msmtp_rc, curl_rc): + """Execute a CMD-SHELL probe against stubbed msmtp and curl. + + Args: + argv: the probe as returned by :func:`probe`. + msmtp_rc: exit code the msmtp stub returns. + curl_rc: exit code the curl stub returns. + + Returns: + ``(exit_code, marker_written)`` for the probe as the container runs it. + """ + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + stubs = root / "bin" + stubs.mkdir() + for name, rc in (("msmtp", msmtp_rc), ("curl", curl_rc)): + stub = stubs / name + stub.write_text(f"#!/bin/sh\ncat >/dev/null 2>&1\nexit {rc}\n") + stub.chmod(0o755) + marker = root / "email_sent" + script = argv[1].replace(_MARKER, str(marker)) + completed = subprocess.run( + ["bash", "-c", script], + env={**os.environ, "PATH": f"{stubs}{os.pathsep}{os.environ['PATH']}"}, + capture_output=True, + check=False, + ) + return completed.returncode, marker.exists() + + def block(flavor, overrides=None, **context): return load_yaml_str(build(flavor, overrides or {}, **context))["healthcheck"] @@ -98,6 +134,26 @@ class TestOtherFlavors(unittest.TestCase): self.assertIn("msmtp -t void@example", argv[1]) self.assertTrue(argv[1].endswith("curl -f http://127.0.0.1:80/ || exit 1")) + def test_msmtp_curl_probe_behaviour(self): + argv = probe( + "msmtp_curl", + port=80, + email_enabled=True, + domain="app.example", + blackhole="void@example", + ) + cases = ( + ("a refused mail leaves the app healthy and retryable", 69, 0, 0, False), + ("an accepted mail marks itself sent", 0, 0, 0, True), + ("a dead web server is unhealthy", 0, 1, 1, None), + ) + for label, msmtp_rc, curl_rc, want_rc, want_marker in cases: + with self.subTest(label): + rc, marker = run_probe(argv, msmtp_rc=msmtp_rc, curl_rc=curl_rc) + self.assertEqual(want_rc, rc) + if want_marker is not None: + self.assertEqual(want_marker, marker) + def test_msmtp_curl_drops_the_mail_branch_when_email_is_disabled(self): """An SMTP outage must not flap the container into unhealthy.""" self.assertEqual( diff --git a/utils/docker/healthcheck.py b/utils/docker/healthcheck.py index 7b6e0dd07e..557a650403 100644 --- a/utils/docker/healthcheck.py +++ b/utils/docker/healthcheck.py @@ -14,6 +14,8 @@ from typing import Any, ClassVar from utils.cache.yaml import dump_yaml_str +MAIL_MARKER = "/tmp/email_sent" # noqa: S108 container-internal path, not a host tmpfile + _HTTP_REQUEST = ( "echo -e 'GET /{path} HTTP/1.1\\r\\nHost: localhost\\r\\n" "Connection: close\\r\\n\\r\\n' >&3" @@ -167,6 +169,16 @@ class MsmtpCurl(Probe): The /tmp/email_sent marker keeps a repeating probe from tripping SMTP rate limits, and a disabled email provider drops the mail branch so an unrelated SMTP outage cannot flap the container into unhealthy. + + The branch is joined to the curl probe with ``;`` rather than ``&&`` on + purpose: msmtp exits non-zero whenever the relay refuses the message, and + chaining that into curl marks a perfectly live web app dead until the + swarm converge gate gives up. Liveness is curl's verdict alone. The + ``&&`` before ``touch`` stays, so a refused mail leaves no marker and the + next probe retries it; its stderr reaches Health.Log either way. + Delivery itself is covered where it belongs — by sys-ctl-hlth-msmtp, + which retries and then exits non-zero, and by the Mailu Playwright + roundtrip. """ flavor = "msmtp_curl" @@ -180,9 +192,9 @@ class MsmtpCurl(Probe): domain = self.context.get("domain", "") blackhole = self.context.get("blackhole", "") mail = ( - "if [ ! -f /tmp/email_sent ]; then " + f"if [ ! -f {MAIL_MARKER} ]; then " f"echo 'Subject: testmessage from {domain}\\n\\nSUCCESSFULL' " - f"| msmtp -t {blackhole} && touch /tmp/email_sent; fi && " + f"| msmtp -t {blackhole} && touch {MAIL_MARKER}; fi; " ) return ["CMD-SHELL", f"{mail}curl -f {self.url} || exit 1"]