fix(mail): stop a refused smoke mail from marking a live container dead

The msmtp_curl probe joined its mail branch to the curl probe with '&&', so a
relay that refuses the startup message short-circuited the whole command: curl
never ran, the container stayed unhealthy forever, and the swarm converge gate
burned all 150 retries before aborting the deploy. Friendica 0 and Fediwall 0
of run 31340965955 died that way, reporting only 'not converged:
friendica_friendica' with the real cause buried in a rescue artifact.

That contradicted the probe's own docstring, which promises an unrelated SMTP
outage cannot flap the container into unhealthy -- it only delivered that for
email_enabled=False. sys-ctl-hlth-msmtp/files/script.sh:23 has always tolerated
exactly these exit codes; the container probe was the outlier.

Liveness is now curl's verdict alone. The '&&' before touch stays, so a refused
mail leaves no marker and the next probe retries it rather than turning the
smoke test one-shot.

Detection does not disappear with the coupling: web-app-friendica now asserts
the delivery marker after convergence and fails with the relay's reason instead
of a timeout three layers away. sys-ctl-hlth-msmtp is a systemd timer and the
Mailu Playwright roundtrip submits through Roundcube, so neither covers a
container relaying over port 25 -- this assertion is what does.

The marker path moves into utils.docker.healthcheck.MAIL_MARKER and reaches
Ansible through lookup('container_mail_marker'), so the probe and the assertion
cannot drift apart.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

plugins/lookup/container_mail_marker.py

@@ -0,0 +1,31 @@
+"""Lookup `container_mail_marker`: path of the msmtp_curl delivery marker.
+
+Single SPOT for the file the ``msmtp_curl`` healthcheck touches after msmtp
+exits 0. A deploy-time assertion that the relay accepted the startup mail has
+to name the same path the probe writes, and a second literal would drift the
+moment the probe moves it.
+
+Usage:
+    {{ lookup('container_mail_marker') }}   -> /tmp/email_sent
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ansible.errors import AnsibleError
+from ansible.plugins.lookup import LookupBase
+
+from utils.docker.healthcheck import MAIL_MARKER
+
+
+class LookupModule(LookupBase):
+    def run(
+        self,
+        terms: list[Any] | None,
+        variables: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ) -> list[str]:
+        if terms:
+            raise AnsibleError("lookup('container_mail_marker') expects no terms.")
+        return [MAIL_MARKER]

roles/web-app-friendica/tasks/01_manager_ops.yml

@@ -51,3 +51,27 @@
 - name: "🎬 Bootstrap Friendica admin user (only when no LDAP provider provisions them)"
   include_tasks: 05_user_bootstrap.yml
   when: not (FRIENDICA_LDAP_ENABLED | bool)
+
+- name: "📧 Verify the relay accepted Friendica's startup mail"
+  when: lookup('config', application_id, 'services.email.enabled', false) | bool
+  block:
+    - name: "⏳ Wait for the delivery marker the healthcheck writes"
+      shell: > # nocheck: container-exec-resolver  FRIENDICA_LOCAL_CID resolved at delegate_to runtime
+        container exec {{ FRIENDICA_LOCAL_CID | quote }} test -f {{ lookup('container_mail_marker') }}
+      delegate_to: "{{ FRIENDICA_HOST_NODE }}"
+      register: friendica_smoke_mail
+      changed_when: false
+      failed_when: false
+      retries: 30
+      delay: 5
+      until: friendica_smoke_mail.rc == 0
+
+    - name: "🛑 Report a relay that refused every attempt"
+      ansible.builtin.assert:
+        that: friendica_smoke_mail.rc == 0
+        fail_msg: >-
+          Friendica never delivered its startup mail. The healthcheck writes
+          {{ lookup('container_mail_marker') }} only after msmtp exits 0, so
+          the relay refused every attempt. Its reason is in the container's
+          State.Health.Log; liveness itself is unaffected, which is why the
+          stack still converged.

tests/unit/utils/docker/test_healthcheck.py

@@ -1,13 +1,49 @@
+import os
+import subprocess
+import tempfile
 import unittest
+from pathlib import Path
 
 from utils.cache.yaml import load_yaml_str
 from utils.docker.healthcheck import PROBES, Curl, HttpStatus, MsmtpCurl, Nc, Tcp, build
 
+_MARKER = "/tmp/email_sent"
+
 
 def probe(flavor, **context):
     return PROBES[flavor](**context).test()
 
 
+def run_probe(argv, *, msmtp_rc, curl_rc):
+    """Execute a CMD-SHELL probe against stubbed msmtp and curl.
+
+    Args:
+        argv: the probe as returned by :func:`probe`.
+        msmtp_rc: exit code the msmtp stub returns.
+        curl_rc: exit code the curl stub returns.
+
+    Returns:
+        ``(exit_code, marker_written)`` for the probe as the container runs it.
+    """
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        stubs = root / "bin"
+        stubs.mkdir()
+        for name, rc in (("msmtp", msmtp_rc), ("curl", curl_rc)):
+            stub = stubs / name
+            stub.write_text(f"#!/bin/sh\ncat >/dev/null 2>&1\nexit {rc}\n")
+            stub.chmod(0o755)
+        marker = root / "email_sent"
+        script = argv[1].replace(_MARKER, str(marker))
+        completed = subprocess.run(
+            ["bash", "-c", script],
+            env={**os.environ, "PATH": f"{stubs}{os.pathsep}{os.environ['PATH']}"},
+            capture_output=True,
+            check=False,
+        )
+        return completed.returncode, marker.exists()
+
+
 def block(flavor, overrides=None, **context):
     return load_yaml_str(build(flavor, overrides or {}, **context))["healthcheck"]
 
@@ -98,6 +134,26 @@ class TestOtherFlavors(unittest.TestCase):
         self.assertIn("msmtp -t void@example", argv[1])
         self.assertTrue(argv[1].endswith("curl -f http://127.0.0.1:80/ || exit 1"))
 
+    def test_msmtp_curl_probe_behaviour(self):
+        argv = probe(
+            "msmtp_curl",
+            port=80,
+            email_enabled=True,
+            domain="app.example",
+            blackhole="void@example",
+        )
+        cases = (
+            ("a refused mail leaves the app healthy and retryable", 69, 0, 0, False),
+            ("an accepted mail marks itself sent", 0, 0, 0, True),
+            ("a dead web server is unhealthy", 0, 1, 1, None),
+        )
+        for label, msmtp_rc, curl_rc, want_rc, want_marker in cases:
+            with self.subTest(label):
+                rc, marker = run_probe(argv, msmtp_rc=msmtp_rc, curl_rc=curl_rc)
+                self.assertEqual(want_rc, rc)
+                if want_marker is not None:
+                    self.assertEqual(want_marker, marker)
+
     def test_msmtp_curl_drops_the_mail_branch_when_email_is_disabled(self):
         """An SMTP outage must not flap the container into unhealthy."""
         self.assertEqual(

utils/docker/healthcheck.py

@@ -14,6 +14,8 @@ from typing import Any, ClassVar
 
 from utils.cache.yaml import dump_yaml_str
 
+MAIL_MARKER = "/tmp/email_sent"  # noqa: S108  container-internal path, not a host tmpfile
+
 _HTTP_REQUEST = (
     "echo -e 'GET /{path} HTTP/1.1\\r\\nHost: localhost\\r\\n"
     "Connection: close\\r\\n\\r\\n' >&3"
@@ -167,6 +169,16 @@ class MsmtpCurl(Probe):
     The /tmp/email_sent marker keeps a repeating probe from tripping SMTP
     rate limits, and a disabled email provider drops the mail branch so an
     unrelated SMTP outage cannot flap the container into unhealthy.
+
+    The branch is joined to the curl probe with ``;`` rather than ``&&`` on
+    purpose: msmtp exits non-zero whenever the relay refuses the message, and
+    chaining that into curl marks a perfectly live web app dead until the
+    swarm converge gate gives up. Liveness is curl's verdict alone. The
+    ``&&`` before ``touch`` stays, so a refused mail leaves no marker and the
+    next probe retries it; its stderr reaches Health.Log either way.
+    Delivery itself is covered where it belongs — by sys-ctl-hlth-msmtp,
+    which retries and then exits non-zero, and by the Mailu Playwright
+    roundtrip.
     """
 
     flavor = "msmtp_curl"
@@ -180,9 +192,9 @@ class MsmtpCurl(Probe):
             domain = self.context.get("domain", "")
             blackhole = self.context.get("blackhole", "")
             mail = (
-                "if [ ! -f /tmp/email_sent ]; then "
+                f"if [ ! -f {MAIL_MARKER} ]; then "
                 f"echo 'Subject: testmessage from {domain}\\n\\nSUCCESSFULL' "
-                f"| msmtp -t {blackhole} && touch /tmp/email_sent; fi && "
+                f"| msmtp -t {blackhole} && touch {MAIL_MARKER}; fi; "
             )
         return ["CMD-SHELL", f"{mail}curl -f {self.url} || exit 1"]