fix(mail): retry a connection the mail server drops before its greeting

Stalwart closes a connection that exceeds its per-IP connection budget
without a greeting. msmtp reports that as "the server sent an empty reply"
with exit 76, and the mail health check treated exit 76 as permanent. One
such drop, in a second in which Alertmanager flushed its notifications,
failed a deploy although the same path accepted a mail one second later.

The health check now retries exit 76 when msmtp names the empty reply,
within the ten attempts it already gives to exit 69, 75 and 124. Every
other protocol error still ends the check at once.

The alarm mail was sent once, so the alarm about a failed unit was lost to
the same drop. It now makes three attempts ten seconds apart for the
transient cases of the health check and still reports the last error.

The limit on the server is left alone. Exempting the node's own senders by
source address would also exempt Tor clients and, in swarm, every external
client. That question is tracked in #793.

Closes #790

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

roles/sys-ctl-alm-email/templates/script.sh.j2

@@ -39,8 +39,38 @@ $STATUS_OUT
 
 ERRMAIL
 
-timeout "{{ email.timeout }}s" "$sendmail_bin" -t < "$mail_tmp"
-rc=$?
+attempts=3
+delay=10
+
+for attempt in $(seq 1 "${attempts}"); do
+  send_err="$(timeout "{{ email.timeout }}s" "$sendmail_bin" -t < "$mail_tmp" 2>&1 1>/dev/null)"
+  rc=$?
+  [ "$rc" -eq 0 ] && exit 0
+
+  case "$rc" in
+    69 | 75 | 124 | 137) ;;
+    76)
+      case "$send_err" in
+        *"the server sent an empty reply"*) ;;
+        *) break ;;
+      esac
+      ;;
+    77)
+      case "$send_err" in
+        *"server message: 4"* | *"Temporary authentication failure"*) ;;
+        *) break ;;
+      esac
+      ;;
+    *) break ;;
+  esac
+
+  if [ "$attempt" -lt "$attempts" ]; then
+    echo "sendmail transient failure (rc=${rc}), attempt ${attempt}/${attempts}; retrying in ${delay}s" >&2
+    sleep "$delay"
+  fi
+done
+
+[ -n "$send_err" ] && echo "$send_err" >&2
 
 if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then
   echo "ERROR: sendmail timed out after {{ email.timeout }}s" >&2

roles/sys-ctl-hlth-msmtp/files/shell/script.sh

@@ -21,6 +21,12 @@ for attempt in $(seq 1 "${attempts}"); do
 
   case "${rc}" in
     69 | 75 | 124) ;;
+    76)
+      case "${msmtp_err}" in
+        *"the server sent an empty reply"*) ;;
+        *) exit "${rc}" ;;
+      esac
+      ;;
     77)
       case "${msmtp_err}" in
         *"server message: 4"* | *"Temporary authentication failure"*) ;;

tests/unit/python/roles/sys-ctl-alm-email/templates/test_script.py

@@ -0,0 +1,123 @@
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+from jinja2 import Environment, FileSystemLoader, StrictUndefined
+
+from utils import PROJECT_ROOT
+
+TEMPLATE_DIR = PROJECT_ROOT / "roles/sys-ctl-alm-email/templates"
+
+EMPTY_REPLY = "sendmail: the server sent an empty reply"
+
+SENDMAIL_STUB = """#!/usr/bin/env bash
+cat >/dev/null
+calls=$(( $(cat "${STUB_DIR}/calls" 2>/dev/null || echo 0) + 1 ))
+echo "${calls}" >"${STUB_DIR}/calls"
+if [ "${calls}" -gt "${STUB_FAILURES}" ]; then
+\texit 0
+fi
+echo "${STUB_MESSAGE}" >&2
+exit "${STUB_RC}"
+"""
+
+QUIET_STUB = """#!/usr/bin/env bash
+exit 0
+"""
+
+ECHO_STUB = """#!/usr/bin/env bash
+printf '%s\\n' "${@: -1}"
+"""
+
+
+def lookup(name, *_args):
+    """Return what the template's two lookups resolve to.
+
+    Args:
+        name: lookup plugin the template calls.
+    """
+    return {
+        "email": {"from": "no-reply@example.test", "timeout": "30"},
+        "users": {"email": "administrator@example.test"},
+    }[name]
+
+
+def rendered_script() -> str:
+    env = Environment(
+        loader=FileSystemLoader(str(TEMPLATE_DIR)),
+        undefined=StrictUndefined,
+        autoescape=False,  # noqa: S701 - shell script, not markup; Ansible renders it the same way
+    )
+    return env.get_template("script.sh.j2").render(lookup=lookup, HOST_CS="utf-8")
+
+
+class TestAlmEmailScript(unittest.TestCase):
+    def run_script(self, *, failures, rc, message):
+        """Run the rendered alarm script against a stubbed sendmail.
+
+        Args:
+            failures: how many sendmail calls fail before one succeeds.
+            rc: exit code of a failing call.
+            message: what a failing call prints to stderr.
+
+        Returns:
+            Tuple of the exit code, the retry lines and the whole stderr.
+        """
+        with tempfile.TemporaryDirectory() as tmp:
+            bin_dir = Path(tmp)
+            stubs = (
+                ("sendmail", SENDMAIL_STUB),
+                ("sleep", QUIET_STUB),
+                ("systemctl", QUIET_STUB),
+                ("systemd-escape", ECHO_STUB),
+            )
+            for name, body in stubs:
+                stub = bin_dir / name
+                stub.write_text(body)
+                stub.chmod(0o755)
+            script = bin_dir / "script.sh"
+            script.write_text(rendered_script())
+            env = dict(os.environ)
+            env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
+            env["STUB_DIR"] = tmp
+            env["STUB_FAILURES"] = str(failures)
+            env["STUB_RC"] = str(rc)
+            env["STUB_MESSAGE"] = message
+            done = subprocess.run(
+                ["bash", str(script), "example.service"],
+                env=env,
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+            retries = [
+                line
+                for line in done.stderr.splitlines()
+                if "sendmail transient failure" in line
+            ]
+            return done.returncode, retries, done.stderr
+
+    def test_a_dropped_connection_is_retried_and_the_alarm_is_sent(self):
+        code, retries, _ = self.run_script(failures=1, rc=76, message=EMPTY_REPLY)
+        self.assertEqual(code, 0)
+        self.assertEqual(len(retries), 1)
+        self.assertIn("(rc=76), attempt 1/3", retries[0])
+
+    def test_a_permanent_rejection_is_reported_without_a_retry(self):
+        rejection = "sendmail: recipient address rejected"
+        code, retries, stderr = self.run_script(failures=99, rc=65, message=rejection)
+        self.assertEqual(code, 65)
+        self.assertEqual(retries, [])
+        self.assertIn(rejection, stderr)
+
+    def test_the_third_failure_ends_the_alarm_with_the_last_exit_code(self):
+        code, retries, stderr = self.run_script(failures=99, rc=76, message=EMPTY_REPLY)
+        self.assertEqual(code, 76)
+        self.assertEqual(len(retries), 2)
+        self.assertIn(EMPTY_REPLY, stderr)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/python/roles/sys-ctl-hlth-msmtp/files/test_script.py

@@ -0,0 +1,91 @@
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+from utils import PROJECT_ROOT
+
+SCRIPT = PROJECT_ROOT / "roles/sys-ctl-hlth-msmtp/files/shell/script.sh"
+
+EMPTY_REPLY = "msmtp: the server sent an empty reply"
+
+MSMTP_STUB = """#!/usr/bin/env bash
+cat >/dev/null
+calls=$(( $(cat "${STUB_DIR}/calls" 2>/dev/null || echo 0) + 1 ))
+echo "${calls}" >"${STUB_DIR}/calls"
+if [ "${calls}" -gt "${STUB_FAILURES}" ]; then
+\texit 0
+fi
+echo "${STUB_MESSAGE}" >&2
+exit "${STUB_RC}"
+"""
+
+SLEEP_STUB = """#!/usr/bin/env bash
+exit 0
+"""
+
+
+class TestHlthMsmtpScript(unittest.TestCase):
+    def run_script(self, *, failures, rc, message):
+        """Run the health script against a stubbed msmtp.
+
+        Args:
+            failures: how many msmtp calls fail before one succeeds.
+            rc: exit code of a failing call.
+            message: what a failing call prints to stderr.
+
+        Returns:
+            Tuple of the exit code and the retry lines the script printed.
+        """
+        with tempfile.TemporaryDirectory() as tmp:
+            bin_dir = Path(tmp)
+            for name, body in (("msmtp", MSMTP_STUB), ("sleep", SLEEP_STUB)):
+                stub = bin_dir / name
+                stub.write_text(body)
+                stub.chmod(0o755)
+            env = dict(os.environ)
+            env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
+            env["MAIL_RECIPIENT"] = "administrator@example.test"
+            env["MAIL_TIMEOUT"] = "30"
+            env["STUB_DIR"] = tmp
+            env["STUB_FAILURES"] = str(failures)
+            env["STUB_RC"] = str(rc)
+            env["STUB_MESSAGE"] = message
+            done = subprocess.run(
+                ["bash", str(SCRIPT)],
+                env=env,
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+            retries = [
+                line
+                for line in done.stderr.splitlines()
+                if "msmtp transient failure" in line
+            ]
+            return done.returncode, retries
+
+    def test_a_dropped_connection_is_retried_until_the_mail_is_accepted(self):
+        code, retries = self.run_script(failures=1, rc=76, message=EMPTY_REPLY)
+        self.assertEqual(code, 0)
+        self.assertEqual(len(retries), 1)
+        self.assertIn("(rc=76), attempt 1/10", retries[0])
+
+    def test_another_protocol_error_ends_the_check_at_once(self):
+        code, retries = self.run_script(
+            failures=99,
+            rc=76,
+            message="msmtp: cannot get initial OK message from server",
+        )
+        self.assertEqual(code, 76)
+        self.assertEqual(retries, [])
+
+    def test_a_server_that_keeps_dropping_fails_after_the_last_attempt(self):
+        code, retries = self.run_script(failures=99, rc=76, message=EMPTY_REPLY)
+        self.assertEqual(code, 76)
+        self.assertEqual(len(retries), 9)
+
+
+if __name__ == "__main__":
+    unittest.main()