diff --git a/roles/sys-ctl-alm-email/templates/script.sh.j2 b/roles/sys-ctl-alm-email/templates/script.sh.j2 index c9ec0bdf61..447bbaf467 100644 --- a/roles/sys-ctl-alm-email/templates/script.sh.j2 +++ b/roles/sys-ctl-alm-email/templates/script.sh.j2 @@ -39,8 +39,38 @@ $STATUS_OUT ERRMAIL -timeout "{{ email.timeout }}s" "$sendmail_bin" -t < "$mail_tmp" -rc=$? +attempts=3 +delay=10 + +for attempt in $(seq 1 "${attempts}"); do + send_err="$(timeout "{{ email.timeout }}s" "$sendmail_bin" -t < "$mail_tmp" 2>&1 1>/dev/null)" + rc=$? + [ "$rc" -eq 0 ] && exit 0 + + case "$rc" in + 69 | 75 | 124 | 137) ;; + 76) + case "$send_err" in + *"the server sent an empty reply"*) ;; + *) break ;; + esac + ;; + 77) + case "$send_err" in + *"server message: 4"* | *"Temporary authentication failure"*) ;; + *) break ;; + esac + ;; + *) break ;; + esac + + if [ "$attempt" -lt "$attempts" ]; then + echo "sendmail transient failure (rc=${rc}), attempt ${attempt}/${attempts}; retrying in ${delay}s" >&2 + sleep "$delay" + fi +done + +[ -n "$send_err" ] && echo "$send_err" >&2 if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then echo "ERROR: sendmail timed out after {{ email.timeout }}s" >&2 diff --git a/roles/sys-ctl-hlth-msmtp/files/shell/script.sh b/roles/sys-ctl-hlth-msmtp/files/shell/script.sh index 372dd280cb..01a0b22c00 100644 --- a/roles/sys-ctl-hlth-msmtp/files/shell/script.sh +++ b/roles/sys-ctl-hlth-msmtp/files/shell/script.sh @@ -21,6 +21,12 @@ for attempt in $(seq 1 "${attempts}"); do case "${rc}" in 69 | 75 | 124) ;; + 76) + case "${msmtp_err}" in + *"the server sent an empty reply"*) ;; + *) exit "${rc}" ;; + esac + ;; 77) case "${msmtp_err}" in *"server message: 4"* | *"Temporary authentication failure"*) ;; diff --git a/tests/unit/python/roles/sys-ctl-alm-email/__init__.py b/tests/unit/python/roles/sys-ctl-alm-email/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/roles/sys-ctl-alm-email/templates/__init__.py b/tests/unit/python/roles/sys-ctl-alm-email/templates/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/roles/sys-ctl-alm-email/templates/test_script.py b/tests/unit/python/roles/sys-ctl-alm-email/templates/test_script.py new file mode 100644 index 0000000000..c53236ab27 --- /dev/null +++ b/tests/unit/python/roles/sys-ctl-alm-email/templates/test_script.py @@ -0,0 +1,123 @@ +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader, StrictUndefined + +from utils import PROJECT_ROOT + +TEMPLATE_DIR = PROJECT_ROOT / "roles/sys-ctl-alm-email/templates" + +EMPTY_REPLY = "sendmail: the server sent an empty reply" + +SENDMAIL_STUB = """#!/usr/bin/env bash +cat >/dev/null +calls=$(( $(cat "${STUB_DIR}/calls" 2>/dev/null || echo 0) + 1 )) +echo "${calls}" >"${STUB_DIR}/calls" +if [ "${calls}" -gt "${STUB_FAILURES}" ]; then +\texit 0 +fi +echo "${STUB_MESSAGE}" >&2 +exit "${STUB_RC}" +""" + +QUIET_STUB = """#!/usr/bin/env bash +exit 0 +""" + +ECHO_STUB = """#!/usr/bin/env bash +printf '%s\\n' "${@: -1}" +""" + + +def lookup(name, *_args): + """Return what the template's two lookups resolve to. + + Args: + name: lookup plugin the template calls. + """ + return { + "email": {"from": "no-reply@example.test", "timeout": "30"}, + "users": {"email": "administrator@example.test"}, + }[name] + + +def rendered_script() -> str: + env = Environment( + loader=FileSystemLoader(str(TEMPLATE_DIR)), + undefined=StrictUndefined, + autoescape=False, # noqa: S701 - shell script, not markup; Ansible renders it the same way + ) + return env.get_template("script.sh.j2").render(lookup=lookup, HOST_CS="utf-8") + + +class TestAlmEmailScript(unittest.TestCase): + def run_script(self, *, failures, rc, message): + """Run the rendered alarm script against a stubbed sendmail. + + Args: + failures: how many sendmail calls fail before one succeeds. + rc: exit code of a failing call. + message: what a failing call prints to stderr. + + Returns: + Tuple of the exit code, the retry lines and the whole stderr. + """ + with tempfile.TemporaryDirectory() as tmp: + bin_dir = Path(tmp) + stubs = ( + ("sendmail", SENDMAIL_STUB), + ("sleep", QUIET_STUB), + ("systemctl", QUIET_STUB), + ("systemd-escape", ECHO_STUB), + ) + for name, body in stubs: + stub = bin_dir / name + stub.write_text(body) + stub.chmod(0o755) + script = bin_dir / "script.sh" + script.write_text(rendered_script()) + env = dict(os.environ) + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + env["STUB_DIR"] = tmp + env["STUB_FAILURES"] = str(failures) + env["STUB_RC"] = str(rc) + env["STUB_MESSAGE"] = message + done = subprocess.run( + ["bash", str(script), "example.service"], + env=env, + capture_output=True, + text=True, + check=False, + ) + retries = [ + line + for line in done.stderr.splitlines() + if "sendmail transient failure" in line + ] + return done.returncode, retries, done.stderr + + def test_a_dropped_connection_is_retried_and_the_alarm_is_sent(self): + code, retries, _ = self.run_script(failures=1, rc=76, message=EMPTY_REPLY) + self.assertEqual(code, 0) + self.assertEqual(len(retries), 1) + self.assertIn("(rc=76), attempt 1/3", retries[0]) + + def test_a_permanent_rejection_is_reported_without_a_retry(self): + rejection = "sendmail: recipient address rejected" + code, retries, stderr = self.run_script(failures=99, rc=65, message=rejection) + self.assertEqual(code, 65) + self.assertEqual(retries, []) + self.assertIn(rejection, stderr) + + def test_the_third_failure_ends_the_alarm_with_the_last_exit_code(self): + code, retries, stderr = self.run_script(failures=99, rc=76, message=EMPTY_REPLY) + self.assertEqual(code, 76) + self.assertEqual(len(retries), 2) + self.assertIn(EMPTY_REPLY, stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/roles/sys-ctl-hlth-msmtp/files/__init__.py b/tests/unit/python/roles/sys-ctl-hlth-msmtp/files/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/roles/sys-ctl-hlth-msmtp/files/test_script.py b/tests/unit/python/roles/sys-ctl-hlth-msmtp/files/test_script.py new file mode 100644 index 0000000000..7803a3d637 --- /dev/null +++ b/tests/unit/python/roles/sys-ctl-hlth-msmtp/files/test_script.py @@ -0,0 +1,91 @@ +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils import PROJECT_ROOT + +SCRIPT = PROJECT_ROOT / "roles/sys-ctl-hlth-msmtp/files/shell/script.sh" + +EMPTY_REPLY = "msmtp: the server sent an empty reply" + +MSMTP_STUB = """#!/usr/bin/env bash +cat >/dev/null +calls=$(( $(cat "${STUB_DIR}/calls" 2>/dev/null || echo 0) + 1 )) +echo "${calls}" >"${STUB_DIR}/calls" +if [ "${calls}" -gt "${STUB_FAILURES}" ]; then +\texit 0 +fi +echo "${STUB_MESSAGE}" >&2 +exit "${STUB_RC}" +""" + +SLEEP_STUB = """#!/usr/bin/env bash +exit 0 +""" + + +class TestHlthMsmtpScript(unittest.TestCase): + def run_script(self, *, failures, rc, message): + """Run the health script against a stubbed msmtp. + + Args: + failures: how many msmtp calls fail before one succeeds. + rc: exit code of a failing call. + message: what a failing call prints to stderr. + + Returns: + Tuple of the exit code and the retry lines the script printed. + """ + with tempfile.TemporaryDirectory() as tmp: + bin_dir = Path(tmp) + for name, body in (("msmtp", MSMTP_STUB), ("sleep", SLEEP_STUB)): + stub = bin_dir / name + stub.write_text(body) + stub.chmod(0o755) + env = dict(os.environ) + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + env["MAIL_RECIPIENT"] = "administrator@example.test" + env["MAIL_TIMEOUT"] = "30" + env["STUB_DIR"] = tmp + env["STUB_FAILURES"] = str(failures) + env["STUB_RC"] = str(rc) + env["STUB_MESSAGE"] = message + done = subprocess.run( + ["bash", str(SCRIPT)], + env=env, + capture_output=True, + text=True, + check=False, + ) + retries = [ + line + for line in done.stderr.splitlines() + if "msmtp transient failure" in line + ] + return done.returncode, retries + + def test_a_dropped_connection_is_retried_until_the_mail_is_accepted(self): + code, retries = self.run_script(failures=1, rc=76, message=EMPTY_REPLY) + self.assertEqual(code, 0) + self.assertEqual(len(retries), 1) + self.assertIn("(rc=76), attempt 1/10", retries[0]) + + def test_another_protocol_error_ends_the_check_at_once(self): + code, retries = self.run_script( + failures=99, + rc=76, + message="msmtp: cannot get initial OK message from server", + ) + self.assertEqual(code, 76) + self.assertEqual(retries, []) + + def test_a_server_that_keeps_dropping_fails_after_the_last_attempt(self): + code, retries = self.run_script(failures=99, rc=76, message=EMPTY_REPLY) + self.assertEqual(code, 76) + self.assertEqual(len(retries), 9) + + +if __name__ == "__main__": + unittest.main()