mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 10:59:59 +00:00
kevinveenbirkenbachandClaude Opus 5.5
b37cf2e9fe
Stalwart closes a connection that exceeds its per-IP connection budget without a greeting. msmtp reports that as "the server sent an empty reply" with exit 76, and the mail health check treated exit 76 as permanent. One such drop, in a second in which Alertmanager flushed its notifications, failed a deploy although the same path accepted a mail one second later. The health check now retries exit 76 when msmtp names the empty reply, within the ten attempts it already gives to exit 69, 75 and 124. Every other protocol error still ends the check at once. The alarm mail was sent once, so the alarm about a failed unit was lost to the same drop. It now makes three attempts ten seconds apart for the transient cases of the health check and still reports the last error. The limit on the server is left alone. Exempting the node's own senders by source address would also exempt Tor clients and, in swarm, every external client. That question is tracked in #793. Closes #790 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
81 lines
2.0 KiB
Django/Jinja
81 lines
2.0 KiB
Django/Jinja
#!/bin/bash
|
|
{# nocheck: shell-template grandfathered, values only: the mail envelope, charset and timeout #}
|
|
set -u
|
|
# Reverse systemd escaping for human-readable mail subject/body
|
|
friendly="$(systemd-escape --unescape "$1")"
|
|
|
|
sendmail_bin="$(command -v sendmail 2>/dev/null || true)"
|
|
if [ -z "$sendmail_bin" ] && [ -x /usr/sbin/sendmail ]; then
|
|
sendmail_bin="/usr/sbin/sendmail"
|
|
fi
|
|
if [ -z "$sendmail_bin" ] && [ -x /usr/bin/sendmail ]; then
|
|
sendmail_bin="/usr/bin/sendmail"
|
|
fi
|
|
if [ -z "$sendmail_bin" ]; then
|
|
echo "ERROR: sendmail binary not found" >&2
|
|
exit 1
|
|
fi
|
|
|
|
STATUS_OUT="$(systemctl status --full "$1" 2>/dev/null | head -n 30)"
|
|
if [ -z "$STATUS_OUT" ]; then
|
|
esc="$(systemd-escape "$1")"
|
|
STATUS_OUT="$(systemctl status --full "$esc" 2>/dev/null | head -n 30)"
|
|
fi
|
|
|
|
mail_tmp="$(mktemp)"
|
|
trap 'rm -f "$mail_tmp"' EXIT
|
|
|
|
{% set email = lookup('email') %}
|
|
cat > "$mail_tmp" <<ERRMAIL
|
|
To: {{ lookup('users', 'administrator').email }}
|
|
From: systemd <{{ email['from'] }}>
|
|
Subject: ${friendly}
|
|
Content-Transfer-Encoding: 8bit
|
|
Content-Type: text/plain; charset={{ HOST_CS }}
|
|
|
|
A problem with the service ${friendly} occurred:
|
|
|
|
$STATUS_OUT
|
|
|
|
ERRMAIL
|
|
|
|
attempts=3
|
|
delay=10
|
|
|
|
for attempt in $(seq 1 "${attempts}"); do
|
|
send_err="$(timeout "{{ email.timeout }}s" "$sendmail_bin" -t < "$mail_tmp" 2>&1 1>/dev/null)"
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] && exit 0
|
|
|
|
case "$rc" in
|
|
69 | 75 | 124 | 137) ;;
|
|
76)
|
|
case "$send_err" in
|
|
*"the server sent an empty reply"*) ;;
|
|
*) break ;;
|
|
esac
|
|
;;
|
|
77)
|
|
case "$send_err" in
|
|
*"server message: 4"* | *"Temporary authentication failure"*) ;;
|
|
*) break ;;
|
|
esac
|
|
;;
|
|
*) break ;;
|
|
esac
|
|
|
|
if [ "$attempt" -lt "$attempts" ]; then
|
|
echo "sendmail transient failure (rc=${rc}), attempt ${attempt}/${attempts}; retrying in ${delay}s" >&2
|
|
sleep "$delay"
|
|
fi
|
|
done
|
|
|
|
[ -n "$send_err" ] && echo "$send_err" >&2
|
|
|
|
if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then
|
|
echo "ERROR: sendmail timed out after {{ email.timeout }}s" >&2
|
|
exit 1
|
|
fi
|
|
|
|
exit "$rc"
|