fix(swarm): read one converge budget and give up on a terminal task

Two callers polled the same stack_ready.sh with different budgets: the
handler read swarm_stack_converge_retries while post_deploy hardcoded
150. Eleven roles had set the override and were unprotected at the
second caller anyway. Both now read one SPOT in group_vars.

The gate also had no way to give up. A crash-looping espocrm burned 600
polls (20 min) before failing, while flowise was killed 45 s short of a
409 s image pull. has_fatal_task now exits 2 - and both callers stop at
once via 'until: rc in [0, 2]' - when a slot's newest attempt is
Rejected or Pending, or when a slot burned three attempts after the
service churned past swarm_stack_fatal_grace. Patience only grows where
nothing is broken.

report_tasks loses its 'awk !/error=$/' filter, which printed only rows
that HAVE an error and so stayed silent in exactly the no-error case
that occurred.

Measured on a live single-node swarm: a service failing 3x and
converging on the 4th returned rc 2 at t+15s before this change and rc 0
at t+32s after; a 3-replica stop-first rolling update gave 68/68 samples
rc 0; a permanent crash loop still reaches rc 2. The deploy epoch comes
from 'docker service inspect .UpdatedAt.Unix' rather than a /tmp marker,
which was reproduced contaminating a later deploy.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

group_vars/all/18_swarm.yml

@@ -13,6 +13,9 @@ IS_BACKUP_HOST: >-
 STACK_HOST: >-
   {{ (groups['svc-swarm-manager'] | default([]))[0] | default(inventory_hostname) }}
 
+swarm_stack_converge_retries: 600
+swarm_stack_fatal_grace: 180
+
 swarm:
   manager:
     advertise_addr: ""

roles/sys-svc-compose/files/swarm/stack_ready.sh

@@ -2,6 +2,7 @@
 set -euo pipefail
 
 : "${STACK:?STACK env var is required}"
+: "${FATAL_GRACE:?FATAL_GRACE env var is required}"
 
 is_completed_oneshot() {
 	local ps
@@ -25,11 +26,51 @@ report_tasks() {
 		printf '  %s: docker service ps failed: %s\n' "$1" "$ps" >&2
 		return 0
 	fi
-	rows="$(printf '%s\n' "$ps" | awk '!/error=$/' | head -10)"
+	rows="$(printf '%s\n' "$ps" | head -10)"
 	[ -n "${rows}" ] || return 0
 	printf '%s\n' "${rows}" | sed 's/^/  /' >&2
 }
 
+# Exception: a slot that burned three attempts and converges on the fourth is
+# byte-identical to one that never will, so strikes may only count once the
+# service has churned past any dependency wait it could be racing. `.UpdatedAt`
+# is the epoch because every deploy path bumps it and task churn does not.
+# An unreadable or non-numeric read must mean "keep waiting" -- the gate may
+# lose an abort, it may never invent one.
+churned_past_grace() {
+	local now updated
+	now=$(date +%s)
+	updated=$(timeout 15 docker service inspect \
+		--format '{{.UpdatedAt.Unix}}' "$1" 2>/dev/null) || return 1
+	case "$updated" in
+	'' | *[!0-9]*) return 1 ;;
+	esac
+	[ "$((now - updated))" -ge "$FATAL_GRACE" ]
+}
+
+# Exception: keys on current-state, never desired-state. A rejected or
+# unschedulable task never ran, so docker has already judged the spec unrunnable
+# and no clock is needed. Filtering on desired=Running instead hides that whole
+# group -- a service dead in three seconds has no task desiring Running, the
+# table comes back empty, and the caller waits out its entire budget.
+has_fatal_task() {
+	local ps grace=0
+	churned_past_grace "$1" && grace=1
+	ps=$(timeout 15 docker service ps --no-trunc \
+		--format '{{.Name}}|{{.CurrentState}}|{{.Error}}' "$1" 2>/dev/null) || return 1
+	[ -n "$ps" ] || return 1
+	awk -F'|' -v grace="$grace" '
+		{
+			if (ended[$1]) next
+			newest = !seen[$1]++
+			if ($3 == "") { if (!newest) ended[$1] = 1; next }
+			if (newest && $2 ~ /^(Rejected|Pending)/) fatal = 1
+			if (++errs[$1] >= 3 && grace) fatal = 1
+		}
+		END { exit fatal ? 0 : 1 }
+	' <<<"$ps"
+}
+
 if ! services=$(timeout 15 docker stack services --format '{{.Name}} {{.Replicas}}' "$STACK"); then
 	echo "not converged: docker stack services failed or timed out for ${STACK}" >&2
 	exit 1
@@ -47,9 +88,16 @@ done <<<"$services"
 
 if [ -n "$not_running" ]; then
 	echo "not converged:$not_running" >&2
-	printf '%s\n' "$not_running" | tr ' ' '\n' | while read -r svc; do
-		[ -n "$svc" ] || continue
+	fatal=0
+	for svc in $not_running; do
 		report_tasks "$svc"
+		if has_fatal_task "$svc"; then
+			echo "stuck: $svc" >&2
+			fatal=1
+		fi
 	done
+	if [ "$fatal" -ne 0 ]; then
+		exit 2
+	fi
 	exit 1
 fi

roles/sys-svc-compose/handlers/swarm.yml

@@ -167,11 +167,12 @@
   ansible.builtin.script: "{{ lookup('path_absolute', 'roles/sys-svc-compose/files/swarm/stack_ready.sh') }}"
   environment:
     STACK: "{{ application_id | get_entity_name }}"
+    FATAL_GRACE: "{{ swarm_stack_fatal_grace }}"
   register: _swarm_stack_ready_handler
-  retries: "{{ swarm_stack_converge_retries | default(150) }}"
+  retries: "{{ swarm_stack_converge_retries }}"
   delay: 2
   changed_when: false
-  until: _swarm_stack_ready_handler is succeeded
+  until: _swarm_stack_ready_handler.rc in [0, 2]
   listen:
     - compose up
     - swarm deploy

roles/sys-svc-compose/tasks/utils/post_deploy.yml

@@ -17,9 +17,10 @@
   ansible.builtin.script: "{{ lookup('path_absolute', 'roles/sys-svc-compose/files/swarm/stack_ready.sh') }}"
   environment:
     STACK: "{{ application_id | get_entity_name }}"
+    FATAL_GRACE: "{{ swarm_stack_fatal_grace }}"
   register: _swarm_tasks_ready
-  retries: 150
+  retries: "{{ swarm_stack_converge_retries }}"
   delay: 2
   changed_when: false
-  until: _swarm_tasks_ready is succeeded
+  until: _swarm_tasks_ready.rc in [0, 2]
   when: (compose_mode_force | default('') or DEPLOYMENT_MODE) == 'swarm'

tests/lint/repository/test_swarm_converge_budget.py

@@ -0,0 +1,92 @@
+"""Contract of the swarm convergence gate's callers.
+
+Both callers poll the same ``stack_ready.sh`` for the same stack, so a budget
+that fits one fits the other. They drifted once - the handler was overridable
+while post_deploy kept a hardcoded 150 - and the shorter one then killed a
+healthy-but-slow stack. Pinned here: both read the single declared budget, both
+give up at once on the gate's terminal rc 2 instead of waiting it out, the
+worst-case wall clock of one gate stays inside the sweep budget, and no role
+re-declares the budget at a value the declaration no longer carries.
+"""
+
+from __future__ import annotations
+
+import unittest
+
+from utils.cache.yaml import load_yaml_any
+from utils.env.parser import parse_static_env
+from utils.roles.mapping import ROLE_FILE_VARS_MAIN
+
+from . import PROJECT_ROOT
+
+COMPOSE = PROJECT_ROOT / "roles" / "sys-svc-compose"
+CALLERS = (
+    COMPOSE / "handlers" / "swarm.yml",
+    COMPOSE / "tasks" / "utils" / "post_deploy.yml",
+)
+BUDGET = PROJECT_ROOT / "group_vars" / "all" / "18_swarm.yml"
+
+
+def _gate_tasks(path):
+    return [
+        task
+        for task in (load_yaml_any(str(path), default_if_missing=[]) or [])
+        if "stack_ready.sh" in str(task.get("ansible.builtin.script", ""))
+    ]
+
+
+class TestSwarmConvergeBudget(unittest.TestCase):
+    def test_every_caller_reads_the_declared_budget(self) -> None:
+        for path in CALLERS:
+            tasks = _gate_tasks(path)
+            self.assertEqual(len(tasks), 1, f"{path} does not call the gate once")
+            self.assertEqual(
+                tasks[0]["retries"], "{{ swarm_stack_converge_retries }}", str(path)
+            )
+
+    def test_every_caller_gives_up_on_a_terminal_gate_verdict(self) -> None:
+        for path in CALLERS:
+            self.assertIn("rc in [0, 2]", _gate_tasks(path)[0]["until"], str(path))
+
+    def test_every_caller_passes_the_declared_grace(self) -> None:
+        for path in CALLERS:
+            self.assertEqual(
+                _gate_tasks(path)[0]["environment"]["FATAL_GRACE"],
+                "{{ swarm_stack_fatal_grace }}",
+                str(path),
+            )
+
+    def test_the_budget_is_declared_once_and_fits_the_sweep(self) -> None:
+        retries = load_yaml_any(str(BUDGET), default_if_missing={})[
+            "swarm_stack_converge_retries"
+        ]
+        delay = _gate_tasks(CALLERS[0])[0]["delay"]
+        sweep = int(
+            parse_static_env(PROJECT_ROOT / "default.env")[
+                "INFINITO_CI_DISTRO_BUDGET_SECONDS"
+            ]
+        )
+        self.assertLess(retries * delay, sweep)
+
+    def test_the_grace_fits_inside_the_budget(self) -> None:
+        declared = load_yaml_any(str(BUDGET), default_if_missing={})
+        delay = _gate_tasks(CALLERS[0])[0]["delay"]
+        self.assertLess(
+            declared["swarm_stack_fatal_grace"],
+            declared["swarm_stack_converge_retries"] * delay,
+        )
+
+    def test_no_role_shadows_the_budget_with_a_different_value(self) -> None:
+        declared = load_yaml_any(str(BUDGET), default_if_missing={})[
+            "swarm_stack_converge_retries"
+        ]
+        for path in sorted((PROJECT_ROOT / "roles").glob(f"*/{ROLE_FILE_VARS_MAIN}")):
+            shadow = (load_yaml_any(str(path), default_if_missing={}) or {}).get(
+                "swarm_stack_converge_retries"
+            )
+            if shadow is not None:
+                self.assertEqual(shadow, declared, str(path))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/roles/sys-svc-compose/files/test_stack_ready.py

@@ -1,11 +1,21 @@
 """Behaviour of the swarm convergence gate.
 
 ``stack_ready.sh`` decides whether a swarm deploy is done. It is driven by an
-Ansible ``until`` loop of up to 150 polls, so both its verdict and how much it
-prints on a failing poll matter. Exercised here against a stub docker: a fully
-replicated stack passes, a short one fails, a completed one-shot counts as done,
-and a non-converged service reports the task rows that carry an error while
-staying quiet about the ones that do not.
+Ansible ``until`` loop that keeps polling on rc 1 and gives up at once on rc 2,
+so both its verdict and how much it prints on a failing poll matter. Exercised
+here against a stub docker: a fully replicated stack passes, a short one keeps
+the caller waiting, a completed one-shot counts as done, and every task row
+reaches the caller.
+
+The two terminal verdicts are split by the task's current state, because that is
+what says whether docker ever ran the container. A task rejected or left
+unschedulable never started, so docker has already judged the spec unrunnable
+and the verdict needs no clock. A task that failed did run, and a crash loop
+waiting out a dependency is indistinguishable from one that never converges, so
+only the grace-gated strike counter may end it - never on the newest row alone,
+and never on errors left below a finished attempt. That counter is armed off the
+service's own ``UpdatedAt``, so an unreadable epoch has to keep the caller
+waiting rather than arm it.
 """
 
 from __future__ import annotations
@@ -13,6 +23,7 @@ from __future__ import annotations
 import os
 import subprocess
 import tempfile
+import time
 import unittest
 from pathlib import Path
 
@@ -22,14 +33,38 @@ GATE = PROJECT_ROOT / "roles" / "sys-svc-compose" / "files" / "swarm" / "stack_r
 STUB = """#!/usr/bin/env bash
 case "$1 $2" in
 "stack services") printf '%s' "${STACK_SERVICES}" ;;
-"service ps") printf '%s' "${SERVICE_PS}" ;;
+"service inspect") printf '%s' "${SERVICE_UPDATED_AT}" ;;
+"service ps")
+	case "$5" in
+	*'{{.Name}}|{{.CurrentState}}|{{.Error}}'*) printf '%s' "${SERVICE_ERRORS}" ;;
+	*) printf '%s' "${SERVICE_PS}" ;;
+	esac
+	;;
 esac
 exit 0
 """
 
 
 class TestStackReady(unittest.TestCase):
-    def _run(self, services: str, service_ps: str = "") -> subprocess.CompletedProcess:
+    def _run(
+        self,
+        services: str,
+        service_ps: str = "",
+        service_errors: str = "",
+        churning: int = 0,
+        updated_at: str | None = None,
+    ) -> subprocess.CompletedProcess:
+        """Poll the gate once, ``churning`` seconds into a non-converged deploy.
+
+        Args:
+            services: ``docker stack services`` table the stub returns.
+            service_ps: verbose ``docker service ps`` table for the report.
+            service_errors: ``name|current state|error`` rows, newest first.
+            churning: seconds since the deploy bumped the service ``UpdatedAt``.
+            updated_at: raw ``UpdatedAt`` epoch, overriding ``churning``.
+        """
+        if updated_at is None:
+            updated_at = str(int(time.time()) - churning)
         with tempfile.TemporaryDirectory() as tmp:
             stub_bin = Path(tmp) / "bin"
             stub_bin.mkdir()
@@ -43,8 +78,11 @@ class TestStackReady(unittest.TestCase):
             env = dict(os.environ)
             env.update(
                 STACK="demo",
+                FATAL_GRACE="180",
                 STACK_SERVICES=services,
                 SERVICE_PS=service_ps,
+                SERVICE_ERRORS=service_errors,
+                SERVICE_UPDATED_AT=updated_at,
                 PATH=f"{stub_bin}:{env['PATH']}",
                 BASH_ENV="",
             )
@@ -73,21 +111,104 @@ class TestStackReady(unittest.TestCase):
         )
         self.assertEqual(proc.returncode, 0)
 
-    def test_a_task_error_reaches_the_caller(self) -> None:
+    def test_a_rejected_task_is_terminal_without_any_grace(self) -> None:
         proc = self._run(
             "demo_reg 0/1\n",
-            "demo_reg.1 desired=Running current=Preparing error=manifest pull timed out\n",
+            "demo_reg.1 desired=Ready current=Rejected error=No such image: reg/nope\n",
+            "demo_reg.1|Rejected 3 seconds ago|No such image: reg/nope\n",
+        )
+        self.assertEqual(proc.returncode, 2)
+        self.assertIn("No such image", proc.stderr)
+        self.assertIn("stuck: demo_reg", proc.stderr)
+
+    def test_an_unschedulable_task_is_terminal_without_any_grace(self) -> None:
+        proc = self._run(
+            "demo_place 0/1\n",
+            "demo_place.1 desired=Running current=Pending error=no suitable node\n",
+            "demo_place.1|Pending 28 seconds ago|no suitable node\n",
+        )
+        self.assertEqual(proc.returncode, 2)
+        self.assertIn("stuck: demo_place", proc.stderr)
+
+    def test_a_newest_failed_attempt_alone_is_never_terminal(self) -> None:
+        """A container that ran and died may still be waiting out a dependency."""
+        proc = self._run(
+            "demo_boot 0/1\n",
+            "demo_boot.1 desired=Shutdown current=Failed error=task: non-zero exit (1)\n",
+            "demo_boot.1|Failed 1 second ago|task: non-zero exit (1)\n",
         )
         self.assertEqual(proc.returncode, 1)
-        self.assertIn("manifest pull timed out", proc.stderr)
+        self.assertNotIn("stuck:", proc.stderr)
 
-    def test_error_free_task_rows_are_not_echoed(self) -> None:
+    def test_error_free_task_rows_are_echoed_and_keep_the_caller_waiting(self) -> None:
         proc = self._run(
             "demo_slow 0/1\n",
             "demo_slow.1 desired=Running current=Preparing error=\n",
+            "demo_slow.1|Preparing 2 seconds ago|\n",
+        )
+        self.assertEqual(proc.returncode, 1)
+        self.assertIn("demo_slow.1", proc.stderr)
+        self.assertIn("current=Preparing", proc.stderr)
+
+    def test_a_fresh_attempt_after_two_failures_still_waits(self) -> None:
+        proc = self._run(
+            "demo_boot 0/1\n",
+            "demo_boot.1 desired=Running current=Starting error=\n",
+            "demo_boot.1|Ready 1 second ago|\n"
+            "demo_boot.1|Failed 2 seconds ago|task: non-zero exit (1)\n"
+            "demo_boot.1|Failed 8 seconds ago|task: non-zero exit (1)\n",
+            churning=600,
+        )
+        self.assertEqual(proc.returncode, 1)
+
+    def test_a_slot_that_burned_three_attempts_is_terminal(self) -> None:
+        proc = self._run(
+            "demo_flap 0/1\n",
+            "demo_flap.1 desired=Running current=Starting error=\n",
+            "demo_flap.1|Ready 1 second ago|\n"
+            "demo_flap.1|Failed 2 seconds ago|task: non-zero exit (1)\n"
+            "demo_flap.1|Failed 8 seconds ago|task: non-zero exit (1)\n"
+            "demo_flap.1|Failed 14 seconds ago|task: non-zero exit (1)\n",
+            churning=600,
+        )
+        self.assertEqual(proc.returncode, 2)
+        self.assertIn("stuck: demo_flap", proc.stderr)
+
+    def test_three_attempts_inside_the_grace_window_still_wait(self) -> None:
+        proc = self._run(
+            "demo_flap 0/1\n",
+            "demo_flap.1 desired=Running current=Starting error=\n",
+            "demo_flap.1|Ready 1 second ago|\n"
+            "demo_flap.1|Failed 2 seconds ago|task: non-zero exit (1)\n"
+            "demo_flap.1|Failed 8 seconds ago|task: non-zero exit (1)\n"
+            "demo_flap.1|Failed 14 seconds ago|task: non-zero exit (1)\n",
+        )
+        self.assertEqual(proc.returncode, 1)
+
+    def test_errors_below_a_finished_attempt_do_not_carry_over(self) -> None:
+        proc = self._run(
+            "demo_again 0/1\n",
+            "demo_again.1 desired=Running current=Pending error=\n",
+            "demo_again.1|Pending 1 second ago|\n"
+            "demo_again.1|Shutdown 4 seconds ago|\n"
+            "demo_again.1|Failed 9 seconds ago|task: non-zero exit (1)\n"
+            "demo_again.1|Failed 15 seconds ago|task: non-zero exit (1)\n"
+            "demo_again.1|Failed 21 seconds ago|task: non-zero exit (1)\n",
+            churning=600,
+        )
+        self.assertEqual(proc.returncode, 1)
+
+    def test_an_unreadable_deploy_epoch_keeps_the_caller_waiting(self) -> None:
+        proc = self._run(
+            "demo_flap 0/1\n",
+            "demo_flap.1 desired=Running current=Starting error=\n",
+            "demo_flap.1|Ready 1 second ago|\n"
+            "demo_flap.1|Failed 2 seconds ago|task: non-zero exit (1)\n"
+            "demo_flap.1|Failed 8 seconds ago|task: non-zero exit (1)\n"
+            "demo_flap.1|Failed 14 seconds ago|task: non-zero exit (1)\n",
+            updated_at="<no value>",
         )
         self.assertEqual(proc.returncode, 1)
-        self.assertNotIn("demo_slow.1", proc.stderr)
 
 
 if __name__ == "__main__":