mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 10:59:59 +00:00
fix(swarm): give both convergence gates one budget that fits a real stack
`files/swarm/stack_ready.sh` has exactly two callers and they disagreed: the
handler defaulted to 150 polls at delay 2, and post_deploy.yml hardcoded the
same 150 with no way to override it. 389 seconds is not enough for a stack that
carries a large image.
Four jobs across two runs died on it, every one starting at "149 retries left":
Nextcloud 2 (run 31340965955) plus Workspace centos, debian and manjaro (run
31311996621), the last three all waiting on svc-db-mariadb, a role that carries
no per-role override. Nextcloud 2 is the measured case — the budget expired at
12:59:54 and the task reached Running at ~13:00:32, 38 seconds later. Nothing
was broken; the deadline was.
600 is not a new number: tasks/utils/swarm/wait_converged.yml already waits that
long at the identical delay, and eleven roles had each hand-raised the handler to
exactly 600. 150 was the outlier.
The declaration goes in group_vars/all, not in the role defaults. A role default
is invisible to the by-path include in roles/svc-db-openldap/tasks/01_docker.yml,
which pulls post_deploy.yml in after include_role has finished — verified with a
scratch play, where the same variable resolves inside the role and raises
"is undefined" one task later. post_deploy.yml already encodes that distinction:
it reads DEPLOYMENT_MODE bare and guards compose_mode_force with `| default('')`.
The lint gains a check that the gate's callers never drift apart again. It
discovers them by scanning for the script rather than listing them, so a third
caller is covered, and it asserts it found at least two so an empty scan cannot
pass silently.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1829da072b
commit
5555d95304
4 files changed
+32
-2
No files matched your search
@@ -3,6 +3,11 @@ DEPLOYMENT_MODE: "{{ 'swarm' if (groups['svc-swarm-node'] | default([]) | length
|
||||
|
||||
SWARM_CLUSTER_AVAILABLE: "{{ (groups['svc-swarm-node'] | default([]) | length) > 0 }}"
|
||||
|
||||
# Exception: a role default would be invisible to the by-path include in
|
||||
# roles/svc-db-openldap/tasks/01_docker.yml, which reaches into
|
||||
# roles/sys-svc-compose/tasks/utils/post_deploy.yml after the role has finished.
|
||||
swarm_stack_converge_retries: 600
|
||||
|
||||
IS_STACK_HOST: >-
|
||||
{{ DEPLOYMENT_MODE == 'compose'
|
||||
or inventory_hostname in (groups['svc-swarm-manager'] | default([])) }}
|
||||
|
||||
@@ -168,7 +168,7 @@
|
||||
environment:
|
||||
STACK: "{{ application_id | get_entity_name }}"
|
||||
register: _swarm_stack_ready_handler
|
||||
retries: "{{ swarm_stack_converge_retries | default(150) }}"
|
||||
retries: "{{ swarm_stack_converge_retries }}"
|
||||
delay: 2
|
||||
changed_when: false
|
||||
until: _swarm_stack_ready_handler is succeeded
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
environment:
|
||||
STACK: "{{ application_id | get_entity_name }}"
|
||||
register: _swarm_tasks_ready
|
||||
retries: 150
|
||||
retries: "{{ swarm_stack_converge_retries }}"
|
||||
delay: 2
|
||||
changed_when: false
|
||||
until: _swarm_tasks_ready is succeeded
|
||||
|
||||
@@ -34,6 +34,9 @@ BUILDS = {
|
||||
}
|
||||
TIMEOUT_LOOKUP = re.compile(r"lookup\('timeout',\s*(\d+)\s*\)")
|
||||
|
||||
ROLE = PROJECT_ROOT / "roles" / "sys-svc-compose"
|
||||
GATE = "swarm/stack_ready.sh"
|
||||
|
||||
|
||||
def _tasks(path):
|
||||
return load_yaml_any(str(path), default_if_missing=[]) or []
|
||||
@@ -94,6 +97,28 @@ class TestComposeBuildRetryBudget(unittest.TestCase):
|
||||
self.assertLess(worst, budget, f"{name} outgrows the sweep budget")
|
||||
self.assertLess(worst, cap, f"{name} outgrows the deploy step cap")
|
||||
|
||||
def test_every_convergence_gate_waits_the_same_budget(self) -> None:
|
||||
"""Both callers poll the same script for the same stack, so a budget that
|
||||
fits one fits the other. They drifted once -- the handler was raised to
|
||||
600 while post_deploy kept a hardcoded 150 -- and the shorter one then
|
||||
killed a healthy-but-slow stack on its own."""
|
||||
budgets = {}
|
||||
for path in sorted(ROLE.rglob("*.yml")):
|
||||
for task in _tasks(path):
|
||||
if not isinstance(task, dict):
|
||||
continue
|
||||
if GATE in str(task.get("ansible.builtin.script", "")):
|
||||
budgets[f"{path.relative_to(PROJECT_ROOT)}:{task.get('name')}"] = (
|
||||
str(task.get("retries")),
|
||||
str(task.get("delay")),
|
||||
)
|
||||
self.assertGreaterEqual(len(budgets), 2, f"gate callers not found: {budgets}")
|
||||
self.assertEqual(
|
||||
len(set(budgets.values())),
|
||||
1,
|
||||
f"convergence gate callers disagree on their budget: {budgets}",
|
||||
)
|
||||
|
||||
def test_the_registry_exclusion_has_a_single_source(self) -> None:
|
||||
self.assertEqual(read_text(str(SWARM)).count("svc-registry-docker"), 1)
|
||||
|
||||
|
||||
Reference in new issue
Block a user