Files
computer-playbook/tests/unit/python/roles/sys-svc-container/test_sandbox_placement.py
T
kevinveenbirkenbachandClaude Opus 5 1e88eb9d1d fix(swarm): let a rollout survive one task that failed and came back
update_config never set failure_action, so swarmkit's default applied: pause,
at a max_failure_ratio of zero. One failed task stops the rollout for good, and
nothing in the deploy resumes it.

That contradicts the restart_policy three lines below. The policy replaces the
task and the service ends up healthy, while the update status stays latched and
service_converged.sh reads that status before it looks at any task, so the
deploy reports a failure over a service that is running. The rescue artifact
shows the sequence: the old task shut down, its replacement tripped over a
containerd object runsc could not reap, and the third task was Running while
the job died on "update latched".

Four swarm jobs failed that way, all in the update pass, never in the first.

The convergence check is untouched and stays as strict as it was: a genuinely
broken spec fails every task and still trips "tasks not running yet".

What this does give up: the latch was also the only thing catching a service
that keeps crash-looping. At ratio zero any failure latched, so a task dying
and restarting forever was guaranteed to be reported. Now the rollout
completes, and the check samples task state at one moment - a task on its
fiftieth restart still reads as Running, and wait_converged's `until` needs a
single lucky sample. Restoring that detection needs the check to look at task
churn, which the service ps history already carries and the script prints only
on failure today. Not in this commit.

max_failure_ratio cannot substitute here: at replicas 1, the common case in
this repository, one failure is always the whole service.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-30 12:43:47 +02:00

127 lines
4.8 KiB
Python

import unittest
from pathlib import Path
from jinja2 import Environment, FileSystemLoader, StrictUndefined, select_autoescape
from utils import PROJECT_ROOT
TEMPLATES = Path(PROJECT_ROOT) / "roles" / "sys-svc-container" / "templates"
SANDBOX_LABEL = "kata-capable"
def _ansible_bool(value):
if isinstance(value, bool):
return value
if value is None:
return False
if isinstance(value, str):
return value.strip().lower() in {"y", "yes", "true", "on", "1"}
return bool(value)
BASE = {
"compose_mode": "swarm",
"application_id": "web-app-hermes",
"service_name": "hermes",
"RUNTIME": "local",
"SANDBOX_NODE_LABEL": SANDBOX_LABEL,
"DOCKER_RESTART_POLICY": "unless-stopped",
"RESOURCE_CPUS": "1",
"RESOURCE_MEM_LIMIT": "1g",
"RESOURCE_MEM_RESERVATION": "256m",
"RESOURCE_PIDS_LIMIT": "512",
}
class TestSandboxPlacement(unittest.TestCase):
def _render(
self, kata_enabled, manager_placed=False, sandbox_tier=True, **overrides
):
env = Environment(
loader=FileSystemLoader(str(TEMPLATES)),
trim_blocks=True,
lstrip_blocks=False,
undefined=StrictUndefined,
autoescape=select_autoescape(),
)
env.filters["bool"] = _ansible_bool
env.filters["resource_filter"] = lambda _apps, *_a, **_k: ""
env.filters["swarm_restart_condition"] = lambda value: value
def _lookup(name, *args, **kwargs):
if name == "applications":
return {}
if name == "compose_replicas":
return "replicas: 3"
if name == "roles_with_placement":
return ["web-app-hermes"] if manager_placed else []
if name == "config":
return kata_enabled
raise AssertionError(f"unexpected lookup: {name}")
env.globals["lookup"] = _lookup
groups = {"svc-virt-kata": ["wrk-02"]} if sandbox_tier else {}
return env.get_template("deploy.yml.j2").render(
{**BASE, "groups": groups, **overrides}
)
def _constraints(self, rendered):
lines = [line.strip() for line in rendered.splitlines()]
return [line[2:] for line in lines if line.startswith("- node.")]
def test_sandbox_consumer_is_pinned_to_a_sandbox_node(self):
constraints = self._constraints(self._render(kata_enabled=True))
self.assertIn(f"node.labels.{SANDBOX_LABEL} == true", constraints)
def test_every_other_service_is_kept_off_the_sandbox_nodes(self):
constraints = self._constraints(self._render(kata_enabled=False))
self.assertIn(f"node.labels.{SANDBOX_LABEL} != true", constraints)
def test_the_two_directions_are_mutually_exclusive(self):
for enabled in (True, False):
with self.subTest(kata_enabled=enabled):
constraints = self._constraints(self._render(kata_enabled=enabled))
self.assertEqual(len([c for c in constraints if SANDBOX_LABEL in c]), 1)
def test_a_cluster_without_a_sandbox_tier_gets_no_sandbox_constraint(self):
for enabled in (True, False):
with self.subTest(kata_enabled=enabled):
constraints = self._constraints(
self._render(kata_enabled=enabled, sandbox_tier=False)
)
self.assertEqual([c for c in constraints if SANDBOX_LABEL in c], [])
def test_manager_placement_composes_with_the_sandbox_constraint(self):
constraints = self._constraints(
self._render(kata_enabled=False, manager_placed=True)
)
self.assertIn("node.role == manager", constraints)
self.assertIn(f"node.labels.{SANDBOX_LABEL} != true", constraints)
def test_manager_placement_survives_without_a_sandbox_tier(self):
constraints = self._constraints(
self._render(kata_enabled=False, manager_placed=True, sandbox_tier=False)
)
self.assertEqual(constraints, ["node.role == manager"])
def test_compose_mode_emits_no_deploy_block(self):
rendered = self._render(kata_enabled=True, compose_mode="compose")
self.assertEqual(rendered.strip(), "")
def test_a_failed_task_does_not_latch_the_rollout(self):
"""Swarm's default pauses forever on the first task that fails.
With no failure_action, swarmkit applies pause at a max_failure_ratio
of zero, so one transient failure stops the rollout for good and no
step in the deploy resumes it. The restart policy replaces the task and
the service ends up healthy, while the update status stays latched and
the convergence check reports the deploy as failed.
"""
rendered = self._render(kata_enabled=False)
self.assertIn("failure_action: continue", rendered)
if __name__ == "__main__":
unittest.main()