mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 02:54:26 +00:00
fix(swarm-test): label a worker kata-capable only when the app is sandboxed
Two Moodle#1 jobs of run 33445831564 -- tor/arch and clearnet/debian -- end with the
same line, 65 minutes apart:
FAILURE: no running moodle task on a worker node
The deploy is clean; PLAY RECAP reports failed=0 on all four hosts. Only the
post-deploy drain drill fails, and it fails because nothing is left to drain.
The chain, each link measured rather than assumed:
meta/variants.yml variant 1 turns openwebui/hermes/openclaw/flowise on;
variant 0 turns all four off
extend_inventory.py:88-89 svc-virt-kata in the closure put BOTH workers in the group
svc-virt-kata/00_core.yml:61 labels every group member kata-capable=true
(2 label tasks in each moodle#1 log, 0 in moodle#0)
deploy.yml.j2:18-23 non-kata services get node.labels.kata-capable != true
05_seed_content.sh:12-22 filters running tasks with grep -vx "${MGR}"
With both workers labelled, moodle and nginx have no eligible node but the stack
host. `docker service ps` at failure shows all three moodle_nginx replicas Running
on mgr-01. The drill then looks for a worker task and finds none.
Controls in the same run: moodle#0 carries no kata group and 0 label tasks, and its
drill was reachable; Nextcloud#10 (green, no kata) logs
"initially scheduled on: ...wrk-01" -> "Rescheduled ... to ...mgr-01" -> drill complete.
Exactly three of the run's 18 swarm inventories carry the kata group, and all three
are red.
b20a70646f widened _WORKERS[-1] to *_WORKERS and said so in its own message: "whether
the stack host holds up under it is what the next swarm run measures". The stack host
held up. What broke is the drill's precondition.
The discriminator is the app's own declaration, and it separates cleanly -- executed,
not reasoned:
web-app-moodle kata.enabled=None -> ('swarm-wrk-02',)
web-app-nextcloud kata.enabled=None -> ('swarm-wrk-02',)
web-app-openclaw kata.enabled=True -> ('swarm-wrk-01', 'swarm-wrk-02')
web-app-hermes kata.enabled=True -> ('swarm-wrk-01', 'swarm-wrk-02')
_WORKERS[-1:] restores exactly the pre-b20a70646f tuple for the non-sandboxed case, so
that commit's drain fix stays intact for the rounds it was written for.
svc-virt-kata's own assert (00_core.yml:80-88, labelled_nodes | length > 0) holds with
one labelled worker.
applications_for_round() is a thin public wrapper over the existing
_applications_for_active_variants(get_merged_applications(...)); extend_inventory would
otherwise have to import a neighbour module's private helper.
Three tests cover the branch, which had none: grep -c "kata\|_WORKERS" on
test_extend_inventory.py returned 0 before this.
Not covered: the full chain -- label, placement, drill -- has not run. One
`make act-swarm apps=web-app-moodle variant=1` closes that.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f07b01025d
commit
8ce66075b5
3 files changed
+52
-2
No files matched your search
@@ -57,5 +57,23 @@ class TestBackupLegGating(unittest.TestCase):
|
||||
self.assertNotIn("svc-bkp-volume-2-local", children)
|
||||
|
||||
|
||||
class TestSandboxWorkerLabelling(unittest.TestCase):
|
||||
def _workers(self, kata_enabled: object) -> tuple[str, ...]:
|
||||
apps = {"an-app": {"services": {"kata": {"enabled": kata_enabled}}}}
|
||||
with mock.patch.object(ei, "applications_for_round", return_value=apps):
|
||||
return ei._sandbox_workers("an-app")
|
||||
|
||||
def test_a_sandboxed_app_labels_every_worker(self):
|
||||
self.assertEqual(self._workers(True), ei._WORKERS)
|
||||
|
||||
def test_a_plain_app_leaves_one_worker_unlabelled(self):
|
||||
self.assertEqual(self._workers(None), ei._WORKERS[-1:])
|
||||
self.assertEqual(self._workers(False), ei._WORKERS[-1:])
|
||||
|
||||
def test_an_app_without_a_kata_service_leaves_one_worker(self):
|
||||
with mock.patch.object(ei, "applications_for_round", return_value={}):
|
||||
self.assertEqual(ei._sandbox_workers("absent"), ei._WORKERS[-1:])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -141,6 +141,23 @@ def _force_shared_db_view(applications: dict[str, Any]) -> dict[str, Any]:
|
||||
return result
|
||||
|
||||
|
||||
def applications_for_round(
|
||||
variants: dict[str, int] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return the round's application configs with its variants applied.
|
||||
|
||||
Args:
|
||||
variants: active ``{app_id: variant_index}`` map; ``None`` falls back
|
||||
to ``INFINITO_APP_VARIANTS``.
|
||||
|
||||
Returns:
|
||||
Application id to its config for this round.
|
||||
"""
|
||||
return _applications_for_active_variants(
|
||||
get_merged_applications(roles_dir=str(_ROLES_DIR)), variants
|
||||
)
|
||||
|
||||
|
||||
def derive_includes(
|
||||
app_id: str, *, variants: dict[str, int] | None = None
|
||||
) -> list[str]:
|
||||
|
||||
@@ -32,7 +32,7 @@ from utils.cache.yaml import dump_yaml, load_yaml_any
|
||||
from utils.env.parser import parse_static_env
|
||||
from utils.roles.meta_lookup import get_role_placement
|
||||
from utils.tests.swarm.backup_repos import repo_placements
|
||||
from utils.tests.swarm.derive_includes import derive_includes
|
||||
from utils.tests.swarm.derive_includes import applications_for_round, derive_includes
|
||||
|
||||
_DOCKER_VARS: dict[str, str] = {
|
||||
"ansible_connection": "docker",
|
||||
@@ -77,6 +77,19 @@ def _placement_dep_groups(app_id: str) -> list[tuple[str, str]]:
|
||||
]
|
||||
|
||||
|
||||
def _sandbox_workers(app_id: str) -> tuple[str, ...]:
|
||||
"""Return the workers that may carry the kata-capable label.
|
||||
|
||||
Args:
|
||||
app_id: the round's primary application.
|
||||
|
||||
Returns:
|
||||
Every worker when the app itself is sandboxed, otherwise one.
|
||||
"""
|
||||
services = (applications_for_round().get(app_id) or {}).get("services") or {}
|
||||
return _WORKERS if ((services.get("kata") or {}).get("enabled")) is True else _WORKERS[-1:]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not os.environ.get("SWARM_NAME"):
|
||||
raise SystemExit("extend_inventory: SWARM_NAME is required (cluster id)")
|
||||
@@ -86,7 +99,9 @@ def main() -> int:
|
||||
closure = derive_includes(app_id)
|
||||
group_hosts = _host_topology(app_id) + _placement_dep_groups(app_id)
|
||||
if "svc-virt-kata" in closure:
|
||||
group_hosts.extend(("svc-virt-kata", host) for host in (_MANAGER, *_WORKERS))
|
||||
group_hosts.extend(
|
||||
("svc-virt-kata", host) for host in (_MANAGER, *_sandbox_workers(app_id))
|
||||
)
|
||||
group_hosts.extend(
|
||||
repo_placements(
|
||||
app_closure=closure,
|
||||
|
||||
Reference in new issue
Block a user