From 7d9a63c98b65d8bb5efe2d8f264bdb803bd7f295 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 29 Sep 2026 16:46:19 +0200 Subject: [PATCH] fix(svc-ai): create the node-local model directory on every swarm node Both roles mount their model cache as a `type: bind` volume, and swarm rejects a task whose bind source is missing on the node it lands on. Neither declares a `placement`, so the scheduler may put the container on any node, while the directory was created by a single task that reached one host. CI showed the consequence directly: s1_s1.1 desired=Ready current=Rejected error="invalid mount config for type "bind": bind source path does not exist: /var/cache/infinito/core/cache/models/s1" with the creating task logging exactly one result line, for the manager. `nfs_prep.yml` already fans directories out across the cluster but gates on `driver_opts is defined`, which a bind volume never carries, so these two volumes were never covered by it. The host list moves into a `node_hosts` lookup rather than an inline expression, because two roles already spelled the same idea by hand and a third and fourth were about to. It answers one question - the hosts a node-local path must exist on - and keeps the swarm group name in one place per plugin. The call sites use `query()`, since `lookup()` joins several results into one string and a loop needs the list. Co-Authored-By: Claude Opus 5 (1M context) --- plugins/lookup/node_hosts.py | 36 ++++++++++++++++ .../tasks/00_core.yml | 3 ++ roles/svc-ai-s1/tasks/00_core.yml | 3 ++ .../python/plugins/lookup/test_node_hosts.py | 43 +++++++++++++++++++ 4 files changed, 85 insertions(+) create mode 100644 plugins/lookup/node_hosts.py create mode 100644 tests/unit/python/plugins/lookup/test_node_hosts.py diff --git a/plugins/lookup/node_hosts.py b/plugins/lookup/node_hosts.py new file mode 100644 index 0000000000..154aad1c24 --- /dev/null +++ b/plugins/lookup/node_hosts.py @@ -0,0 +1,36 @@ +"""Every host a node-local path must exist on before a container binds it. + +Swarm rejects a task whose bind source is missing on the node it lands on, and +a role that declares no ``placement`` may be scheduled onto any node, so the +path has to exist on all of them. Without a swarm group there is one host, the +one the task already runs on. +""" + +from __future__ import annotations + +from typing import Any + +from ansible.plugins.lookup import LookupBase + +SWARM_GROUP = "svc-swarm-node" + + +def node_hosts(vars_: dict[str, Any]) -> list[str]: + """Return every host a node-local path must exist on. + + Args: + vars_: the task variables the lookup was called with. + """ + hosts = [str(host) for host in (vars_.get("groups") or {}).get(SWARM_GROUP) or []] + return hosts or [str(vars_["inventory_hostname"])] + + +class LookupModule(LookupBase): + def run( + self, + terms: list[Any], + variables: dict[str, Any] | None = None, + **kwargs: Any, + ) -> list[str]: + vars_ = variables or getattr(self._templar, "available_variables", {}) or {} + return node_hosts(vars_) diff --git a/roles/svc-ai-libretranslate-engine/tasks/00_core.yml b/roles/svc-ai-libretranslate-engine/tasks/00_core.yml index 81782f639e..5ea3a13763 100644 --- a/roles/svc-ai-libretranslate-engine/tasks/00_core.yml +++ b/roles/svc-ai-libretranslate-engine/tasks/00_core.yml @@ -7,6 +7,9 @@ state: directory owner: "{{ LIBRETRANSLATE_ENGINE_UID }}" mode: "0755" + delegate_to: "{{ item }}" + run_once: true + loop: "{{ query('node_hosts') }}" - name: "🧩 load the container manager for {{ application_id }}" include_role: diff --git a/roles/svc-ai-s1/tasks/00_core.yml b/roles/svc-ai-s1/tasks/00_core.yml index 1e320e23b0..1f28d2c943 100644 --- a/roles/svc-ai-s1/tasks/00_core.yml +++ b/roles/svc-ai-s1/tasks/00_core.yml @@ -19,6 +19,9 @@ path: "{{ S1_MODELS_DIR }}" state: directory mode: "0755" + delegate_to: "{{ item }}" + run_once: true + loop: "{{ query('node_hosts') }}" - name: "🎮 Look for the GPU and the runtime that hands it to a container" ansible.builtin.stat: diff --git a/tests/unit/python/plugins/lookup/test_node_hosts.py b/tests/unit/python/plugins/lookup/test_node_hosts.py new file mode 100644 index 0000000000..01ed1354be --- /dev/null +++ b/tests/unit/python/plugins/lookup/test_node_hosts.py @@ -0,0 +1,43 @@ +import unittest + +from plugins.lookup.node_hosts import SWARM_GROUP, LookupModule, node_hosts + +LOCAL = "localhost" +NODES = ["swarm-mgr-01", "swarm-wrk-01", "swarm-wrk-02"] + + +class TestNodeHosts(unittest.TestCase): + def test_every_swarm_node_is_returned(self) -> None: + self.assertEqual( + node_hosts({"groups": {SWARM_GROUP: NODES}, "inventory_hostname": LOCAL}), + NODES, + ) + + def test_without_the_group_the_local_host_stands_alone(self) -> None: + self.assertEqual( + node_hosts({"groups": {}, "inventory_hostname": LOCAL}), [LOCAL] + ) + + def test_an_empty_group_falls_back_to_the_local_host(self) -> None: + """An empty group would otherwise create the path on no host at all.""" + self.assertEqual( + node_hosts({"groups": {SWARM_GROUP: []}, "inventory_hostname": LOCAL}), + [LOCAL], + ) + + def test_absent_groups_are_not_an_error(self) -> None: + self.assertEqual(node_hosts({"inventory_hostname": LOCAL}), [LOCAL]) + + def test_the_lookup_reads_the_variables_it_is_handed(self) -> None: + lookup = LookupModule() + lookup._templar = None + self.assertEqual( + lookup.run( + [], {"groups": {SWARM_GROUP: NODES}, "inventory_hostname": LOCAL} + ), + NODES, + ) + + +if __name__ == "__main__": + unittest.main()