From ae259a9f58856cdeb4f7190bb782242396a67e32 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 30 Oct 2026 11:44:54 +0100 Subject: [PATCH] fix(sys-svc-container): count a host as GPU host only beside the NVIDIA driver The local 061 Nextcloud swarm step failed on every lab node in "Ask the driver which CUDA devices this host carries" with "[Errno 2] No such file or directory: b'nvidia-smi'". The lab nodes are privileged containers (compose/swarm/compose.yml), so on a workstation with an NVIDIA card they see the host's /dev/nvidia0, but they carry no driver. The device node alone made them GPU hosts, for the GPU include and for lookup('gpu') alike, which would also have picked a -cuda image and runtime: nvidia for a GPU service there. The role now stats SYS_SVC_CONTAINER_GPU_DRIVER (/usr/bin/nvidia-smi) first and stats the device node only where the driver exists; the skipped register reads as "no device" in lookup('gpu'), and the include requires a device stat that ran and found the node. The query runs the same binary the detection probed. Hosts with driver and device (bare metal with the driver, the compose dev container through the GPU override) and hosts without a device (CI) behave as before. test_gpu_detection walks the real tasks over the four driver/device cells and fails on the old 00_core.yml in exactly the device-without- driver cell. Co-Authored-By: Claude Opus 5.5 --- plugins/lookup/gpu.py | 3 +- roles/sys-svc-container/tasks/00_core.yml | 12 +- roles/sys-svc-container/vars/main.yml | 3 +- tests/unit/python/plugins/lookup/test_gpu.py | 15 +++ .../sys-svc-container/test_gpu_detection.py | 109 ++++++++++++++++++ 5 files changed, 138 insertions(+), 4 deletions(-) create mode 100644 tests/unit/python/roles/sys-svc-container/test_gpu_detection.py diff --git a/plugins/lookup/gpu.py b/plugins/lookup/gpu.py index ef15fa69c9..ba40fa231b 100644 --- a/plugins/lookup/gpu.py +++ b/plugins/lookup/gpu.py @@ -3,7 +3,8 @@ Two facts have to agree and used to be restated at every call site. The declaration lives in ``applications`` under ``services..gpu``; the hardware is ``sys-svc-container``'s stat of ``/dev/nvidia0`` on the target -host, registered as ``sys_svc_container_nvidia_device``. Asking only the +host, registered as ``sys_svc_container_nvidia_device`` and skipped where the +NVIDIA driver is not installed. Asking only the declaration selects a ``-cuda`` image on a host that has no device, and that tag has no arm64 manifest at all, so the build fails with ``no match for platform in manifest``. diff --git a/roles/sys-svc-container/tasks/00_core.yml b/roles/sys-svc-container/tasks/00_core.yml index 7de40e0ec5..25b5d0b249 100644 --- a/roles/sys-svc-container/tasks/00_core.yml +++ b/roles/sys-svc-container/tasks/00_core.yml @@ -30,14 +30,22 @@ until: sys_svc_container_packages_install is succeeded notify: container-manager-restart -- name: "🎮 Check whether this host carries an NVIDIA GPU" +- name: "🎮 Check whether this host carries the NVIDIA driver" + ansible.builtin.stat: + path: "{{ SYS_SVC_CONTAINER_GPU_DRIVER }}" + register: sys_svc_container_nvidia_driver + +- name: "🎮 Check whether this host carries an NVIDIA GPU (only beside the driver: a privileged container sees the host's /dev/nvidia0 without it)" ansible.builtin.stat: path: /dev/nvidia0 register: sys_svc_container_nvidia_device + when: sys_svc_container_nvidia_driver.stat.exists - name: "🎮 Hand this host's GPUs to the containers that asked for one" include_tasks: 00_core/gpu.yml - when: sys_svc_container_nvidia_device.stat.exists + when: + - sys_svc_container_nvidia_device is not skipped + - sys_svc_container_nvidia_device.stat.exists - name: 📦 Install Docker Python SDK dependencies via pip include_role: diff --git a/roles/sys-svc-container/vars/main.yml b/roles/sys-svc-container/vars/main.yml index 088c223611..f8906a22f5 100644 --- a/roles/sys-svc-container/vars/main.yml +++ b/roles/sys-svc-container/vars/main.yml @@ -34,6 +34,7 @@ SYS_SVC_CONTAINER_NESTED_VIRTUALIZATION: - lxc - container -SYS_SVC_CONTAINER_GPU_QUERY: "nvidia-smi --query-gpu=memory.total,uuid --format=csv,noheader,nounits" +SYS_SVC_CONTAINER_GPU_DRIVER: /usr/bin/nvidia-smi +SYS_SVC_CONTAINER_GPU_QUERY: "{{ SYS_SVC_CONTAINER_GPU_DRIVER }} --query-gpu=memory.total,uuid --format=csv,noheader,nounits" SYS_SVC_CONTAINER_PREPULL_ASYNC_TIME: "{{ lookup('timeout', 3600) | int }}" diff --git a/tests/unit/python/plugins/lookup/test_gpu.py b/tests/unit/python/plugins/lookup/test_gpu.py index 7132657c20..08dbccb666 100644 --- a/tests/unit/python/plugins/lookup/test_gpu.py +++ b/tests/unit/python/plugins/lookup/test_gpu.py @@ -77,6 +77,21 @@ class TestGpuLookup(unittest.TestCase): _run(lm, ["web-svc-libretranslate", "libretranslate"]), [False] ) + def test_a_device_stat_skipped_for_a_missing_driver_is_false(self): + """A privileged container sees the host's /dev/nvidia0 without the driver.""" + lm = _make_lookup( + { + "sys_svc_container_nvidia_device": { + "changed": False, + "skipped": True, + "false_condition": "sys_svc_container_nvidia_driver.stat.exists", + } + } + ) + self.assertEqual( + _run(lm, ["web-svc-libretranslate", "libretranslate"]), [False] + ) + def test_non_mapping_register_is_false(self): lm = _make_lookup({"sys_svc_container_nvidia_device": "skipped"}) self.assertEqual( diff --git a/tests/unit/python/roles/sys-svc-container/test_gpu_detection.py b/tests/unit/python/roles/sys-svc-container/test_gpu_detection.py new file mode 100644 index 0000000000..f685883dfd --- /dev/null +++ b/tests/unit/python/roles/sys-svc-container/test_gpu_detection.py @@ -0,0 +1,109 @@ +"""Which hosts the role treats as GPU hosts. + +A privileged container, such as a swarm lab node on a workstation with an +NVIDIA card, sees the host's ``/dev/nvidia0`` but carries no driver, so +``nvidia-smi`` does not exist there. The device node alone must therefore not +make a GPU host: both the GPU include and the ``gpu`` lookup read the same +register, and both have to say no on such a host. +""" + +import unittest +from pathlib import Path + +from jinja2 import Environment + +from plugins.lookup.gpu import _device_present +from utils import PROJECT_ROOT +from utils.cache.yaml import load_yaml_any + +ROLE = Path(PROJECT_ROOT) / "roles" / "sys-svc-container" +CORE = ROLE / "tasks" / "00_core.yml" +GPU_INCLUDE = "00_core/gpu.yml" +DEVICE = "/dev/nvidia0" + +_ENV = Environment(autoescape=True) +_ENV.tests["skipped"] = lambda result: ( + isinstance(result, dict) and bool(result.get("skipped")) +) + + +def _role_vars() -> dict: + return load_yaml_any(str(ROLE / "vars" / "main.yml")) + + +def _render(value: str, context: dict) -> str: + return _ENV.from_string(value).render(**context) + + +def _holds(task: dict, context: dict) -> bool: + when = task.get("when", []) + guards = when if isinstance(when, list) else [when] + return all(bool(_ENV.compile_expression(expr)(**context)) for expr in guards) + + +def _walk(*, driver: bool, device: bool) -> tuple[bool, dict]: + """Run the GPU detection of 00_core.yml over a fake filesystem. + + Args: + driver: whether the NVIDIA driver binary exists on the host. + device: whether the host shows an NVIDIA device node. + + Returns: + Whether the GPU include runs, and the registers the tasks left. + """ + context = dict(_role_vars()) + existing = { + _render(context["SYS_SVC_CONTAINER_GPU_DRIVER"], context): driver, + DEVICE: device, + } + included = False + for task in load_yaml_any(str(CORE)): + stat = task.get("ansible.builtin.stat") + if stat is not None: + register = task["register"] + if not _holds(task, context): + context[register] = {"changed": False, "skipped": True} + continue + path = _render(stat["path"], context) + context[register] = {"stat": {"exists": existing.get(path, False)}} + elif task.get("include_tasks") == GPU_INCLUDE: + included = _holds(task, context) + return included, context + + +class TestGpuDetection(unittest.TestCase): + def test_a_device_node_without_the_driver_is_no_gpu_host(self): + included, registers = _walk(driver=False, device=True) + self.assertFalse( + included, + "a privileged container sees the host's /dev/nvidia0 without " + "nvidia-smi; the include would fail on the missing binary", + ) + self.assertFalse(_device_present(registers)) + + def test_driver_and_device_make_a_gpu_host(self): + included, registers = _walk(driver=True, device=True) + self.assertTrue(included) + self.assertTrue(_device_present(registers)) + + def test_a_driver_without_a_device_is_no_gpu_host(self): + included, registers = _walk(driver=True, device=False) + self.assertFalse(included) + self.assertFalse(_device_present(registers)) + + def test_neither_is_no_gpu_host(self): + included, registers = _walk(driver=False, device=False) + self.assertFalse(included) + self.assertFalse(_device_present(registers)) + + def test_the_query_runs_the_binary_the_detection_probed(self): + role_vars = _role_vars() + self.assertTrue( + role_vars["SYS_SVC_CONTAINER_GPU_QUERY"].startswith( + "{{ SYS_SVC_CONTAINER_GPU_DRIVER }} " + ) + ) + + +if __name__ == "__main__": + unittest.main()