diff --git a/plugins/lookup/gpu.py b/plugins/lookup/gpu.py index ef15fa69c9..ba40fa231b 100644 --- a/plugins/lookup/gpu.py +++ b/plugins/lookup/gpu.py @@ -3,7 +3,8 @@ Two facts have to agree and used to be restated at every call site. The declaration lives in ``applications`` under ``services..gpu``; the hardware is ``sys-svc-container``'s stat of ``/dev/nvidia0`` on the target -host, registered as ``sys_svc_container_nvidia_device``. Asking only the +host, registered as ``sys_svc_container_nvidia_device`` and skipped where the +NVIDIA driver is not installed. Asking only the declaration selects a ``-cuda`` image on a host that has no device, and that tag has no arm64 manifest at all, so the build fails with ``no match for platform in manifest``. diff --git a/roles/sys-svc-container/tasks/00_core.yml b/roles/sys-svc-container/tasks/00_core.yml index 7de40e0ec5..25b5d0b249 100644 --- a/roles/sys-svc-container/tasks/00_core.yml +++ b/roles/sys-svc-container/tasks/00_core.yml @@ -30,14 +30,22 @@ until: sys_svc_container_packages_install is succeeded notify: container-manager-restart -- name: "🎮 Check whether this host carries an NVIDIA GPU" +- name: "🎮 Check whether this host carries the NVIDIA driver" + ansible.builtin.stat: + path: "{{ SYS_SVC_CONTAINER_GPU_DRIVER }}" + register: sys_svc_container_nvidia_driver + +- name: "🎮 Check whether this host carries an NVIDIA GPU (only beside the driver: a privileged container sees the host's /dev/nvidia0 without it)" ansible.builtin.stat: path: /dev/nvidia0 register: sys_svc_container_nvidia_device + when: sys_svc_container_nvidia_driver.stat.exists - name: "🎮 Hand this host's GPUs to the containers that asked for one" include_tasks: 00_core/gpu.yml - when: sys_svc_container_nvidia_device.stat.exists + when: + - sys_svc_container_nvidia_device is not skipped + - sys_svc_container_nvidia_device.stat.exists - name: 📦 Install Docker Python SDK dependencies via pip include_role: diff --git a/roles/sys-svc-container/vars/main.yml b/roles/sys-svc-container/vars/main.yml index 088c223611..f8906a22f5 100644 --- a/roles/sys-svc-container/vars/main.yml +++ b/roles/sys-svc-container/vars/main.yml @@ -34,6 +34,7 @@ SYS_SVC_CONTAINER_NESTED_VIRTUALIZATION: - lxc - container -SYS_SVC_CONTAINER_GPU_QUERY: "nvidia-smi --query-gpu=memory.total,uuid --format=csv,noheader,nounits" +SYS_SVC_CONTAINER_GPU_DRIVER: /usr/bin/nvidia-smi +SYS_SVC_CONTAINER_GPU_QUERY: "{{ SYS_SVC_CONTAINER_GPU_DRIVER }} --query-gpu=memory.total,uuid --format=csv,noheader,nounits" SYS_SVC_CONTAINER_PREPULL_ASYNC_TIME: "{{ lookup('timeout', 3600) | int }}" diff --git a/tests/unit/python/plugins/lookup/test_gpu.py b/tests/unit/python/plugins/lookup/test_gpu.py index 7132657c20..08dbccb666 100644 --- a/tests/unit/python/plugins/lookup/test_gpu.py +++ b/tests/unit/python/plugins/lookup/test_gpu.py @@ -77,6 +77,21 @@ class TestGpuLookup(unittest.TestCase): _run(lm, ["web-svc-libretranslate", "libretranslate"]), [False] ) + def test_a_device_stat_skipped_for_a_missing_driver_is_false(self): + """A privileged container sees the host's /dev/nvidia0 without the driver.""" + lm = _make_lookup( + { + "sys_svc_container_nvidia_device": { + "changed": False, + "skipped": True, + "false_condition": "sys_svc_container_nvidia_driver.stat.exists", + } + } + ) + self.assertEqual( + _run(lm, ["web-svc-libretranslate", "libretranslate"]), [False] + ) + def test_non_mapping_register_is_false(self): lm = _make_lookup({"sys_svc_container_nvidia_device": "skipped"}) self.assertEqual( diff --git a/tests/unit/python/roles/sys-svc-container/test_gpu_detection.py b/tests/unit/python/roles/sys-svc-container/test_gpu_detection.py new file mode 100644 index 0000000000..f685883dfd --- /dev/null +++ b/tests/unit/python/roles/sys-svc-container/test_gpu_detection.py @@ -0,0 +1,109 @@ +"""Which hosts the role treats as GPU hosts. + +A privileged container, such as a swarm lab node on a workstation with an +NVIDIA card, sees the host's ``/dev/nvidia0`` but carries no driver, so +``nvidia-smi`` does not exist there. The device node alone must therefore not +make a GPU host: both the GPU include and the ``gpu`` lookup read the same +register, and both have to say no on such a host. +""" + +import unittest +from pathlib import Path + +from jinja2 import Environment + +from plugins.lookup.gpu import _device_present +from utils import PROJECT_ROOT +from utils.cache.yaml import load_yaml_any + +ROLE = Path(PROJECT_ROOT) / "roles" / "sys-svc-container" +CORE = ROLE / "tasks" / "00_core.yml" +GPU_INCLUDE = "00_core/gpu.yml" +DEVICE = "/dev/nvidia0" + +_ENV = Environment(autoescape=True) +_ENV.tests["skipped"] = lambda result: ( + isinstance(result, dict) and bool(result.get("skipped")) +) + + +def _role_vars() -> dict: + return load_yaml_any(str(ROLE / "vars" / "main.yml")) + + +def _render(value: str, context: dict) -> str: + return _ENV.from_string(value).render(**context) + + +def _holds(task: dict, context: dict) -> bool: + when = task.get("when", []) + guards = when if isinstance(when, list) else [when] + return all(bool(_ENV.compile_expression(expr)(**context)) for expr in guards) + + +def _walk(*, driver: bool, device: bool) -> tuple[bool, dict]: + """Run the GPU detection of 00_core.yml over a fake filesystem. + + Args: + driver: whether the NVIDIA driver binary exists on the host. + device: whether the host shows an NVIDIA device node. + + Returns: + Whether the GPU include runs, and the registers the tasks left. + """ + context = dict(_role_vars()) + existing = { + _render(context["SYS_SVC_CONTAINER_GPU_DRIVER"], context): driver, + DEVICE: device, + } + included = False + for task in load_yaml_any(str(CORE)): + stat = task.get("ansible.builtin.stat") + if stat is not None: + register = task["register"] + if not _holds(task, context): + context[register] = {"changed": False, "skipped": True} + continue + path = _render(stat["path"], context) + context[register] = {"stat": {"exists": existing.get(path, False)}} + elif task.get("include_tasks") == GPU_INCLUDE: + included = _holds(task, context) + return included, context + + +class TestGpuDetection(unittest.TestCase): + def test_a_device_node_without_the_driver_is_no_gpu_host(self): + included, registers = _walk(driver=False, device=True) + self.assertFalse( + included, + "a privileged container sees the host's /dev/nvidia0 without " + "nvidia-smi; the include would fail on the missing binary", + ) + self.assertFalse(_device_present(registers)) + + def test_driver_and_device_make_a_gpu_host(self): + included, registers = _walk(driver=True, device=True) + self.assertTrue(included) + self.assertTrue(_device_present(registers)) + + def test_a_driver_without_a_device_is_no_gpu_host(self): + included, registers = _walk(driver=True, device=False) + self.assertFalse(included) + self.assertFalse(_device_present(registers)) + + def test_neither_is_no_gpu_host(self): + included, registers = _walk(driver=False, device=False) + self.assertFalse(included) + self.assertFalse(_device_present(registers)) + + def test_the_query_runs_the_binary_the_detection_probed(self): + role_vars = _role_vars() + self.assertTrue( + role_vars["SYS_SVC_CONTAINER_GPU_QUERY"].startswith( + "{{ SYS_SVC_CONTAINER_GPU_DRIVER }} " + ) + ) + + +if __name__ == "__main__": + unittest.main()