diff --git a/group_vars/all/19_virt.yml b/group_vars/all/19_virt.yml index 80c9843184..4cb99178c0 100644 --- a/group_vars/all/19_virt.yml +++ b/group_vars/all/19_virt.yml @@ -2,5 +2,6 @@ KATA_SHIM_BINARY: /usr/bin/containerd-shim-kata-v2 RUNSC_SHIM_BINARY: /usr/local/bin/runsc NVIDIA_RUNTIME_BINARY: /usr/bin/nvidia-container-runtime +NVIDIA_DRIVER_CAPABILITIES: compute,utility SANDBOX_NODE_LABEL: kata-capable SANDBOX_RUNTIME: runsc diff --git a/roles/sys-svc-container/tasks/00_core.yml b/roles/sys-svc-container/tasks/00_core.yml index c6f161e7ff..03c678ee2b 100644 --- a/roles/sys-svc-container/tasks/00_core.yml +++ b/roles/sys-svc-container/tasks/00_core.yml @@ -35,11 +35,9 @@ path: /dev/nvidia0 register: sys_svc_container_nvidia_device -- name: "๐ŸŽฎ Let the container manager hand that GPU to its containers" - include_tasks: 00_core/nvidia.yml - when: - - sys_svc_container_nvidia_device.stat.exists - - ansible_facts['distribution'] in ['Debian', 'Ubuntu'] +- name: "๐ŸŽฎ Hand this host's GPUs to the containers that asked for one" + include_tasks: 00_core/gpu.yml + when: sys_svc_container_nvidia_device.stat.exists - name: ๐Ÿ“ฆ Install Docker Python SDK dependencies via pip include_role: diff --git a/roles/sys-svc-container/tasks/00_core/gpu.yml b/roles/sys-svc-container/tasks/00_core/gpu.yml new file mode 100644 index 0000000000..892a778cc3 --- /dev/null +++ b/roles/sys-svc-container/tasks/00_core/gpu.yml @@ -0,0 +1,23 @@ +--- +- name: "๐ŸŽฎ Ask the driver which CUDA devices this host carries" + ansible.builtin.command: + cmd: "{{ SYS_SVC_CONTAINER_GPU_QUERY }}" + register: sys_svc_container_nvidia_inventory + changed_when: false + +- name: "๐ŸŽฎ Publish those devices, the one with the most memory first" + ansible.builtin.set_fact: + SYS_SVC_CONTAINER_GPU_DEVICES: >- + {{ (sys_svc_container_nvidia_rows | map(attribute='0') | map('int') | list) + | zip(sys_svc_container_nvidia_rows | map(attribute='1') | list) + | sort(reverse=true) | map(attribute='1') | join(',') }} + vars: + sys_svc_container_nvidia_rows: >- + {{ sys_svc_container_nvidia_inventory.stdout_lines | map('split', ', ') | list }} + +- name: "๐Ÿ“ฃ Report the devices a GPU service will claim" + ansible.builtin.debug: + msg: "NVIDIA_VISIBLE_DEVICES={{ SYS_SVC_CONTAINER_GPU_DEVICES }}" + +- name: "๐ŸŽฎ Let the container manager hand that GPU to its containers" + include_tasks: nvidia.yml diff --git a/roles/sys-svc-container/tasks/00_core/nvidia.yml b/roles/sys-svc-container/tasks/00_core/nvidia.yml index bb812bc45a..926ca71ce3 100644 --- a/roles/sys-svc-container/tasks/00_core/nvidia.yml +++ b/roles/sys-svc-container/tasks/00_core/nvidia.yml @@ -1,49 +1,11 @@ --- -- name: "๐Ÿ”‘ Fetch the NVIDIA container toolkit apt key" - get_url_retry: - url: https://nvidia.github.io/libnvidia-container/gpgkey - dest: /etc/apt/keyrings/nvidia-container-toolkit.asc - owner: root - group: root - mode: "0644" +- name: "๐Ÿ”ง Configure the NVIDIA container toolkit repository on Debian and Ubuntu" + include_tasks: nvidia/debian.yml + when: ansible_facts['distribution'] in ['Debian', 'Ubuntu'] -- name: "๐Ÿ” Find rival NVIDIA container toolkit apt sources" - ansible.builtin.find: - paths: /etc/apt/sources.list.d - contains: .*libnvidia-container.* - excludes: nvidia-container-toolkit.list - register: sys_svc_container_nvidia_rivals - -- name: "๐Ÿงน Drop rival NVIDIA container toolkit apt sources" - ansible.builtin.file: - path: "{{ item.path }}" - state: absent - loop: "{{ sys_svc_container_nvidia_rivals.files }}" - loop_control: - label: "{{ item.path }}" - register: sys_svc_container_nvidia_rivals_dropped - -- name: "๐Ÿ”ง Configure the NVIDIA container toolkit apt repository" - ansible.builtin.copy: - dest: /etc/apt/sources.list.d/nvidia-container-toolkit.list - content: >- - deb [signed-by=/etc/apt/keyrings/nvidia-container-toolkit.asc] - https://nvidia.github.io/libnvidia-container/stable/deb/$(ARCH) / - owner: root - group: root - mode: "0644" - register: sys_svc_container_nvidia_repo_file - -- name: "๐Ÿ”„ Update apt cache after the NVIDIA repo change" - ansible.builtin.apt: # nocheck: dynamic-package - apt repository and cache setup, not a package install - update_cache: true - when: >- - sys_svc_container_nvidia_repo_file is changed - or sys_svc_container_nvidia_rivals_dropped is changed - register: sys_svc_container_nvidia_cache - until: sys_svc_container_nvidia_cache is succeeded - retries: 5 - delay: 15 +- name: "๐Ÿ”ง Configure the NVIDIA container toolkit repository on the RedHat family" + include_tasks: nvidia/redhat.yml + when: ansible_facts['os_family'] == 'RedHat' - name: "๐Ÿ“ฆ Install the NVIDIA container toolkit" package_install: @@ -52,3 +14,15 @@ until: sys_svc_container_nvidia_install is succeeded retries: 5 delay: 15 + +- name: "๐ŸŽฎ Skip the cgroup setup nvidia-container-cli cannot own in a container" + ansible.builtin.lineinfile: + path: /etc/nvidia-container-runtime/config.toml + regexp: '^#?\s*no-cgroups\s*=' + line: "no-cgroups = true" + owner: root + group: root + mode: "0644" + when: >- + ansible_facts['virtualization_type'] + in SYS_SVC_CONTAINER_NESTED_VIRTUALIZATION diff --git a/roles/sys-svc-container/tasks/00_core/nvidia/debian.yml b/roles/sys-svc-container/tasks/00_core/nvidia/debian.yml new file mode 100644 index 0000000000..a6ebd59638 --- /dev/null +++ b/roles/sys-svc-container/tasks/00_core/nvidia/debian.yml @@ -0,0 +1,46 @@ +--- +- name: "๐Ÿ”‘ Fetch the NVIDIA container toolkit apt key" + get_url_retry: + url: https://nvidia.github.io/libnvidia-container/gpgkey + dest: /etc/apt/keyrings/nvidia-container-toolkit.asc + owner: root + group: root + mode: "0644" + +- name: "๐Ÿ” Find rival NVIDIA container toolkit apt sources" + ansible.builtin.find: + paths: /etc/apt/sources.list.d + contains: .*libnvidia-container.* + excludes: nvidia-container-toolkit.list + register: sys_svc_container_nvidia_rivals + +- name: "๐Ÿงน Drop rival NVIDIA container toolkit apt sources" + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + loop: "{{ sys_svc_container_nvidia_rivals.files }}" + loop_control: + label: "{{ item.path }}" + register: sys_svc_container_nvidia_rivals_dropped + +- name: "๐Ÿ”ง Configure the NVIDIA container toolkit apt repository" + ansible.builtin.copy: + dest: /etc/apt/sources.list.d/nvidia-container-toolkit.list + content: >- + deb [signed-by=/etc/apt/keyrings/nvidia-container-toolkit.asc] + https://nvidia.github.io/libnvidia-container/stable/deb/$(ARCH) / + owner: root + group: root + mode: "0644" + register: sys_svc_container_nvidia_repo_file + +- name: "๐Ÿ”„ Update apt cache after the NVIDIA repo change" + ansible.builtin.apt: # nocheck: dynamic-package - apt repository and cache setup, not a package install + update_cache: true + when: >- + sys_svc_container_nvidia_repo_file is changed + or sys_svc_container_nvidia_rivals_dropped is changed + register: sys_svc_container_nvidia_cache + until: sys_svc_container_nvidia_cache is succeeded + retries: 5 + delay: 15 diff --git a/roles/sys-svc-container/tasks/00_core/nvidia/redhat.yml b/roles/sys-svc-container/tasks/00_core/nvidia/redhat.yml new file mode 100644 index 0000000000..e215308cac --- /dev/null +++ b/roles/sys-svc-container/tasks/00_core/nvidia/redhat.yml @@ -0,0 +1,6 @@ +--- +- name: "๐Ÿ”ง Configure the NVIDIA container toolkit yum repository" + get_url_retry: + url: https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo + dest: /etc/yum.repos.d/nvidia-container-toolkit.repo + mode: "0644" diff --git a/roles/sys-svc-container/templates/resource.yml.j2 b/roles/sys-svc-container/templates/resource.yml.j2 index 845b0fa4e0..6eda55df32 100644 --- a/roles/sys-svc-container/templates/resource.yml.j2 +++ b/roles/sys-svc-container/templates/resource.yml.j2 @@ -4,4 +4,7 @@ mem_limit: {{ lookup('applications') | resource_filter(application_id, pids_limit: {{ lookup('applications') | resource_filter(application_id, 'pids_limit', service_name | default(''), RESOURCE_PIDS_LIMIT) }} {% if lookup('gpu', application_id, service_name | default('')) %} runtime: nvidia +environment: + NVIDIA_VISIBLE_DEVICES: "{{ SYS_SVC_CONTAINER_GPU_DEVICES }}" + NVIDIA_DRIVER_CAPABILITIES: "{{ NVIDIA_DRIVER_CAPABILITIES }}" {% endif %} diff --git a/roles/sys-svc-container/vars/main.yml b/roles/sys-svc-container/vars/main.yml index c6df0f88e2..4b67cf41cf 100644 --- a/roles/sys-svc-container/vars/main.yml +++ b/roles/sys-svc-container/vars/main.yml @@ -24,3 +24,12 @@ SYS_SVC_CONTAINER_DPKG_ARCH_BY_ANSIBLE_ARCH: SYS_SVC_CONTAINER_DOCKER_FIREWALL_BACKEND_BY_OS_FAMILY: Archlinux: nftables + +SYS_SVC_CONTAINER_NESTED_VIRTUALIZATION: + - docker + - containerd + - podman + - lxc + - container + +SYS_SVC_CONTAINER_GPU_QUERY: "nvidia-smi --query-gpu=memory.total,uuid --format=csv,noheader,nounits" diff --git a/tests/unit/python/roles/sys-svc-container/test_gpu_runtime.py b/tests/unit/python/roles/sys-svc-container/test_gpu_runtime.py index 78342020a7..74f1cc140e 100644 --- a/tests/unit/python/roles/sys-svc-container/test_gpu_runtime.py +++ b/tests/unit/python/roles/sys-svc-container/test_gpu_runtime.py @@ -1,8 +1,12 @@ -"""The ``runtime: nvidia`` key of the shared resource template. +"""The GPU keys of the shared resource template. The template asks the ``gpu`` lookup and decides nothing itself: combining the declared flag with the host's device is that lookup's job, and its own tests own the matrix. + +``runtime: nvidia`` alone hands the container nothing. The runtime's hook +reads ``NVIDIA_VISIBLE_DEVICES`` off the container's environment and is a +no-op while that is unset, so the two keys below belong to the same branch. """ import unittest @@ -27,6 +31,8 @@ BASE = { "RESOURCE_MEM_LIMIT": "1g", "RESOURCE_MEM_RESERVATION": "256m", "RESOURCE_PIDS_LIMIT": "512", + "SYS_SVC_CONTAINER_GPU_DEVICES": "GPU-strong,GPU-weak", + "NVIDIA_DRIVER_CAPABILITIES": "compute,utility", } @@ -57,6 +63,21 @@ class TestGpuRuntimeRegistration(unittest.TestCase): def test_the_lookups_no_leaves_the_runtime_key_out(self): self.assertNotIn("runtime:", self._render(gpu=False)) + def test_the_runtime_is_told_which_devices_to_hand_over(self): + self.assertIn( + 'NVIDIA_VISIBLE_DEVICES: "GPU-strong,GPU-weak"', + self._render(gpu=True), + ) + + def test_the_runtime_is_told_to_inject_the_cuda_driver(self): + self.assertIn( + 'NVIDIA_DRIVER_CAPABILITIES: "compute,utility"', + self._render(gpu=True), + ) + + def test_the_lookups_no_leaves_the_device_env_out(self): + self.assertNotIn("NVIDIA_VISIBLE_DEVICES", self._render(gpu=False)) + def test_the_lookup_is_asked_for_this_service(self): self._render(gpu=True)