From 9b7f25f2feedd05926e3cfedcdca14f617c5b9f5 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 20 Oct 2026 16:28:34 +0200 Subject: [PATCH] fix(svc-storage-nfs-client): stop requesting directory delegations from ganesha < 9.11 Every swarm job on ubuntu-26.04 runners (kernel 7.0) died at the NFS probe write with `[Errno 121] Remote I/O error`. ganesha's NFSv4 debug log in the rescue bundles shows the cause on every client, the same second as the failure: Request 2: opcode 46 is OP_GET_DIR_DELEGATION Status of OP_GET_DIR_DELEGATION in position 2 = NFS4ERR_OP_ILLEGAL Linux 6.19 added directory delegations: after ACCESS, CREATE, REMOVE or RENAME the client adds GET_DIR_DELEGATION to the directory's next GETATTR (`should_request_dir_deleg`, gated per request by the nfsv4 parameter `directory_delegations`). nfs-ganesha before 9.11 routes op 46 to `nfs4_op_illegal`, whose reply carries opcode 10044; the client decoder expects 46 and fails the call with EREMOTEIO. Upstream fixed it in nfs-ganesha#1385 (98eb77c4, first in V9.11), which answers NOTSUPP; the client then drops the capability. The distro packages the role installs predate it: Debian trixie 6.5-5, Ubuntu 26.04 6.5-8. Every client and the controller now read the server's installed nfs-ganesha release and, below 9.11, set `directory_delegations=N` right after the mount and before the first access (the mount itself never sends op 46), plus `options nfsv4 directory_delegations=N` in /etc/modprobe.d for later module loads. No ganesha release grants directory delegations, so the end state equals a fixed server. The parameter is kernel-wide and is only ever set to N, so clients sharing one kernel cannot flip it back on each other. Servers on 9.11 or later and the kernel nfsd flavor keep the client default. Co-Authored-By: Claude Opus 5.5 (1M context) --- roles/svc-storage-nfs-client/README.md | 6 +++++ .../files/shell/directory_delegations.sh | 8 ++++++ .../tasks/directory_delegations.yml | 27 +++++++++++++++++++ roles/svc-storage-nfs-client/tasks/main.yml | 4 +++ .../tasks/03_controller_mount.yml | 4 +++ 5 files changed, 49 insertions(+) create mode 100644 roles/svc-storage-nfs-client/files/shell/directory_delegations.sh create mode 100644 roles/svc-storage-nfs-client/tasks/directory_delegations.yml diff --git a/roles/svc-storage-nfs-client/README.md b/roles/svc-storage-nfs-client/README.md index ec97c70692..b526991b84 100644 --- a/roles/svc-storage-nfs-client/README.md +++ b/roles/svc-storage-nfs-client/README.md @@ -50,6 +50,12 @@ Solid `1:1` edges are fixed relationships; dashed `0..1` edges are conditional ( Generations written to the share before the bind stay under the export's `backup/` directory, hidden from the nodes; remove them there by hand. +- **Directory delegations off against old ganesha:** When the server + runs nfs-ganesha older than 9.11, every client and the controller set + the kernel's `nfsv4` parameter `directory_delegations` to `N`, at + runtime and in `/etc/modprobe.d`. The parameter is kernel-wide, so on + a lab whose nodes share one kernel it covers every NFS mount of that + host until the module reloads. ## Quick Setup diff --git a/roles/svc-storage-nfs-client/files/shell/directory_delegations.sh b/roles/svc-storage-nfs-client/files/shell/directory_delegations.sh new file mode 100644 index 0000000000..2d95882dbd --- /dev/null +++ b/roles/svc-storage-nfs-client/files/shell/directory_delegations.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -eu +param=/sys/module/nfsv4/parameters/directory_delegations + +[ -e "${param}" ] || exit 0 +[ "$(cat "${param}")" = N ] && exit 0 +echo N >"${param}" +echo CHANGED diff --git a/roles/svc-storage-nfs-client/tasks/directory_delegations.yml b/roles/svc-storage-nfs-client/tasks/directory_delegations.yml new file mode 100644 index 0000000000..e7418ef8f2 --- /dev/null +++ b/roles/svc-storage-nfs-client/tasks/directory_delegations.yml @@ -0,0 +1,27 @@ +--- +- name: "🔍 Read the NFS server's installed packages" + ansible.builtin.package_facts: + delegate_to: "{{ groups['svc-storage-nfs-server'] | first }}" + delegate_facts: true + run_once: true + +- name: "🔧 Stop requesting directory delegations from a ganesha that rejects them" + when: >- + (hostvars[groups['svc-storage-nfs-server'] | first].ansible_facts.packages['nfs-ganesha'] | first).version + is version('9.11', '<') + block: + - name: "🔧 Keep directory delegations off whenever nfsv4 loads" + ansible.builtin.lineinfile: + path: /etc/modprobe.d/infinito-nfs-directory-delegations.conf + line: "options nfsv4 directory_delegations=N" + create: true + owner: root + group: root + mode: "0644" + + - name: "🔧 Turn directory delegations off in the running kernel" + ansible.builtin.script: + cmd: "{{ lookup('path_absolute', 'roles/svc-storage-nfs-client/files/shell/directory_delegations.sh') }}" + executable: /bin/bash + register: svc_storage_nfs_client_directory_delegations + changed_when: "'CHANGED' in svc_storage_nfs_client_directory_delegations.stdout" diff --git a/roles/svc-storage-nfs-client/tasks/main.yml b/roles/svc-storage-nfs-client/tasks/main.yml index 0fc67d391c..68241b326d 100644 --- a/roles/svc-storage-nfs-client/tasks/main.yml +++ b/roles/svc-storage-nfs-client/tasks/main.yml @@ -51,6 +51,10 @@ failed_when: - svc_storage_nfs_client_probe is failed +- name: "🔧 Align the NFS client's directory delegations with the ganesha server" + ansible.builtin.include_tasks: directory_delegations.yml + when: lookup('nfs_flavor') == 'ganesha' + - name: "🔍 Probe-write a marker to confirm the mount is RW" ansible.builtin.copy: dest: "{{ SVC_STORAGE_NFS_CLIENT_PROBE_DIR }}/.svc-storage-nfs-client-probe.{{ inventory_hostname }}" diff --git a/roles/svc-storage-nfs-server/tasks/03_controller_mount.yml b/roles/svc-storage-nfs-server/tasks/03_controller_mount.yml index fdaddf1a6b..62e83c01e4 100644 --- a/roles/svc-storage-nfs-server/tasks/03_controller_mount.yml +++ b/roles/svc-storage-nfs-server/tasks/03_controller_mount.yml @@ -59,3 +59,7 @@ retries: 12 delay: 5 until: _ctrl_nfs_mount is succeeded + +- name: "🔧 Align the controller's directory delegations with the ganesha server" + ansible.builtin.include_tasks: "{{ lookup('path_absolute', 'roles/svc-storage-nfs-client/tasks/directory_delegations.yml') }}" + when: lookup('nfs_flavor') == 'ganesha'