fix(hlth-volumes): probe the NFS port the server actually serves

The reachability check called showmount, which speaks the MOUNT protocol. The ganesha export sets NFS_CORE_PARAM Protocols = 4 and Enable_NLM = false, and every EXPORT block repeats Protocols = 4, so MOUNT is never registered and showmount cannot succeed against a healthy server. The repo already records this elsewhere: scripts/tests/deploy/swarm/utils/collect/diagnostics.sh:125 prints "showmount -e failed; NFSv4-only servers do not answer MOUNT".

Every failure fed status, and a nonzero exit reaches OnFailure, which starts sys-ctl-cln-anon-volumes (dockreap --no-confirmation). So on any host holding both anonymous and NFS volumes the check has been failing on every run and reaping volumes for a reason that was never true. Probing tcp/2049 is what the role does for its own readiness gate in files/nfs-ganesha-restart.conf:9, and it is correct for the kernel flavor too.

The two tests drive a whitelisted anonymous volume so the anonymous section contributes nothing and the exit status reflects only the NFS check. One asserts a listening port passes, the other that a refused one fails; the first is the regression guard, the second keeps the check from being one that always fails.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

roles/sys-ctl-hlth-volumes/files/shell/script.sh

@@ -110,7 +110,7 @@ if [ -n "$nfs_volumes" ]; then
             continue
         fi
 
-        if showmount -e "$server" >/dev/null 2>&1; then
+        if timeout 5 bash -c "exec 3<>/dev/tcp/${server}/2049" 2>/dev/null; then
             echo "✅ Volume $vol: server $server reachable for export $export_path"
         else
             echo "❌ Volume $vol: server $server NOT reachable (export $export_path)"

tests/unit/python/roles/sys-ctl-hlth-volumes/files/test_hlth_volumes_script.py

@@ -1,4 +1,5 @@
 import os
+import socket
 import subprocess
 import tempfile
 import unittest
@@ -15,11 +16,22 @@ VOL_B = "b" * 64
 CONTAINER_STUB = """#!/usr/bin/env bash
 if [ "$1" = "volume" ] && [ "$2" = "ls" ]; then
 \tif [[ "$*" == *"driver=local"* ]]; then
+\t\tif [ -n "$STUB_NFS_VOLUMES" ]; then
+\t\t\tprintf '%s\\n' $STUB_NFS_VOLUMES
+\t\tfi
 \t\texit 0
 \tfi
 \tprintf '%s\\n' $STUB_VOLUMES
 \texit 0
 fi
+if [ "$1" = "volume" ] && [ "$2" = "inspect" ]; then
+\tcase "$*" in
+\t\t*Options.type*) printf 'nfs\\n' ;;
+\t\t*Options.device*) printf ':/export\\n' ;;
+\t\t*Options.o*) printf 'addr=127.0.0.1\\n' ;;
+\tesac
+\texit 0
+fi
 if [ "$1" = "ps" ]; then
 \targs="$*"
 \tvol="${args##*volume=}"
@@ -43,7 +55,9 @@ exit 0
 
 
 class TestHlthVolumesScript(unittest.TestCase):
-    def run_script(self, *, volumes, containers, mounts, running=None, whitelist=""):
+    def run_script(
+        self, *, volumes, containers, mounts, running=None, whitelist="", nfs=()
+    ):
         """Run the health script against a stubbed container CLI.
 
         Args:
@@ -53,6 +67,8 @@ class TestHlthVolumesScript(unittest.TestCase):
             running: volume name -> the subset of those ids still running; a
                 volume absent here has no running container left.
             whitelist: value for the script's only positional argument.
+            nfs: nfs-backed volume names; their server is 127.0.0.1, whose
+                port 2049 refuses the connection in the test environment.
 
         Returns:
             Tuple of the exit code and stdout.
@@ -66,6 +82,7 @@ class TestHlthVolumesScript(unittest.TestCase):
             env = dict(os.environ)
             env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
             env["STUB_VOLUMES"] = " ".join(volumes)
+            env["STUB_NFS_VOLUMES"] = " ".join(nfs)
             for volume, ids in containers.items():
                 env[f"STUB_CONTAINERS_{volume}"] = ids
                 env[f"STUB_RUNNING_{volume}"] = running.get(volume, "")
@@ -124,6 +141,30 @@ class TestHlthVolumesScript(unittest.TestCase):
         self.assertEqual(code, 1)
         self.assertIn("is not used by any container", out)
 
+    def test_an_unreachable_nfs_server_fails_the_probe(self):
+        code, _ = self.run_script(
+            volumes=[VOL_B],
+            containers={},
+            mounts={},
+            whitelist=VOL_B,
+            nfs=[VOL_A],
+        )
+        self.assertEqual(code, 1)
+
+    def test_a_listening_nfs_port_passes(self):
+        with socket.socket() as server:
+            server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            server.bind(("127.0.0.1", 2049))
+            server.listen(1)
+            code, _ = self.run_script(
+                volumes=[VOL_B],
+                containers={},
+                mounts={},
+                whitelist=VOL_B,
+                nfs=[VOL_A],
+            )
+        self.assertEqual(code, 0)
+
 
 if __name__ == "__main__":
     unittest.main()