feat(diagnostics): sample what the resolver answers, not just its socket

Run 32118850138 lost the :53 listener, so the sampler counts listeners.
Run 32538854067 lost DNS with the listener still bound: sockets-udp.txt
showed dnsmasq holding 0.0.0.0:53 while every forwarded query timed out,
and the sampler reported that hour as healthy.

Each tick now also records what a glibc caller gets back — for the zone
this resolver owns, derived from its own address= line, and for one name
it must forward — with the latency that caller paid. The rescue collector
probes the in-zone name too: probing external names alone cannot separate
"forwarding is gone" from "dnsmasq is gone".

The sampler now also runs on the swarm lab DNS nodes; it only ever ran in
the compose path, which is why the Magento artifact carried no sampler log
at all. Its location comes from group_vars FILE_DNS53_SAMPLER_LOG and is
required at every call site, so a run cannot quietly collect nothing.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

.github/workflows/call-test-workspace.yml

@@ -90,8 +90,10 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
+          make dotenv
           {
             grep '^INFINITO_RESCUE_DIAGNOSTICS_BASE=' default.env
+            grep '^INFINITO_DNS53_SAMPLER_LOG=' .env
             echo "INFINITO_WORKSPACE_SLUG=$(printf '%s' "${DEV_RUNTIME_IMAGE}" | tr '/:' '--')-${{ matrix.track }}"
           } >> "$GITHUB_ENV"
 

compose/swarm/playbook.yml

@@ -23,6 +23,7 @@
     lab_export_base: "{{ lookup('env', 'NFS_EXPORT_BASE') }}"
     lab_node_src: "{{ lookup('env', 'INFINITO_NODE_SRC_DIR') }}"
     lab_runner_temp: "{{ lookup('env', 'RUNNER_TEMP') }}"
+    lab_sampler_log: "{{ lookup('env', 'INFINITO_DNS53_SAMPLER_LOG') }}"
     lab_systemd_timeout: "{{ lookup('env', 'SWARM_PILOT_SYSTEMD_TIMEOUT') | default('180', true) }}"
     lab_is_mgr: "{{ inventory_hostname == lab_mgr }}"
     lab_is_worker: "{{ inventory_hostname in [lab_wrk1, lab_wrk2] }}"
@@ -185,3 +186,14 @@
         src: "{{ lab_runner_temp }}/infinito-src.tar"
         dest: "{{ lab_node_src }}"
       timeout: 300
+
+    - name: sample what the resolver answers, not just whether it listens
+      shell: |
+        set -eu
+        sampler={{ lab_node_src }}/scripts/tests/deploy/ci/dns53-sampler.sh
+        INFINITO_DNS53_SAMPLER_LOG={{ lab_sampler_log }} \
+          setsid sh "${sampler}" </dev/null >/dev/null 2>&1 &
+      args:
+        creates: "{{ lab_sampler_log }}"
+      timeout: 30
+      when: lab_dns_node

group_vars/all/05_paths.yml

@@ -27,5 +27,6 @@ FILE_TOKENS: "{{ [DIR_SECRETS, 'tokens.yml'] | path_join }}"
 FILE_DATABASE_SECRETS: "{{ [DIR_SECRETS, 'databases.csv'] | path_join }}"
 
 DIR_RESCUE_DIAGNOSTICS: "/tmp/infinito-rescue-diagnostics"
+FILE_DNS53_SAMPLER_LOG: "/var/log/infinito/dns53-sampler.log"
 
 FILE_ENVIRONMENT: "/etc/environment" # File to safe environment variables in

scripts/tests/deploy/ci/dedicated.sh

@@ -21,7 +21,7 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
 # shellcheck source=/dev/null
-source <(grep -E '^INFINITO_(PLAYWRIGHT_REPORTS_BASE_DIR|RESCUE_DIAGNOSTICS_DIR|RESCUE_DIAGNOSTICS_BASE)=' "${REPO_ROOT}/.env")
+source <(grep -E '^INFINITO_(PLAYWRIGHT_REPORTS_BASE_DIR|RESCUE_DIAGNOSTICS_DIR|RESCUE_DIAGNOSTICS_BASE|DNS53_SAMPLER_LOG)=' "${REPO_ROOT}/.env")
 : "${INFINITO_RESCUE_DIAGNOSTICS_BASE:?INFINITO_RESCUE_DIAGNOSTICS_BASE must be set}"
 
 apps=""
@@ -77,6 +77,7 @@ cleanup() {
 	echo ">>> Capturing rescue diagnostics inside ${INFINITO_CONTAINER} (recursive DiD snapshot) before teardown removes it"
 	timeout 1500 docker exec \
 		-e "INFINITO_RESCUE_DIAGNOSTICS_DIR=${INFINITO_RESCUE_DIAGNOSTICS_DIR}" \
+		-e "INFINITO_DNS53_SAMPLER_LOG=${INFINITO_DNS53_SAMPLER_LOG}" \
 		"${INFINITO_CONTAINER}" \
 		python3 /opt/src/infinito/utils/diagnostics/container.py \
 		"${apps}" "compose post-deploy failure" 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
@@ -199,9 +200,9 @@ docker exec "${_up_container}" install -m 755 \
 	/opt/src/infinito/roles/sys-ca-selfsigned/files/shell/with-ca-trust.sh \
 	/usr/bin/ca-trust-wrapper 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
 
-echo ">>> Starting the :53 listener sampler inside ${_up_container}"
+echo ">>> Starting the :53 sampler inside ${_up_container}"
 docker exec -d \
-	-e "INFINITO_RESCUE_DIAGNOSTICS_DIR=${INFINITO_RESCUE_DIAGNOSTICS_DIR}" \
+	-e "INFINITO_DNS53_SAMPLER_LOG=${INFINITO_DNS53_SAMPLER_LOG:?INFINITO_DNS53_SAMPLER_LOG must be set (SPOT: group_vars/all/05_paths.yml FILE_DNS53_SAMPLER_LOG); regenerate .env}" \
 	"${_up_container}" \
 	sh /opt/src/infinito/scripts/tests/deploy/ci/dns53-sampler.sh
 

scripts/tests/deploy/ci/dns53-sampler.sh

@@ -4,16 +4,45 @@
 # sample bucket and names which :53 listener (hex local_address) was left.
 set -u
 
+EXTERNAL_PROBE_NAME=deb.debian.org
+
 sample_once() {
 	socks="$(awk '$2 ~ /:0035$/ && $3 == "00000000:0000" {printf " %s", $2}' "${1}")"
 	printf '%s%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "${socks:- none}"
 }
 
+zone_probe_name() {
+	zone="$(sed -n 's|^[[:space:]]*address=/\([^/]*\)/.*|\1|p' "${1}" 2>/dev/null | head -n1)"
+	[ -n "${zone}" ] || return 0
+	printf 'rescue-probe.%s' "${zone}"
+}
+
+answer_once() {
+	name="${1}"
+	start="$(date -u '+%s%3N')"
+	if timeout 3 getent ahosts "${name}" >/dev/null 2>&1; then
+		verdict=ok
+	else
+		verdict=fail
+	fi
+	printf ' %s=%s/%sms' "${name}" "${verdict}" "$(($(date -u '+%s%3N') - start))"
+}
+
 [ "${DNS53_SAMPLER_LIB:-}" = "1" ] && return 0
 
-: "${INFINITO_RESCUE_DIAGNOSTICS_DIR:?INFINITO_RESCUE_DIAGNOSTICS_DIR must be set}"
-mkdir -p "${INFINITO_RESCUE_DIAGNOSTICS_DIR}"
+: "${INFINITO_DNS53_SAMPLER_LOG:?INFINITO_DNS53_SAMPLER_LOG must be set}"
+mkdir -p "$(dirname "${INFINITO_DNS53_SAMPLER_LOG}")"
 while :; do
-	sample_once /proc/net/udp >>"${INFINITO_RESCUE_DIAGNOSTICS_DIR}/dns53-sampler.log"
+	{
+		printf '%s' "$(sample_once /proc/net/udp)"
+		inzone="$(zone_probe_name /etc/dnsmasq.conf)"
+		if [ -n "${inzone}" ]; then
+			answer_once "${inzone}"
+		else
+			printf ' inzone=unconfigured'
+		fi
+		answer_once "${EXTERNAL_PROBE_NAME}"
+		printf '\n'
+	} >>"${INFINITO_DNS53_SAMPLER_LOG}"
 	sleep 10
 done

tests/unit/python/scripts/tests/deploy/ci/test_dns53_sampler.py

@@ -57,5 +57,62 @@ class SamplerTickTests(unittest.TestCase):
         self.assertRegex(line, TIMESTAMP + r" none$")
 
 
+class AnswerProbeTests(unittest.TestCase):
+    def _answer(self, name: str) -> str:
+        proc = subprocess.run(
+            [
+                "sh",
+                "-c",
+                'DNS53_SAMPLER_LIB=1 . "$1" && answer_once "$2"',
+                "sh",
+                str(SAMPLER),
+                name,
+            ],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        return proc.stdout
+
+    def test_a_resolvable_name_is_reported_ok_with_its_latency(self):
+        self.assertRegex(self._answer("localhost"), r"^ localhost=ok/\d+ms$")
+
+    def test_an_unresolvable_name_is_reported_fail_not_skipped(self):
+        self.assertRegex(
+            self._answer("rescue-probe.invalid"),
+            r"^ rescue-probe\.invalid=fail/\d+ms$",
+        )
+
+
+class ZoneProbeNameTests(unittest.TestCase):
+    def _zone(self, conf: str) -> str:
+        with tempfile.NamedTemporaryFile("w", suffix=".conf") as handle:
+            handle.write(conf)
+            handle.flush()
+            proc = subprocess.run(
+                [
+                    "sh",
+                    "-c",
+                    'DNS53_SAMPLER_LIB=1 . "$1" && zone_probe_name "$2"',
+                    "sh",
+                    str(SAMPLER),
+                    handle.name,
+                ],
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+        return proc.stdout
+
+    def test_the_owned_zone_becomes_a_probe_name(self):
+        self.assertEqual(
+            self._zone("no-resolv\naddress=/infinito.example/192.168.244.10\n"),
+            "rescue-probe.infinito.example",
+        )
+
+    def test_a_config_without_a_zone_yields_nothing(self):
+        self.assertEqual(self._zone("no-resolv\nserver=192.0.2.1\n"), "")
+
+
 if __name__ == "__main__":
     unittest.main()

tests/unit/python/utils/diagnostics/test_container.py

@@ -14,9 +14,23 @@ import unittest
 import unittest.mock as mock
 from pathlib import Path
 
+from utils.paths import read_group_path
+
 from . import PROJECT_ROOT
 
 RESCUE = PROJECT_ROOT / "utils" / "diagnostics" / "container.py"
+_ENV = mock.patch.dict(
+    os.environ,
+    {"INFINITO_DNS53_SAMPLER_LOG": read_group_path("FILE_DNS53_SAMPLER_LOG")},
+)
+
+
+def setUpModule():
+    _ENV.start()
+
+
+def tearDownModule():
+    _ENV.stop()
 
 
 def _load():
@@ -100,6 +114,17 @@ class CollectTests(unittest.TestCase):
             self.assertIn("context: ctx", meta)
             self.assertIn("host: myhost", meta)
 
+    def test_collect_host_refuses_to_run_without_the_sampler_log_variable(self):
+        mod = _load()
+        with (
+            tempfile.TemporaryDirectory() as td,
+            mock.patch.dict(os.environ, {}, clear=False),
+            mock.patch.object(mod, "run", return_value=_cp([], stdout=b"h\n")),
+        ):
+            os.environ.pop("INFINITO_DNS53_SAMPLER_LOG", None)
+            with self.assertRaises(KeyError):
+                mod.collect_host(Path(td), "app", "ctx", "STAMP")
+
     def test_each_probed_name_gets_its_own_verdict(self):
         mod = _load()
         with tempfile.TemporaryDirectory() as td:

utils/diagnostics/container.py

@@ -11,11 +11,10 @@ capture there, pulling each nested snapshot back under
 runtime, however deep the nesting goes. ``RESCUE_SEEN`` carries the ids already
 entered, so a runtime that lists itself is cut as a cycle.
 
-Every collector is best-effort: a missing source must never abort the
-capture. ``INFINITO_RESCUE_DIAGNOSTICS_DIR`` (SPOT:
-``group_vars/all/05_paths.yml`` ``DIR_RESCUE_DIAGNOSTICS``) is the required
-output root; it is never defaulted here so there is one source. Prints one
-condensed summary and ALWAYS exits 1 so a failing pipeline stays failing.
+Every collector is best-effort: a missing source must never abort the capture.
+``INFINITO_RESCUE_DIAGNOSTICS_DIR`` and ``INFINITO_DNS53_SAMPLER_LOG`` are
+required and never defaulted here, so ``group_vars/all/05_paths.yml`` stays the
+one source. Prints one summary and ALWAYS exits 1 so a failure stays a failure.
 
 Usage:
     container.py [APP_ID] [CONTEXT]
@@ -42,6 +41,7 @@ _TAR_TIMEOUT = 300
 _SELF_IN_CONTAINER = "/tmp/rescue-self.py"  # noqa: S108 - fixed staging path inside the inspected container
 _LOCAL_DUMPS_ENV = "INFINITO_RESCUE_LOCAL_DUMPS_DIR"
 _PROBE_HOSTS = ("deb.debian.org", "ghcr.io", "repo.packagist.org")
+_INZONE_PROBE = "getent hosts rescue-probe.$(awk -F/ '/^address=/{print $2;exit}' /etc/dnsmasq.conf)"
 
 
 def runtime_bin() -> str | None:
@@ -118,6 +118,7 @@ def collect_host(out: Path, app_id: str, context: str, stamp: str) -> None:
         ("sockets-tcp.txt", ["ss", "-lntp"]),
         ("processes.txt", ["ps", "-eo", "pid,ppid,stat,etimes,rss,comm"]),
         ("dnsmasq-conf-d.txt", ["grep", "-rH", ".", "/etc/dnsmasq.d"]),
+        ("resolve-inzone.txt", ["sh", "-c", _INZONE_PROBE]),
         ("zfs-dev.txt", ["ls", "-l", "/dev/zfs"]),
         ("zfs-version.txt", ["zfs", "version"]),
     ):
@@ -130,6 +131,7 @@ def collect_host(out: Path, app_id: str, context: str, stamp: str) -> None:
         "/etc/nsswitch.conf",
         "/etc/hosts",
         "/etc/dnsmasq.conf",
+        os.environ["INFINITO_DNS53_SAMPLER_LOG"],
         "/etc/docker/daemon.json",
         "/proc/modules",
         "/proc/devices",
@@ -427,6 +429,8 @@ def recurse(
                 "RESCUE_SEEN=" + ",".join([*seen, cid] if cid else seen),
                 "-e",
                 f"{_LOCAL_DUMPS_ENV}={os.environ.get(_LOCAL_DUMPS_ENV, '')}",
+                "-e",
+                f"INFINITO_DNS53_SAMPLER_LOG={os.environ['INFINITO_DNS53_SAMPLER_LOG']}",
                 name,
                 "python3",
                 _SELF_IN_CONTAINER,

utils/env/handlers/__init__.py

@@ -41,6 +41,7 @@ from .infinito.package_cache import heap as package_cache_heap
 from .infinito.playwright import reports_base_dir as playwright_reports_base_dir
 from .infinito.playwright import stage_base_dir as playwright_stage_base_dir
 from .infinito.rescue import diagnostics_dir as rescue_diagnostics_dir
+from .infinito.rescue import dns53_sampler_log as rescue_dns53_sampler_log
 from .infinito.rescue import local_dumps_dir as rescue_local_dumps_dir
 from .infinito.running_on import act as running_on_act
 from .infinito.running_on import github as running_on_github
@@ -56,6 +57,7 @@ ORDERED_HANDLERS = [
     dir_backups,
     dir_secrets,
     rescue_diagnostics_dir,
+    rescue_dns53_sampler_log,
     rescue_local_dumps_dir,
     swarm_nfs_export_base,
     swarm_nfs_state_path,

utils/env/handlers/infinito/rescue/dns53_sampler_log.py

@@ -0,0 +1,18 @@
+"""INFINITO_DNS53_SAMPLER_LOG: :53 sampler log, read from the group_vars
+paths SPOT (group_vars/all/05_paths.yml FILE_DNS53_SAMPLER_LOG)."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from utils.paths import read_group_path
+
+if TYPE_CHECKING:
+    from utils.env.builder import BuildContext, EnvBuilder
+
+KEY = "INFINITO_DNS53_SAMPLER_LOG"
+COMMENT = "Where the :53 sampler appends its ticks and the rescue collector reads them (SPOT: group_vars/all/05_paths.yml FILE_DNS53_SAMPLER_LOG)."
+
+
+def apply(eb: EnvBuilder, ctx: BuildContext) -> None:
+    eb.setdefault(KEY, read_group_path("FILE_DNS53_SAMPLER_LOG"), comment=COMMENT)