fix(test): stop the package probes from throttling themselves

archlinux.org answers 429 and resets connections when the suite fires its
16 workers at it: 22 arch probes in parallel, 12 answered and 4 refused in
a reproduction inside the test container, while each name resolves fine on
its own. Those 22 self-inflicted failures then took the warning path,
because an unreachable index is supposed to say nothing about the
declaration — so the suite reported green on 22 packages it never checked.

_get now holds at most two requests per host and retries 429 and 5xx with
backoff; a DNS failure is not retried, it is not transient. That alone
removed all 22 warnings, 45 down to 23, with the runtime unchanged.

The remaining 23 are declared: a third-party repository the role
configures itself, or a name the index legitimately does not list because
the package is virtual. Outcome carries that as a flag now, so those stay
warnings while anything else that could not be checked fails the suite and
names each case. Verified by pointing the arch probe at an unreachable
host: 94 packages reported as unchecked, and the run went red.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

tests/external/repository/_package_probes.py

@@ -13,6 +13,9 @@ from __future__ import annotations
 
 import json
 import re
+import socket
+import threading
+import time
 import urllib.error
 import urllib.parse
 import urllib.request
@@ -23,6 +26,11 @@ from utils.packages.schema import SOURCE_AUR
 _TIMEOUT = 20
 _MAX_WORKERS = 16
 _USER_AGENT = "infinito-nexus package availability check"
+_PER_HOST_REQUESTS = 2
+_RETRY_ATTEMPTS = 4
+_RETRY_BACKOFF = 1.0
+_RETRY_STATUS = frozenset({429, 500, 502, 503, 504})
+_HOST_LOCKS: dict[str, threading.Semaphore] = {}
 
 DEBIAN_SUITE = "stable"
 UBUNTU_SUITE = "noble"
@@ -62,15 +70,35 @@ class Outcome(NamedTuple):
     probe: Probe
     available: bool | None
     detail: str
+    declared: bool = False
 
 
 def _get(url: str) -> tuple[int, bytes]:
+    """Fetch one index URL, serialising and retrying per host.
+
+    Args:
+        url: absolute https URL of a package index.
+
+    Returns:
+        The HTTP status and body; the body is empty for an error status.
+    """
     request = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT})  # noqa: S310 - literal https package index URLs only
-    try:
-        with urllib.request.urlopen(request, timeout=_TIMEOUT) as response:  # noqa: S310 - literal https package index URLs only
-            return response.getcode(), response.read()
-    except urllib.error.HTTPError as exc:
-        return exc.code, b""
+    host = urllib.parse.urlsplit(url).hostname or ""
+    with _HOST_LOCKS.setdefault(host, threading.Semaphore(_PER_HOST_REQUESTS)):
+        for attempt in range(_RETRY_ATTEMPTS):
+            try:
+                with urllib.request.urlopen(request, timeout=_TIMEOUT) as response:  # noqa: S310 - literal https package index URLs only
+                    return response.getcode(), response.read()
+            except urllib.error.HTTPError as exc:
+                if exc.code not in _RETRY_STATUS or attempt == _RETRY_ATTEMPTS - 1:
+                    return exc.code, b""
+            except urllib.error.URLError as exc:
+                if isinstance(exc.reason, socket.gaierror):
+                    raise
+                if attempt == _RETRY_ATTEMPTS - 1:
+                    raise
+            time.sleep(_RETRY_BACKOFF * 2**attempt)
+    raise AssertionError("unreachable")
 
 
 def _probe_arch(name: str) -> tuple[bool | None, str]:
@@ -170,7 +198,7 @@ def _externally_managed(repo: dict | None) -> str | None:
 def _probe(probe: Probe) -> Outcome:
     external = _externally_managed(probe.repo)
     if external:
-        return Outcome(probe, None, f"third-party repository: {external}")
+        return Outcome(probe, None, f"third-party repository: {external}", True)
     try:
         if probe.source == SOURCE_AUR:
             available, detail = _probe_aur(probe.name)
@@ -191,5 +219,7 @@ def _probe(probe: Probe) -> Outcome:
     except Exception as exc:
         return Outcome(probe, None, f"{type(exc).__name__}: {exc}")
     if available is False and probe.virtual:
-        return Outcome(probe, None, f"{detail} lists no such name; declared virtual")
+        return Outcome(
+            probe, None, f"{detail} lists no such name; declared virtual", True
+        )
     return Outcome(probe, available, detail)

tests/external/repository/test_packages_available.py

@@ -60,7 +60,7 @@ class TestPackagesAvailable(unittest.TestCase):
             outcomes.extend(pool.map(_probe, probes))
 
         unknown = [o for o in outcomes if o.available is None]
-        for outcome in unknown:
+        for outcome in (o for o in unknown if o.declared):
             warnings.warn(
                 f"{outcome.probe.package_id} / {outcome.probe.distro} / "
                 f"{outcome.probe.name}: {outcome.detail}",
@@ -68,6 +68,22 @@ class TestPackagesAvailable(unittest.TestCase):
                 stacklevel=1,
             )
 
+        unverified = [o for o in unknown if not o.declared]
+        if unverified:
+            self.fail(
+                f"{len(unverified)} package(s) could not be checked at all, so "
+                "reporting this run green would claim a verification that did "
+                "not happen. Fix the index access, or declare the exemption "
+                "(third-party repo / virtual) where the package is declared:\n"
+                + "\n".join(
+                    f"  - {o.probe.package_id} / {o.probe.distro} / "
+                    f"{o.probe.name}: {o.detail}"
+                    for o in sorted(
+                        unverified, key=lambda o: (o.probe.package_id, o.probe.distro)
+                    )
+                )
+            )
+
         missing = [o for o in outcomes if o.available is False]
         if not missing:
             return