fix(ci): retry-harden every egress path (curl filter, shell downloads, package managers)

Transient CDN/mirror TLS resets are a recurring hard-fail class (docker.com, PyPI). Harden all three egress paths at their SPOT: the curl filter now emits --retry-all-errors (retries=0 opts a call out), so every ansible curl retries; a new lint requires --retry-all-errors on file-downloading curl and --tries on wget in verbatim files/*.sh (which cannot use the filter), fixing the moodle/alias downloads it flagged; and the update role configures native retries for every package manager - apt Acquire::Retries, dnf retries=, pacman XferCommand over retrying curl - covering all five CI distros (ubuntu shares apt, centos shares dnf).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

plugins/filter/curl.py

@@ -1,13 +1,16 @@
-"""Render the timeout-hardened curl base command (SPOT).
+"""Render the timeout-hardened, retry-hardened curl base command (SPOT).
 
-    {{ 30 | curl }}                      -> curl -s --connect-timeout 5 --max-time 30
-    {{ 300 | curl }}                     -> curl -s --connect-timeout 5 --max-time 300
+    {{ 30 | curl }}                      -> curl -s --connect-timeout 5 --max-time 30 --retry 3 --retry-all-errors --retry-delay 2
+    {{ 300 | curl }}                     -> curl -s --connect-timeout 5 --max-time 300 --retry 3 --retry-all-errors --retry-delay 2
     {{ lookup('timeout', 30) | curl }}   -> TIMEOUT_FACTOR-scaled variant
 
 Args:
     max_time: Seconds for the whole transfer; without it curl hangs forever
         on an accepted-but-silent peer and task retries never fire.
     connect_timeout: Seconds curl waits for the TCP/TLS connect (default 5).
+    retries: --retry count; --retry-all-errors makes it cover transport-layer
+        resets (exit 35/52/56), the class that hard-fails deploys on flaky CI
+        egress. Pass 0 to opt a call out of retrying.
 
 Callers append their own flags (``-S``, ``-f``, ``-o``, ...) after the filter.
 """
@@ -19,21 +22,27 @@ from typing import Any
 from ansible.errors import AnsibleFilterError
 
 
-def curl(max_time: Any, connect_timeout: Any = 5) -> str:
+def curl(max_time: Any, connect_timeout: Any = 5, retries: Any = 3) -> str:
     try:
         max_time_val = int(max_time)
         connect_timeout_val = int(connect_timeout)
+        retries_val = int(retries)
     except (TypeError, ValueError) as exc:
         raise AnsibleFilterError(
             f"curl filter: max_time {max_time!r} / connect_timeout "
-            f"{connect_timeout!r} must be integers (seconds)"
+            f"{connect_timeout!r} / retries {retries!r} must be integers"
         ) from exc
     if max_time_val <= 0 or connect_timeout_val <= 0:
         raise AnsibleFilterError(
             f"curl filter: max_time {max_time_val} and connect_timeout "
             f"{connect_timeout_val} must be > 0"
         )
-    return f"curl -s --connect-timeout {connect_timeout_val} --max-time {max_time_val}"
+    if retries_val < 0:
+        raise AnsibleFilterError(f"curl filter: retries {retries_val} must be >= 0")
+    base = f"curl -s --connect-timeout {connect_timeout_val} --max-time {max_time_val}"
+    if retries_val:
+        base += f" --retry {retries_val} --retry-all-errors --retry-delay 2"
+    return base
 
 
 class FilterModule:

roles/update/tasks/arch.yml

@@ -1,4 +1,12 @@
 ---
+- name: "🔁 update (pacman) | make pacman retry transient mirror/egress failures natively"
+  become: true
+  ansible.builtin.lineinfile:
+    path: /etc/pacman.conf
+    regexp: '^XferCommand\s*='
+    line: "XferCommand = /usr/bin/curl -L -C - -f --connect-timeout 5 --retry 3 --retry-all-errors --retry-delay 2 -o %o %u"
+    insertafter: '^\[options\]'
+
 - name: "🔄 update (pacman) | refresh pacman cache and upgrade all installed packages"
   community.general.pacman:
     update_cache: true

roles/update/tasks/debian.yml

@@ -1,4 +1,16 @@
 ---
+- name: "🔁 update (apt) | make apt retry transient mirror/egress failures natively"
+  become: true
+  ansible.builtin.copy:
+    dest: /etc/apt/apt.conf.d/80-infinito-retries
+    content: |
+      Acquire::Retries "5";
+      Acquire::http::Timeout "30";
+      Acquire::https::Timeout "30";
+    owner: root
+    group: root
+    mode: "0644"
+
 - name: "🔄 update (apt) | refresh apt cache and dist-upgrade all installed packages"
   ansible.builtin.apt:
     update_cache: true

roles/update/tasks/fedora.yml

@@ -1,4 +1,12 @@
 ---
+- name: "🔁 update (dnf) | make dnf retry transient mirror/egress failures natively"
+  become: true
+  ansible.builtin.lineinfile:
+    path: /etc/dnf/dnf.conf
+    regexp: '^retries='
+    line: "retries=5"
+    insertafter: '^\[main\]'
+
 - name: "🔄 update (dnf) | refresh dnf cache and upgrade all installed packages"
   ansible.builtin.dnf:
     name: "*"

roles/web-app-moodle/files/build/install/auth-oidc.sh

@@ -28,7 +28,7 @@ VERSION="$(curl --connect-timeout 5 --max-time 60 -sfSL https://api.github.com/r
 echo "Installing auth_oidc ${VERSION} (Moodle ${MOODLE_VER})"
 test -d "${MOODLE_SOURCE_DIR}/${MOODLE_AUTH_SUBDIR}"
 
-curl --connect-timeout 5 --max-time 300 -fSL -o "${ZIP_PATH}" "https://github.com/microsoft/moodle-auth_oidc/archive/refs/tags/${VERSION}.zip"
+curl --connect-timeout 5 --max-time 300 --retry 3 --retry-all-errors --retry-delay 2 -fSL -o "${ZIP_PATH}" "https://github.com/microsoft/moodle-auth_oidc/archive/refs/tags/${VERSION}.zip"
 unzip -q "${ZIP_PATH}" -d "${EXTRACT_DIR}"
 rm -rf "${PLUGIN_DIR}"
 

roles/web-app-moodle/files/build/install/moodle-source.sh

@@ -18,7 +18,7 @@ EXTRACT_DIR="$(mktemp -d -t moodle-extract.XXXXXX)"
 TARBALL_PATH="$(mktemp -t moodle-tarball.XXXXXX)"
 trap 'rm -rf "${EXTRACT_DIR}" "${TARBALL_PATH}"' EXIT
 
-curl --connect-timeout 5 --max-time 600 -fSL -o "${TARBALL_PATH}" "${MOODLE_TARBALL_URL}"
+curl --connect-timeout 5 --max-time 600 --retry 3 --retry-all-errors --retry-delay 2 -fSL -o "${TARBALL_PATH}" "${MOODLE_TARBALL_URL}"
 file "${TARBALL_PATH}"
 
 rm -rf "${MOODLE_SOURCE_DIR}"

scripts/make/alias.sh

@@ -111,7 +111,7 @@ render() {
 		"${INFINITO_ALIAS_REPOSITORY}"
 	if [[ ! -f "${TERMINAL_ALIASES_CACHE}" ]]; then
 		mkdir -p "$(dirname "${TERMINAL_ALIASES_CACHE}")"
-		curl -fsSL --max-time 15 "${TERMINAL_ALIASES_URL}" -o "${TERMINAL_ALIASES_CACHE}" 2>/dev/null || true
+		curl -fsSL --max-time 15 --retry 3 --retry-all-errors --retry-delay 2 "${TERMINAL_ALIASES_URL}" -o "${TERMINAL_ALIASES_CACHE}" 2>/dev/null || true
 	fi
 	if [[ -s "${TERMINAL_ALIASES_CACHE}" ]]; then
 		print_alias_file "${TERMINAL_ALIASES_CACHE}"

tests/lint/repository/test_shell_download_retry.py

@@ -0,0 +1,71 @@
+"""Lint: curl/wget that DOWNLOAD to a real file in a verbatim shell script
+MUST carry native retries, because ``files/*.sh`` are copied unrendered and
+cannot reach the retry-hardened ``curl`` filter (plugins/filter/curl.py). A
+transport-layer reset (curl exit 35) on flaky CI egress then hard-fails the
+deploy with no retry.
+
+Scope is deliberately narrow to avoid flagging the many status probes and
+local-API calls that legitimately fast-fail:
+
+* curl offender: ``-o <file>`` / ``-O`` / ``--remote-name`` writing to a real
+  path (not ``/dev/null`` or ``-``) without ``--retry-all-errors``.
+* wget offender: ``-O <file>`` writing to a real path (not ``-``), or a plain
+  ``wget <url>`` download, without ``--tries=``.
+
+Exempt: ``-o /dev/null`` / ``-O-`` stdout probes, ``command -v`` checks, and
+lines suppressed via ``# nocheck: shell-download-retry``.
+"""
+
+from __future__ import annotations
+
+import re
+import unittest
+
+from utils.annotations.suppress import is_suppressed_at
+from utils.cache.files import read_text
+
+from . import PROJECT_ROOT
+
+_RULE = "shell-download-retry"
+
+_CURL_RE = re.compile(r"(?<![\w-])curl\s+(?:-|http)")
+_CURL_OUT_RE = re.compile(r"(?:-o|--output)\s+(?!/dev/null\b|-\s)(\S+)|(?:-O\b|--remote-name\b)")
+_WGET_RE = re.compile(r"(?<![\w-])wget\s+(?:-|http)")
+_WGET_STDOUT_RE = re.compile(r"-O\s*-|-\w*O-")
+
+_GLOBS = ("roles/*/files/**/*.sh", "scripts/**/*.sh")
+
+
+class TestShellDownloadRetry(unittest.TestCase):
+    def test_shell_downloads_carry_native_retries(self) -> None:
+        offenders: list[str] = []
+        for glob in _GLOBS:
+            for path in sorted(PROJECT_ROOT.glob(glob)):
+                rel = path.relative_to(PROJECT_ROOT).as_posix()
+                lines = read_text(str(path)).splitlines()
+                for no, line in enumerate(lines, start=1):
+                    bad = None
+                    if _CURL_RE.search(line) and _CURL_OUT_RE.search(line):
+                        if "--retry-all-errors" not in line:
+                            bad = "curl download without --retry-all-errors"
+                    elif _WGET_RE.search(line) and not _WGET_STDOUT_RE.search(line):
+                        if "--tries=" not in line and "-t " not in line:
+                            bad = "wget download without --tries="
+                    if not bad:
+                        continue
+                    if is_suppressed_at(lines, no, _RULE):
+                        continue
+                    offenders.append(f"{rel}:{no}: {line.strip()[:100]}")
+
+        if offenders:
+            self.fail(
+                f"{len(offenders)} shell download(s) without native retries. Add "
+                "`--retry-all-errors --retry-delay 2` to curl or `--tries=3 "
+                "--waitretry=2` to wget (files/*.sh cannot use the curl filter), "
+                f"or mark a legitimate exception with `# nocheck: {_RULE}`:\n"
+                + "\n".join(f"  - {o}" for o in offenders)
+            )
+
+
+if __name__ == "__main__":  # pragma: no cover
+    unittest.main()

tests/unit/plugins/filter/test_curl.py

@@ -5,18 +5,36 @@ from ansible.errors import AnsibleFilterError
 from plugins.filter.curl import curl
 
 
+_RETRY = " --retry 3 --retry-all-errors --retry-delay 2"
+
+
 class TestCurlFilter(unittest.TestCase):
     def test_default_connect_timeout(self):
-        self.assertEqual(curl(30), "curl -s --connect-timeout 5 --max-time 30")
+        self.assertEqual(
+            curl(30), "curl -s --connect-timeout 5 --max-time 30" + _RETRY
+        )
 
     def test_custom_values(self):
         self.assertEqual(
             curl(300, connect_timeout=10),
-            "curl -s --connect-timeout 10 --max-time 300",
+            "curl -s --connect-timeout 10 --max-time 300" + _RETRY,
         )
 
     def test_numeric_string_input(self):
-        self.assertEqual(curl("45"), "curl -s --connect-timeout 5 --max-time 45")
+        self.assertEqual(
+            curl("45"), "curl -s --connect-timeout 5 --max-time 45" + _RETRY
+        )
+
+    def test_retries_zero_opts_out(self):
+        self.assertEqual(
+            curl(30, retries=0), "curl -s --connect-timeout 5 --max-time 30"
+        )
+
+    def test_custom_retries(self):
+        self.assertEqual(
+            curl(60, retries=5),
+            "curl -s --connect-timeout 5 --max-time 60 --retry 5 --retry-all-errors --retry-delay 2",
+        )
 
     def test_non_numeric_raises(self):
         with self.assertRaises(AnsibleFilterError):
@@ -28,6 +46,10 @@ class TestCurlFilter(unittest.TestCase):
         with self.assertRaises(AnsibleFilterError):
             curl(30, connect_timeout=-1)
 
+    def test_negative_retries_raises(self):
+        with self.assertRaises(AnsibleFilterError):
+            curl(30, retries=-1)
+
 
 if __name__ == "__main__":  # pragma: no cover
     unittest.main()