fix(images): make a dnf retry fetch again instead of replaying its cache

Both image-build jobs of run 37494541850 failed, and only centos broke; arch,
debian, ubuntu and fedora built. The headline reads like an outage:

    Error: Failed to download metadata for repo 'baseos':
      Cannot download repomd.xml ... All mirrors were tried

The line above it says otherwise:

    Errors during downloading metadata for repository 'baseos':
      - Downloading successful, but checksum doesn't match.
        Calculated: 76995e25878232aba76bc22834280007be1baf40...

The download succeeds; the repomd.xml served does not match the checksum the
metalink advertises. The decisive detail is that this digest is byte-identical
on attempts 1, 2 and 3. A real re-fetch reaches a different mirror and yields a
different digest, so attempts 2 and 3 fetched nothing: dnf had written the bad
metadata to /var/cache/dnf and retry only slept and re-invoked the same command
against the same file. Three attempts, one download, and the same for all seven
build attempts.

retry_dnf is retry plus a metadata clean before each re-run, so the next
attempt is a fetch and can land on a mirror that is not mid-sync. All ten dnf
call sites use it, not just the one that failed: the zfs builddep and install
paths carry the identical weakness. apt and pacman keep retry, having no
equivalent cache trap.

The success path is unchanged, so the common case behaves identically. On the
failure path the script runs under set -e, which aborts if the clean itself
returns non-zero: measured, and it fails closed and loud rather than continuing
against a stale cache.

This fixes the retry, not the mirror. If the EL mirrors served stale metadata
for the whole window, three real fetches still fail.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

scripts/install/filesystem.sh

@@ -69,6 +69,21 @@ retry() {
 	done
 }
 
+# Param: $@ dnf command, retried with the metadata cache dropped between tries
+retry_dnf() {
+	local attempt=1
+	until "$@"; do
+		if [ "${attempt}" -ge "${RETRIES}" ]; then
+			log "ERROR: '$*' failed after ${attempt} attempts"
+			return 1
+		fi
+		log "attempt ${attempt} of '$1' failed; dropping dnf metadata and retrying"
+		dnf clean metadata
+		sleep $((attempt * 5))
+		attempt=$((attempt + 1))
+	done
+}
+
 install_apt() {
 	export DEBIAN_FRONTEND=noninteractive
 	local apt_opts=(-o Acquire::Retries="${RETRIES}" -qq)
@@ -120,13 +135,13 @@ install_pacman() {
 install_dnf_native() {
 	case "${ID:-}" in
 	fedora)
-		retry dnf install -y e2fsprogs btrfs-progs
+		retry_dnf dnf install -y e2fsprogs btrfs-progs
 		return
 		;;
 	esac
 	log "adding EPEL, the only place the EL family carries btrfs-progs"
-	retry dnf install -y epel-release
-	retry dnf install -y e2fsprogs btrfs-progs
+	retry_dnf dnf install -y epel-release
+	retry_dnf dnf install -y e2fsprogs btrfs-progs
 }
 
 # Param: $1 path segment of the zfsonlinux repository, `fedora` or `epel`
@@ -154,11 +169,11 @@ download_zfs_source() {
 	rm -rf "${dir}"
 	mkdir -p "${dir}"
 	if command -v dnf5 >/dev/null 2>&1; then
-		retry dnf install -y --disablerepo=zfs dnf5-plugins >&2
+		retry_dnf dnf install -y --disablerepo=zfs dnf5-plugins >&2
 	else
-		retry dnf install -y --disablerepo=zfs dnf-plugins-core >&2
+		retry_dnf dnf install -y --disablerepo=zfs dnf-plugins-core >&2
 	fi
-	retry dnf download --source --disablerepo=zfs --enablerepo=zfs-source --destdir="${dir}" zfs >&2
+	retry_dnf dnf download --source --disablerepo=zfs --enablerepo=zfs-source --destdir="${dir}" zfs >&2
 	srpm="$(find "${dir}" -maxdepth 1 -name 'zfs-*.src.rpm' | head -n1)"
 	if [ -z "${srpm}" ]; then
 		log "ERROR: the zfs-source repository offers no zfs source RPM" >&2
@@ -176,7 +191,7 @@ install_zfs_from_source() {
 	if [ "${ID:-}" != fedora ]; then
 		crb=(--enablerepo=crb)
 	fi
-	retry dnf builddep -y "${crb[@]}" "${srpm}"
+	retry_dnf dnf builddep -y "${crb[@]}" "${srpm}"
 	mapfile -t added < <(rpm -qa --qf '%{NAME}\n' | sort | comm -13 "${before}" -)
 
 	rm -rf "${topdir}"
@@ -186,7 +201,7 @@ install_zfs_from_source() {
 		! -name '*-debug*' ! -name '*-devel-*' ! -name 'zfs-test*' \
 		! -name 'zfs-dracut*' ! -name 'python3-pyzfs*' ! -name 'pam_zfs_key*' \
 		! -name 'zfs-dkms*' ! -name 'zfs-kmod*')
-	retry dnf install -y "${runtime[@]}"
+	retry_dnf dnf install -y "${runtime[@]}"
 
 	log "dropping ${#added[@]} build dependencies of the zfs rebuild"
 	if command -v dnf5 >/dev/null 2>&1; then
@@ -218,7 +233,7 @@ install_dnf() {
 	log "zfs userland version: ${zfs_version}"
 
 	if ! command -v rpmbuild >/dev/null 2>&1; then
-		retry dnf install -y rpm-build
+		retry_dnf dnf install -y rpm-build
 	fi
 
 	local topdir=/tmp/zfs-kmod-container
@@ -246,7 +261,7 @@ SPEC
 	if [ -n "${source_rpm}" ]; then
 		install_zfs_from_source "${source_rpm}" "${before}"
 	else
-		retry dnf install -y zfs
+		retry_dnf dnf install -y zfs
 		dnf remove -y zfs-release
 	fi