computer-playbook/scripts/tests/deploy/distros.sh

#!/usr/bin/env bash
set -euo pipefail

# SPOT: run one command once per distro, in random order, under a shared time
# budget. A distro the budget drops BEFORE it starts is reported as skipped; a
# distro the budget kills MID-RUN fails the job. Every outcome lands in a
# job-summary table in execution order.
#
# Param:
#   $@                                  command + args to run once per distro
#   INFINITO_DISTROS                    space-separated distro list
#   INFINITO_CI_DISTRO_BUDGET_SECONDS   wall-clock budget for the whole run
#   GITHUB_STEP_SUMMARY                 optional; the markdown table is appended there
#
# Exports:
#   INFINITO_CI_DISTRO_DEADLINE_EPOCH   epoch the running distro gets the budget SIGTERM at
#   INFINITO_CI_DISTRO_KILL_EPOCH       epoch the SIGKILL follows at, KILL_GRACE_SECONDS later
#
# Exports per iteration:
#   INFINITO_DISTRO                     the distro under test

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
cd "${REPO_ROOT}"

# shellcheck source=scripts/meta/env/load.sh
source "scripts/meta/env/load.sh"

: "${INFINITO_DISTROS:?INFINITO_DISTROS is required (e.g. 'arch debian ubuntu fedora centos')}"
: "${INFINITO_CI_DISTRO_BUDGET_SECONDS:?INFINITO_CI_DISTRO_BUDGET_SECONDS is required (declared in default.env)}"

BUDGET_HEADROOM_PERCENT=15
KILL_GRACE_SECONDS=60

if (($# == 0)); then
	echo "[ERROR] a per-distro command is required" >&2
	exit 2
fi

if ! [[ "${INFINITO_CI_DISTRO_BUDGET_SECONDS}" =~ ^[0-9]+$ ]]; then
	echo "[ERROR] INFINITO_CI_DISTRO_BUDGET_SECONDS must be an integer (seconds), got: '${INFINITO_CI_DISTRO_BUDGET_SECONDS}'" >&2
	exit 2
fi

read -r -a distro_arr <<<"${INFINITO_DISTROS}"
mapfile -t distro_arr < <(printf '%s\n' "${distro_arr[@]}" | shuf)
echo "=== Distro execution order: ${distro_arr[*]} ==="

global_start="$(date +%s)"
deadline="$((global_start + INFINITO_CI_DISTRO_BUDGET_SECONDS))"
export INFINITO_CI_DISTRO_DEADLINE_EPOCH="${deadline}"
export INFINITO_CI_DISTRO_KILL_EPOCH="$((deadline + KILL_GRACE_SECONDS))"
echo "=== Global time budget: ${INFINITO_CI_DISTRO_BUDGET_SECONDS}s (deadline epoch=${deadline}) ==="

max_seen=0
skipped=0
ran=0
passed=0
durations=()
statuses=()
seconds=()
notes=()

for _ in "${distro_arr[@]}"; do
	statuses+=("skipped")
	seconds+=("")
	notes+=("not reached within the budget")
done

# Renders the per-distro outcome to stdout and, when GITHUB_STEP_SUMMARY is set,
# appends the same run as a markdown table in execution order.
render_summary() {
	local i distro icon

	echo
	echo "=== Summary ==="
	echo "ran=${ran} skipped=${skipped}"
	echo "total_runtime=$(($(date +%s) - global_start))s max_seen_duration=${max_seen}s"
	echo "budget=${INFINITO_CI_DISTRO_BUDGET_SECONDS}s remaining=$((deadline - $(date +%s)))s"
	echo "per-distro:"
	for line in "${durations[@]}"; do
		echo "  - ${line}"
	done

	[[ -n "${GITHUB_STEP_SUMMARY:-}" ]] || return 0

	{
		echo "## 🐧 Distro coverage"
		echo
		echo "| # | Distro | Status | Duration | Note |"
		echo "|--:|---|:--:|--:|---|"
		for i in "${!distro_arr[@]}"; do
			distro="${distro_arr[$i]}"
			case "${statuses[$i]}" in
			passed) icon="✅ passed" ;;
			failed) icon="❌ failed" ;;
			*) icon="🟦 skipped" ;;
			esac
			# shellcheck disable=SC2016  # backticks are markdown, not expansion
			printf '| %s | `%s` | %s | %s | %s |\n' \
				"$((i + 1))" "${distro}" "${icon}" \
				"${seconds[$i]:+${seconds[$i]}s}" "${notes[$i]}"
		done
		echo
		echo "${ran}/${#distro_arr[@]} ran, ${skipped} skipped, budget ${INFINITO_CI_DISTRO_BUDGET_SECONDS}s, order randomised per run."
		echo
	} >>"${GITHUB_STEP_SUMMARY}"
}

for i in "${!distro_arr[@]}"; do
	distro="${distro_arr[$i]}"
	remaining="$((deadline - $(date +%s)))"

	if ((remaining <= 0)); then
		echo "[WARN] Global budget exhausted (remaining=${remaining}s). Stopping further distro runs."
		skipped=$((skipped + ${#distro_arr[@]} - i))
		notes[i]="budget exhausted"
		break
	fi

	needed="$((max_seen * (100 + BUDGET_HEADROOM_PERCENT) / 100))"
	if ((max_seen > 0 && remaining < needed)); then
		echo "[WARN] Skipping distro=${distro}: remaining=${remaining}s < ${needed}s (max_seen=${max_seen}s + ${BUDGET_HEADROOM_PERCENT}% headroom)"
		skipped=$((skipped + 1))
		notes[i]="remaining ${remaining}s < ${needed}s needed"
		continue
	fi

	echo "=== Running distro=${distro}: ${1} ==="
	echo ">>> Time budget: remaining=${remaining}s max_seen=${max_seen}s"

	export INFINITO_DISTRO="${distro}"
	source "scripts/meta/env/load.sh"

	distro_start="$(date +%s)"

	set +e
	timeout -k "${KILL_GRACE_SECONDS}" "${remaining}s" "$@"
	rc=$?
	set -e

	dur="$(($(date +%s) - distro_start))"
	durations+=("${distro}=${dur}s")
	ran=$((ran + 1))
	seconds[i]="${dur}"
	notes[i]=""

	if ((dur > max_seen)); then
		max_seen="$dur"
	fi

	echo ">>> Duration: distro=${distro} took ${dur}s (max_seen=${max_seen}s)"

	if [[ $rc -ne 0 ]]; then
		statuses[i]="failed"
		if [[ $rc -eq 124 || $rc -eq 137 ]] && ((dur >= remaining)); then
			notes[i]="rc=${rc} after ${dur}s, with ${remaining}s of the ${INFINITO_CI_DISTRO_BUDGET_SECONDS}s budget left at start"
		else
			notes[i]="rc=${rc}"
		fi
		skipped=$((skipped + ${#distro_arr[@]} - i - 1))
		for ((j = i + 1; j < ${#distro_arr[@]}; j++)); do
			notes[j]="not run: aborted after distro=${distro} failed"
		done
		echo "[ERROR] Run failed for distro=${distro}: ${notes[i]}" >&2
		render_summary
		exit "$rc"
	fi

	statuses[i]="passed"
	passed=$((passed + 1))
done

render_summary