diff --git a/default.env b/default.env index b4c628a0eb..23257e0f80 100644 --- a/default.env +++ b/default.env @@ -334,6 +334,9 @@ INFINITO_SWARM_NODE_PIDS_LIMIT=8192 # Ceiling in minutes for one distro's swarm matrix deploy in scripts/tests/deploy/swarm/routine/00_one.sh; swarm's step level. INFINITO_SWARM_STEP_TIMEOUT_MINUTES=350 +# Wall clock 00_one.sh leaves collect_and_teardown under the sweep budget; measured worst case 12m44s on the swarm moodle#1 rescue path. +INFINITO_SWARM_TEARDOWN_RESERVE_SECONDS=900 + # Docker label stamped on every act-swarm-test container (DinD + NFS sidecar) so cleanup selects by label, not a guessed name. INFINITO_SWARM_TEST_LABEL=infinito.swarm-test diff --git a/scripts/tests/deploy/distros.sh b/scripts/tests/deploy/distros.sh index 112eb046b9..776ad47335 100755 --- a/scripts/tests/deploy/distros.sh +++ b/scripts/tests/deploy/distros.sh @@ -43,6 +43,7 @@ echo "=== Distro execution order: ${distro_arr[*]} ===" global_start="$(date +%s)" deadline="$((global_start + INFINITO_CI_DISTRO_BUDGET_SECONDS))" +export INFINITO_CI_DISTRO_DEADLINE_EPOCH="${deadline}" echo "=== Global time budget: ${INFINITO_CI_DISTRO_BUDGET_SECONDS}s (deadline epoch=${deadline}) ===" max_seen=0 diff --git a/scripts/tests/deploy/swarm/routine/00_one.sh b/scripts/tests/deploy/swarm/routine/00_one.sh index 18247026d0..505fb24a71 100755 --- a/scripts/tests/deploy/swarm/routine/00_one.sh +++ b/scripts/tests/deploy/swarm/routine/00_one.sh @@ -42,6 +42,7 @@ export SWARM_DRILL_ENV : "${INFINITO_IMAGE_TAG:?INFINITO_IMAGE_TAG is required (source scripts/meta/env/load.sh before invoking this script)}" : "${INFINITO_RESCUE_DIAGNOSTICS_BASE:?INFINITO_RESCUE_DIAGNOSTICS_BASE is required}" : "${INFINITO_SWARM_STEP_TIMEOUT_MINUTES:?INFINITO_SWARM_STEP_TIMEOUT_MINUTES is required}" +: "${INFINITO_SWARM_TEARDOWN_RESERVE_SECONDS:?INFINITO_SWARM_TEARDOWN_RESERVE_SECONDS is required}" SWARM_REQUIRED_SERVICES="node nfs-server container_backup nfs_backup" if [ -n "${disable:-}" ]; then @@ -110,7 +111,19 @@ make setup # shellcheck source=scripts/meta/env/load.sh source scripts/meta/env/load.sh -matrix_cmd=(python3 -m utils.tests.swarm.matrix) +step_timeout="$((INFINITO_SWARM_STEP_TIMEOUT_MINUTES * 60))" +if [ -n "${INFINITO_CI_DISTRO_DEADLINE_EPOCH:-}" ]; then + under_governor="$((INFINITO_CI_DISTRO_DEADLINE_EPOCH - $(date +%s) - INFINITO_SWARM_TEARDOWN_RESERVE_SECONDS))" + if [ "${under_governor}" -le 0 ]; then + echo "[ERROR] no budget left under the sweep governor for the matrix deploy" >&2 + exit 2 + fi + if [ "${under_governor}" -lt "${step_timeout}" ]; then + step_timeout="${under_governor}" + fi +fi + +matrix_cmd=(timeout "${step_timeout}" python3 -m utils.tests.swarm.matrix) if [ "$(id -u)" -ne 0 ]; then matrix_cmd=( sudo -E env @@ -127,7 +140,7 @@ if [ "$(id -u)" -ne 0 ]; then "${matrix_cmd[@]}" ) fi -timeout "$((INFINITO_SWARM_STEP_TIMEOUT_MINUTES * 60))" "${matrix_cmd[@]}" +"${matrix_cmd[@]}" bash "${SCRIPT_DIR}/05_seed_content.sh" bash "${SCRIPT_DIR}/06_drain_worker.sh"