mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 10:59:59 +00:00
kevinveenbirkenbachandClaude Opus 5
dc54e6d83f
Every one of these call sites ended in `|| true`, so a failure left no trace and the surrounding script reported a clean run. service_converged.sh needed the `|| true` for a reason that had nothing to do with the command: under `set -o pipefail`, `head -10` closes the pipe and awk dies of SIGPIPE, which pipefail then reports as the pipeline's status. That made a genuine `container service ps` failure indistinguishable from a normal truncation. Capping inside awk (`n++ < 10`) removes the SIGPIPE, so the command status can be tested on its own and a real failure now prints why. give_up and report_failed_tasks carried a byte-identical copy of that block, so the fix lifts it into one report_tasks helper rather than applying it twice. The swarm-clean scripts keep going after a failed step on purpose, since the host layer below them clears what survives. They now say what did not work before continuing, instead of continuing silently. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
107 lines
5.4 KiB
Bash
Executable File
107 lines
5.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Reclaim ALL leftover act-swarm state (DinD nodes, NFS sidecars, lab networks,
|
|
# act outer containers) from aborted or wedged roundtrip swarm runs, across every
|
|
# cluster id. Unlike teardown.sh (one cluster via SWARM_NAME), this nukes every
|
|
# act-swarm container/network so the next swarm step starts from a clean host.
|
|
# Run BETWEEN swarm runs: it would kill an in-flight one.
|
|
#
|
|
# Scope is precise, never by guessed name prefix, so unrelated containers (a
|
|
# production `nfs-server`, someone's own `swarm-mgr`, ...) are NEVER touched:
|
|
# - DinD nodes + NFS sidecars carry the INFINITO_SWARM_TEST_LABEL label (SPOT in
|
|
# default.env, stamped by routine/01_bootstrap.sh).
|
|
# - the act outer container is matched by act's own job-name prefix.
|
|
# - lab networks are matched by the `swarm-lab` token the harness assigns.
|
|
# Containers created before the label existed are not matched; those are wedged
|
|
# D-state remnants that a host `systemctl restart docker` clears anyway.
|
|
#
|
|
# Two layers:
|
|
# 1. docker (no privileges): rm matching containers + lab networks + prune.
|
|
# 2. host (root): D-state containers (wedged kernel NFS, "did not receive an
|
|
# exit event") survive `docker rm -f`; clear with umount + exportfs + a docker
|
|
# restart. Attempted with passwordless sudo, reported under a no-priv sandbox.
|
|
set -uo pipefail
|
|
|
|
# INFINITO_SWARM_TEST_LABEL lives in the env SPOT (default.env -> .env).
|
|
# shellcheck source=scripts/meta/env/load.sh
|
|
source "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../.." && pwd)/scripts/meta/env/load.sh"
|
|
|
|
_act_name='act--Test-Deploy-swarm'
|
|
|
|
_select_ids() {
|
|
docker ps -aq --filter "label=${INFINITO_SWARM_TEST_LABEL}" 2>/dev/null
|
|
docker ps -aq --filter "name=${_act_name}" 2>/dev/null
|
|
}
|
|
_select_names() {
|
|
docker ps -a --format '{{.Names}}' --filter "label=${INFINITO_SWARM_TEST_LABEL}" 2>/dev/null
|
|
docker ps -a --format '{{.Names}}' --filter "name=${_act_name}" 2>/dev/null
|
|
}
|
|
|
|
_names="$(_select_names | sort -u)"
|
|
if [ -n "${_names}" ]; then
|
|
echo ">>> swarm-clean: quiesce nested engines and detach NFS before removal"
|
|
for _node in ${_names}; do
|
|
if ! timeout 60 docker exec "${_node}" systemctl stop docker.socket docker >/dev/null 2>&1; then
|
|
echo " ${_node}: nested engine did not stop; removing it anyway"
|
|
fi
|
|
done
|
|
# shellcheck disable=SC2086
|
|
if ! timeout 600 bash "$(dirname "$0")/../unmount/nfs_mounts.sh" ${_names} 2>&1 | sed 's/^/ /'; then
|
|
echo " nfs detach reported failures; the host layer below clears what survives"
|
|
fi
|
|
fi
|
|
|
|
echo ">>> swarm-clean: leftover containers"
|
|
_ctrs="$(_select_ids | sort -u)"
|
|
if [ -n "${_ctrs}" ]; then
|
|
# shellcheck disable=SC2086 # intentional word-split of the id list
|
|
docker rm -f ${_ctrs} 2>&1 | sed 's/^/ /' || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
fi
|
|
|
|
for _reap in 1 2 3 4 5; do
|
|
_rest="$(_select_ids | sort -u)"
|
|
[ -n "${_rest}" ] || break
|
|
# shellcheck disable=SC2086 # intentional word-split of the id list
|
|
docker kill -s KILL ${_rest} >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
# shellcheck disable=SC2086 # intentional word-split of the id list
|
|
docker rm ${_rest} 2>&1 | sed 's/^/ /' || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
sleep 2
|
|
done
|
|
|
|
echo ">>> swarm-clean: leftover networks"
|
|
_nets="$(docker network ls --format '{{.Name}}' | grep -E "${INFINITO_SWARM_LAB_NET_NAME}" || true)"
|
|
if [ -n "${_nets}" ]; then
|
|
# shellcheck disable=SC2086 # intentional word-split of the network list
|
|
docker network rm ${_nets} 2>&1 | sed 's/^/ /' || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
fi
|
|
docker network prune -f >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo ">>> swarm-clean: leftover nfs-export volumes"
|
|
_vols="$(docker volume ls --format '{{.Name}}' | grep -E '_nfs-export$' || true)"
|
|
if [ -n "${_vols}" ]; then
|
|
# shellcheck disable=SC2086
|
|
if ! docker volume rm ${_vols} 2>&1 | sed 's/^/ /'; then
|
|
echo " some nfs-export volumes survived removal; a node still holds them"
|
|
fi
|
|
fi
|
|
|
|
_left="$(_select_names | sort -u)"
|
|
if [ -z "${_left}" ]; then
|
|
echo ">>> swarm-clean: done, no remnants"
|
|
exit 0
|
|
fi
|
|
|
|
echo ">>> swarm-clean: D-state remnants survived docker rm -f:"
|
|
echo " ${_left//$'\n'/$'\n' }"
|
|
if sudo -n true 2>/dev/null; then
|
|
echo ">>> clearing wedged kernel NFS on host (sudo)"
|
|
sudo umount -f -l "${INFINITO_DIR_VAR_LIB:?}" 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
sudo exportfs -ua 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
sudo systemctl restart containerd docker
|
|
echo ">>> containerd and docker restarted; D-state remnants cleared"
|
|
else
|
|
echo "!!! sudo unavailable here (sandbox). Clear on the host (this kills every container):"
|
|
echo " sudo umount -f -l ${INFINITO_DIR_VAR_LIB}; sudo exportfs -ua; sudo systemctl restart containerd docker"
|
|
echo " Never run this while a deploy is in flight: it wipes every exec instance and the run dies."
|
|
exit 1
|
|
fi
|