mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-10 04:36:40 +00:00
kevinveenbirkenbachandClaude Opus 5
bf70d0e0ee
01_bootstrap.sh ran apt-get update with no timeout, so a stalled mirror
connection hung until the drill's global budget killed it - twice in run
32284541284 (jobs 96178507396 funkwhale, 96178507443 friendica), each
after hours of silence. The journal pins it exactly: sudo[462047] opened
19:51:36 and closed 00:07:30, 4h15m54s in a single apt-get update, and
the session only ended because timeout fired. Healthy runs of the same
line in the same jobs took 3s, 21s and 41s.
The cost is not one red job. distros.sh:130 is the only ceiling and it
wraps the whole per-distro sequence in the remaining global budget, so
one stalled fetch also takes the untried distros with it: funkwhale ran
2 of 5, friendica 3 of 5, both ending at budget remaining=-19s/-20s.
-qq hid the stall and || true cannot catch a hang.
Same failure class and same remedy as ba8a72309, which bounded
scripts/install/apt.sh; this closes the sibling call site it left open.
Constants are taken verbatim from there rather than re-derived. timeout
runs under sudo so SIGTERM reaches apt-get instead of the sudo wrapper -
the 4h session above is what the other order looks like. -E stays so the
package-cache proxy variables survive.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
87 lines
4.2 KiB
Bash
Executable File
87 lines
4.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# SPOT for the cluster bring-up, one CI step: host-side compose build + up (node
|
|
# image layered on the distro's infinito image, so python + ansible + the CLI are
|
|
# baked -- no per-node .deb bootstrap), then every node concern (systemd wait,
|
|
# IPs, lab DNS, repo unpack) via compose/swarm/playbook.yml over the docker
|
|
# connection. Host-side pre-clean here is only the bind-mount dir + leftover
|
|
# containers; stale root-owned NFS writes are wiped in-node by the play (this
|
|
# often non-root act runner cannot delete them).
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
set -a
|
|
# shellcheck source=scripts/tests/deploy/swarm/utils/topology/base.sh
|
|
. "${SCRIPT_DIR}/../utils/topology/base.sh"
|
|
set +a
|
|
|
|
# shellcheck source=scripts/meta/env/load.sh
|
|
source "${SCRIPT_DIR}/../../../../../scripts/meta/env/load.sh"
|
|
|
|
: "${RUNNER_TEMP:?}" "${APP_ID:?}" "${INFINITO_DOMAIN:?}" "${INFINITO_CONTAINER:?}"
|
|
|
|
if command -v apt-get >/dev/null 2>&1; then
|
|
APT_TIMEOUT=10m
|
|
APT_OPTS=(-o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30)
|
|
if [ "$(id -u)" -eq 0 ]; then
|
|
timeout -k 30 "${APT_TIMEOUT}" apt-get "${APT_OPTS[@]}" update -qq || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
else
|
|
sudo -E timeout -k 30 "${APT_TIMEOUT}" apt-get "${APT_OPTS[@]}" update -qq || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
fi
|
|
fi
|
|
|
|
if docker exec "${INFINITO_CONTAINER}" docker ps --format '{{.Names}}' 2>/dev/null | grep -qx 'tor'; then
|
|
echo "==> quiescing CI compose tor (competing onion publisher)"
|
|
docker exec "${INFINITO_CONTAINER}" docker stop tor
|
|
fi
|
|
|
|
bash "${SCRIPT_DIR}/../utils/unmount/nfs_mounts.sh" "${NFS_SERVER}" >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
bash "${SCRIPT_DIR}/../utils/unmount/host_state.sh" "${INFINITO_DIR_VAR_LIB:?}"
|
|
for node in "${MGR}" "${WRK1}" "${WRK2}" "${NFS_SERVER}" "${BACKUP_NODE}"; do
|
|
docker rm -f "${node}" >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
done
|
|
docker volume rm "${SWARM_NAME}_nfs-export" >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../../.." && pwd)"
|
|
if [ -z "${INFINITO_IMAGE:-}" ]; then
|
|
INFINITO_IMAGE="$(bash "${REPO_ROOT}/scripts/meta/resolve/image/local.sh"):${INFINITO_IMAGE_TAG:?}"
|
|
export INFINITO_IMAGE
|
|
if ! docker image inspect "${INFINITO_IMAGE}" >/dev/null 2>&1; then
|
|
echo "==> building local infinito image ${INFINITO_IMAGE} for distro ${INFINITO_DISTRO}"
|
|
make -C "${REPO_ROOT}" build
|
|
fi
|
|
fi
|
|
|
|
COMPOSE_FILE="${SCRIPT_DIR}/../../../../../compose/swarm/compose.yml"
|
|
COMPOSE_ARGS=(-f "${COMPOSE_FILE}")
|
|
CACHE_FRONTEND="infinito-package-cache-frontend"
|
|
CACHE_NET=""
|
|
if docker inspect "${CACHE_FRONTEND}" >/dev/null 2>&1; then
|
|
CACHE_NET="$(docker inspect "${CACHE_FRONTEND}" --format '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' | head -n1)"
|
|
echo "==> package-cache detected on ${CACHE_NET}; wiring swarm nodes to it"
|
|
COMPOSE_ARGS+=(-f "${SCRIPT_DIR}/../../../../../compose/swarm/cache.override.yml")
|
|
fi
|
|
|
|
build_attempts=3
|
|
for attempt in $(seq 1 "${build_attempts}"); do
|
|
docker compose "${COMPOSE_ARGS[@]}" -p "${SWARM_NAME}" --profile drill build && break
|
|
if [ "${attempt}" -eq "${build_attempts}" ]; then
|
|
echo "FAILURE: node image build failed after ${build_attempts} attempts" >&2
|
|
exit 1
|
|
fi
|
|
sleep $((attempt * 5))
|
|
done
|
|
|
|
docker compose "${COMPOSE_ARGS[@]}" -p "${SWARM_NAME}" --profile drill up -d
|
|
|
|
if [ -n "${CACHE_NET}" ]; then
|
|
for node in "${MGR}" "${WRK1}" "${WRK2}" "${NFS_SERVER}" "${BACKUP_NODE}"; do
|
|
docker network connect "${CACHE_NET}" "${node}" >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
done
|
|
fi
|
|
|
|
ansible-playbook \
|
|
-i "${MGR},${WRK1},${WRK2},${NFS_SERVER},${BACKUP_NODE}," \
|
|
-c docker \
|
|
"${SCRIPT_DIR}/../../../../../compose/swarm/playbook.yml"
|