mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-09 12:17:13 +00:00
kevinveenbirkenbachandClaude Opus 4.8
e9b0d5f76e
The failover-drill assert polled service convergence for 60 iterations (120s). OpenProject web boots in ~140s, so after 06_drain_worker rescheduled it the replicas turned healthy ~20s after the poll gave up, and 07_assert_state.sh reported 'Replicas not fully converged: 1/3' and exited 1 even though the stack did converge moments later. The loop breaks early once converged, so raising the ceiling to 120 iterations (240s) costs fast apps nothing and only widens the window for heavy apps. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
61 lines
1.7 KiB
Bash
Executable File
61 lines
1.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
# shellcheck disable=SC1091
|
|
source "${SCRIPT_DIR}/../utils/_context.sh"
|
|
|
|
skip_if_no_swarm_service
|
|
skip_chaos_if_manager_pinned
|
|
|
|
REPL=""
|
|
for i in $(seq 1 120); do
|
|
REPL=$(docker exec "${MGR}" docker service ls \
|
|
--filter "name=${SERVICE_NAME}" --format '{{.Replicas}}')
|
|
echo "[${i}] ${ENTITY} replicas: ${REPL}"
|
|
if echo "${REPL}" | grep -qE '^([0-9]+)/\1$'; then
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "${REPL}" | grep -qE '^([0-9]+)/\1$' || {
|
|
echo "Replicas not fully converged: ${REPL}"
|
|
exit 1
|
|
}
|
|
|
|
if [ -n "${PRIMARY_NFS_VOLUME}" ]; then
|
|
docker exec "${NEW_NODE}" sh -c "
|
|
mkdir -p ${NFS_CHECK_MOUNTPOINT}
|
|
mount -t nfs4 -o vers=4 ${NFS_IP}:${NFS_STATE_PATH} ${NFS_CHECK_MOUNTPOINT} \
|
|
|| mount -t nfs4 -o vers=4 ${NFS_IP}:/ ${NFS_CHECK_MOUNTPOINT} \
|
|
|| mount -t nfs -o vers=3,nolock ${NFS_IP}:${NFS_STATE_PATH} ${NFS_CHECK_MOUNTPOINT} \
|
|
|| exit 1
|
|
grep -q 'pre-drain marker' ${NFS_CHECK_MOUNTPOINT}/${PRIMARY_NFS_VOLUME}/.pre-drain
|
|
rc=\$?
|
|
umount ${NFS_CHECK_MOUNTPOINT}
|
|
exit \$rc
|
|
" || {
|
|
echo "FAILURE: marker missing on NFS volume '${PRIMARY_NFS_VOLUME}'"
|
|
exit 1
|
|
}
|
|
else
|
|
echo "No NFS-flagged volume for '${APP_ID}' — skipping NFS marker assertion"
|
|
fi
|
|
|
|
APP_CTR=$(docker exec "${NEW_NODE}" docker ps \
|
|
--filter "name=${SERVICE_NAME}" \
|
|
--format '{{.ID}}' | head -1)
|
|
if [ -z "${APP_CTR}" ]; then
|
|
echo "FAILURE: cannot locate ${ENTITY} container on ${NEW_NODE}"
|
|
exit 1
|
|
fi
|
|
for i in $(seq 1 30); do
|
|
if probe_app_reachable "${NEW_NODE}" "${APP_CTR}" "${PROBE_PORT}"; then
|
|
echo "${ENTITY} reachable after reschedule"
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "FAILURE: ${ENTITY} not reachable after reschedule"
|
|
exit 1
|