mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-08 03:38:23 +00:00
kevinveenbirkenbachandClaude Opus 5
f9fef77b1e
docker's name filter matches substrings, so every stack whose sidecar is named after its app answered with two containers and two services. The drill took the first, which after a reschedule was the MCP sidecar: it declares no healthcheck and does not listen on the app port, so the step waited sixty seconds on the wrong process. Selection now goes through the swarm service label and an exact name match, both behind one helper. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
74 lines
2.4 KiB
Bash
Executable File
74 lines
2.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
# shellcheck disable=SC1091
|
|
source "${SCRIPT_DIR}/../utils/_context.sh"
|
|
|
|
skip_if_no_swarm_service
|
|
skip_chaos_if_manager_pinned
|
|
|
|
REPL=""
|
|
for i in $(seq 1 120); do
|
|
REPL=$(service_replicas "${MGR}" "${SERVICE_NAME}")
|
|
echo "[${i}] ${ENTITY} replicas: ${REPL}"
|
|
if echo "${REPL}" | grep -qE '^([0-9]+)/\1$'; then
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "${REPL}" | grep -qE '^([0-9]+)/\1$' || {
|
|
echo "Replicas not fully converged: ${REPL}"
|
|
exit 1
|
|
}
|
|
|
|
if [ -n "${PRIMARY_NFS_VOLUME}" ]; then
|
|
docker exec "${NEW_NODE}" sh -c "
|
|
mkdir -p ${NFS_CHECK_MOUNTPOINT}
|
|
mount -t nfs4 -o vers=4 ${NFS_IP}:${NFS_STATE_PATH} ${NFS_CHECK_MOUNTPOINT} \
|
|
|| mount -t nfs4 -o vers=4 ${NFS_IP}:/ ${NFS_CHECK_MOUNTPOINT} \
|
|
|| mount -t nfs -o vers=3,nolock ${NFS_IP}:${NFS_STATE_PATH} ${NFS_CHECK_MOUNTPOINT} \
|
|
|| exit 1
|
|
grep -q 'pre-drain marker' ${NFS_CHECK_MOUNTPOINT}/${PRIMARY_NFS_VOLUME}/.pre-drain
|
|
rc=\$?
|
|
umount ${NFS_CHECK_MOUNTPOINT}
|
|
exit \$rc
|
|
" || {
|
|
echo "FAILURE: marker missing on NFS volume '${PRIMARY_NFS_VOLUME}'"
|
|
exit 1
|
|
}
|
|
else
|
|
echo "No NFS-flagged volume for '${APP_ID}' — skipping NFS marker assertion"
|
|
fi
|
|
|
|
APP_CTR=$(service_container_id "${NEW_NODE}" "${SERVICE_NAME}")
|
|
if [ -z "${APP_CTR}" ]; then
|
|
echo "FAILURE: cannot locate ${ENTITY} container on ${NEW_NODE}"
|
|
exit 1
|
|
fi
|
|
DRILL_MIN_WAIT_SECONDS=60
|
|
DECLARED_READY_SECONDS=$(docker exec "${NEW_NODE}" docker inspect "${APP_CTR}" 2>/dev/null |
|
|
awk '
|
|
/"Healthcheck": \{/ { inhc = 1 }
|
|
inhc && /"StartPeriod":/ { gsub(/[^0-9]/, "", $2); sp = $2 }
|
|
inhc && /"Interval":/ { gsub(/[^0-9]/, "", $2); iv = $2 }
|
|
inhc && /"Retries":/ { gsub(/[^0-9]/, "", $2); rt = $2; inhc = 0 }
|
|
END {
|
|
budget = 0
|
|
if (sp != "" && iv != "" && rt != "") {
|
|
budget = int(sp / 1000000000) + int(iv / 1000000000) * rt
|
|
}
|
|
print budget
|
|
}')
|
|
READY_BUDGET=$((DECLARED_READY_SECONDS > DRILL_MIN_WAIT_SECONDS ? DECLARED_READY_SECONDS : DRILL_MIN_WAIT_SECONDS))
|
|
echo "waiting up to ${READY_BUDGET}s for ${ENTITY} to answer on ${NEW_NODE} (declared ${DECLARED_READY_SECONDS}s)"
|
|
for _ in $(seq 1 $((READY_BUDGET / 2))); do
|
|
if probe_app_reachable "${NEW_NODE}" "${APP_CTR}" "${PROBE_PORT}"; then
|
|
echo "${ENTITY} reachable after reschedule"
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "FAILURE: ${ENTITY} not reachable after reschedule within ${READY_BUDGET}s"
|
|
exit 1
|