mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-10 04:36:40 +00:00
kevinveenbirkenbachandClaude Fable 5
c6ed8234b4
The post-reschedule assert mounted the state export with vers=3, which
ganesha does not serve ('Protocol not supported'), so the marker check
false-negatived and killed the gitea roundtrip; reproduced and the
v4 mount shown to read the marker on the live cluster. Replace the
single mount with a cascade covering every production client_src form:
nfs4 vers=4 against the full path (ganesha), against :/ (kernel fsid=0
root), then the previous vers=3 full-path arm (kernel v3). The block
also returned the umount status instead of the grep result, letting a
missing marker pass silently; propagate the grep rc explicitly.
The in-container reachability probe's last arm used bash /dev/tcp, but
e.g. the ejabberd image ships neither bash nor curl; use busybox
nc -z instead and move the probe chain into probe_app_reachable() in
_context.sh alongside a NFS_CHECK_MOUNTPOINT SPOT.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
61 lines
1.7 KiB
Bash
Executable File
61 lines
1.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
# shellcheck disable=SC1091
|
|
source "${SCRIPT_DIR}/_context.sh"
|
|
|
|
skip_if_no_swarm_service
|
|
skip_chaos_if_manager_pinned
|
|
|
|
REPL=""
|
|
for i in $(seq 1 60); do
|
|
REPL=$(docker exec "${MGR}" docker service ls \
|
|
--filter "name=${SERVICE_NAME}" --format '{{.Replicas}}')
|
|
echo "[${i}] ${ENTITY} replicas: ${REPL}"
|
|
if echo "${REPL}" | grep -qE '^([0-9]+)/\1$'; then
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "${REPL}" | grep -qE '^([0-9]+)/\1$' || {
|
|
echo "Replicas not fully converged: ${REPL}"
|
|
exit 1
|
|
}
|
|
|
|
if [ -n "${PRIMARY_NFS_VOLUME}" ]; then
|
|
docker exec "${NEW_NODE}" sh -c "
|
|
mkdir -p ${NFS_CHECK_MOUNTPOINT}
|
|
mount -t nfs4 -o vers=4 ${NFS_IP}:${NFS_STATE_PATH} ${NFS_CHECK_MOUNTPOINT} \
|
|
|| mount -t nfs4 -o vers=4 ${NFS_IP}:/ ${NFS_CHECK_MOUNTPOINT} \
|
|
|| mount -t nfs -o vers=3,nolock ${NFS_IP}:${NFS_STATE_PATH} ${NFS_CHECK_MOUNTPOINT} \
|
|
|| exit 1
|
|
grep -q 'pre-drain marker' ${NFS_CHECK_MOUNTPOINT}/${PRIMARY_NFS_VOLUME}/.pre-drain
|
|
rc=\$?
|
|
umount ${NFS_CHECK_MOUNTPOINT}
|
|
exit \$rc
|
|
" || {
|
|
echo "FAILURE: marker missing on NFS volume '${PRIMARY_NFS_VOLUME}'"
|
|
exit 1
|
|
}
|
|
else
|
|
echo "No NFS-flagged volume for '${APP_ID}' — skipping NFS marker assertion"
|
|
fi
|
|
|
|
APP_CTR=$(docker exec "${NEW_NODE}" docker ps \
|
|
--filter "name=${SERVICE_NAME}" \
|
|
--format '{{.ID}}' | head -1)
|
|
if [ -z "${APP_CTR}" ]; then
|
|
echo "FAILURE: cannot locate ${ENTITY} container on ${NEW_NODE}"
|
|
exit 1
|
|
fi
|
|
for i in $(seq 1 30); do
|
|
if probe_app_reachable "${NEW_NODE}" "${APP_CTR}" "${PROBE_PORT}"; then
|
|
echo "${ENTITY} reachable after reschedule"
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "FAILURE: ${ENTITY} not reachable after reschedule"
|
|
exit 1
|