Files
computer-playbook/scripts/tests/deploy/swarm/utils/_context.sh
T
kevinveenbirkenbachandClaude Opus 5.5 4fcf097e5b fix(svc-bkp-nfs-2-local): leave backup-disabled volumes out of the export snapshot
The NFS backup rsynced the whole export and honoured only one exclude, the
shared backup root. Volumes declared `backup: false` were copied anyway and
then pulled a second time by remote-2-local; in the swarm DR drill of
Magento#0 (run 40625679270) docs_sites alone (a full repo clone plus its
prepared and work trees) took about 15.5 GiB per copy and filled the
runner. The volume backup already skipped these through the backup_volume
lookup; only the NFS path ignored the declaration.

- utils/storage/nfs.py: volume_dir() names the state directory a volume
  binds to (pinned name or key), shared with compose_volumes;
  backup_disabled_dirs() returns every NFS-backed `backup: false` volume
  dir that no backed-up volume also binds to.
- The role-local lookup backup_nfs_excludes feeds them into
  BKP_NFS_2_LOCAL_EXCLUDES next to the backup root; script.sh takes any
  number of excludes. Compose installs still pass only the backup root.
- recover.py protects every live state directory the snapshot does not
  hold, so `rsync --delete` keeps the excluded volumes (model weights,
  the docs tree, caches) and volumes created after the snapshot instead
  of wiping them.
- The swarm DR drill restores BACKUP_NFS_VOLUME, the first NFS volume
  not declared `backup: false`; PRIMARY_NFS_VOLUME stays for the drain
  tests. BookWyrm now drills bookwyrm_media; decidim, fusiondirectory,
  meta, coturn, svc-registry-cache and svc-registry-docker have nothing
  backed up on NFS and skip the drill.

Measured: compose_volumes renders identically before and after for 291
roles across compose/swarm x local/nfs (1940 of 1940); the 15 excluded
directories are exactly the backup-disabled device dirs; the systemd unit
renders shell-safe on ansible-core 2.20 and 2.21 and stays byte-identical
in compose and swarm-local. A new unit test runs the real script.sh over
two generations and the real recover.py back into the export, and fails
without the restore protection.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-28 17:43:31 +01:00

185 lines
6.2 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
_REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../.." && pwd)"
# shellcheck source=scripts/meta/env/load.sh
source "${_REPO_ROOT}/scripts/meta/env/load.sh"
if [ -s "${SWARM_DRILL_ENV:-}" ]; then
set -a
# shellcheck source=/dev/null
source "${SWARM_DRILL_ENV}"
set +a
fi
: "${APP_ID:?APP_ID required}"
ENTITY="$(PYTHONPATH="${_REPO_ROOT}" "${PYTHON}" -c "from utils.roles.entity.name import entity_name; print(entity_name('${APP_ID}'))")"
STACK_NAME="${ENTITY}"
CUSTOM_IMAGE_REPO="${ENTITY}_custom"
ROLE_DIR="${_REPO_ROOT}/roles/${APP_ID}"
PRIMARY_SERVICE_KEY="$(PYTHONPATH="${_REPO_ROOT}" "${PYTHON}" -c "
import os, yaml
key = '${ENTITY}'
svc = os.path.join('${ROLE_DIR}', 'meta', 'services.yml')
if os.path.exists(svc):
data = yaml.safe_load(open(svc)) or {}
entity = data.get('${ENTITY}') if isinstance(data, dict) else None
if isinstance(entity, dict):
k = (entity.get('proxy') or {}).get('service_key')
if isinstance(k, str) and k:
key = k
print(key)
")"
SERVICE_NAME="${STACK_NAME}_${PRIMARY_SERVICE_KEY}"
DB_DEP=none
if [ -f "${ROLE_DIR}/meta/services.yml" ]; then
if grep -qE '^mariadb:' "${ROLE_DIR}/meta/services.yml"; then
DB_DEP=mariadb
elif grep -qE '^postgres:' "${ROLE_DIR}/meta/services.yml"; then
DB_DEP=postgres
fi
fi
DEFAULT_PLACEMENT_MANAGER="$(PYTHONPATH="${_REPO_ROOT}" "${PYTHON}" -c "
from utils.roles.meta_lookup import get_role_placement
placement = get_role_placement('${ROLE_DIR}', role_name='${APP_ID}') or ''
print('true' if placement.strip() == 'manager' else 'false')
")"
PROBE_PORT="$(PYTHONPATH="${_REPO_ROOT}" "${PYTHON}" -c "
import os, yaml
port = 80
svc = os.path.join('${ROLE_DIR}', 'meta', 'services.yml')
if os.path.exists(svc):
data = yaml.safe_load(open(svc)) or {}
entity = data.get('${ENTITY}') if isinstance(data, dict) else None
if isinstance(entity, dict):
try:
port = entity['ports']['internal']['http'] or 80
except Exception:
port = 80
print(port)
")"
NFS_VOLUMES=""
BACKUP_NFS_VOLUMES=""
if [ -f "${ROLE_DIR}/meta/volumes.yml" ]; then
_nfs_volumes_py="
import sys, yaml
with open('${ROLE_DIR}/meta/volumes.yml') as f:
data = yaml.safe_load(f) or {}
if isinstance(data, list):
entries = [e for e in data if isinstance(e, dict)]
elif isinstance(data, dict):
entries = [v for v in data.values() if isinstance(v, dict)]
else:
entries = []
for v in entries:
if v.get('type') != 'volume':
continue
if v.get('nfs') is False:
continue
if sys.argv[1:] == ['backed-up'] and v.get('backup') is False:
continue
docker_name = v.get('docker_name') or v.get('name')
if isinstance(docker_name, str):
print(docker_name)
"
NFS_VOLUMES="$(python3 -c "${_nfs_volumes_py}")"
BACKUP_NFS_VOLUMES="$(python3 -c "${_nfs_volumes_py}" backed-up)"
fi
PRIMARY_NFS_VOLUME="${NFS_VOLUMES%%$'\n'*}"
BACKUP_NFS_VOLUME="${BACKUP_NFS_VOLUMES%%$'\n'*}"
NFS_CHECK_MOUNTPOINT="/mnt/nfs-check"
export APP_ID ENTITY STACK_NAME SERVICE_NAME PRIMARY_SERVICE_KEY CUSTOM_IMAGE_REPO DB_DEP NFS_VOLUMES PRIMARY_NFS_VOLUME BACKUP_NFS_VOLUME DEFAULT_PLACEMENT_MANAGER PROBE_PORT NFS_CHECK_MOUNTPOINT
# Exits the calling chaos step (09/10/11) with success when the role is
# manager-pinned (no worker task exists to drain).
skip_chaos_if_manager_pinned() {
if [ "${DEFAULT_PLACEMENT_MANAGER}" = true ]; then
echo "SKIP: ${ENTITY} declares placement: manager (single-node on the swarm manager) — drain-a-worker reschedule chaos does not apply"
exit 0
fi
}
has_swarm_service() {
local _names=""
[ -n "${MGR:-}" ] || return 1
if ! _names="$(docker exec "${MGR}" docker stack services \
--format '{{.Name}}' "${STACK_NAME}" 2>/dev/null)"; then
_names=""
fi
printf '%s\n' "${_names}" | grep -qxF "${SERVICE_NAME}"
}
# Prints the swarm service carrying APP_ID's database. A role that pins its own
# engine runs it as <entity>_database inside its own stack, a central provider
# as <dep>_<dep> in its own (plugins/lookup/database.py).
resolve_db_service() {
local _names=""
[ "${DB_DEP}" != "none" ] || return 0
[ -n "${MGR:-}" ] || return 0
if ! _names="$(timeout 30 docker exec "${MGR}" docker service ls --format '{{.Name}}' 2>/dev/null)"; then
return 0
fi
if printf '%s\n' "${_names}" | grep -qxF "${STACK_NAME}_database"; then
printf '%s\n' "${STACK_NAME}_database"
elif printf '%s\n' "${_names}" | grep -qxF "${DB_DEP}_${DB_DEP}"; then
printf '%s\n' "${DB_DEP}_${DB_DEP}"
fi
}
# Exits the calling gate (03/05/06/07) with success when the role deploys no
# swarm service to converge or drain.
skip_if_no_swarm_service() {
if ! has_swarm_service; then
echo "SKIP: ${ENTITY} (${APP_ID}) deployed no swarm service — converge/chaos gate does not apply"
exit 0
fi
}
# A stack whose sidecar is named after its app (<app>mcp) makes every
# `--filter name=<app>` a two-hit match, so both helpers below match exactly.
# Param: $1 node container, $2 swarm service name
service_container_id() {
docker exec "$1" docker ps \
--filter "label=com.docker.swarm.service.name=$2" \
--format '{{.ID}}' | head -1
}
# Param: $1 manager container, $2 swarm service name
service_replicas() {
docker exec "$1" docker service ls \
--filter "name=$2" --format '{{.Name}} {{.Replicas}}' |
awk -v svc="$2" '$1 == svc { print $2 }'
}
# Param: $1 node container, $2 app container id, $3 port
probe_app_reachable() {
docker exec "$1" docker exec "$2" sh -c \
"curl -sS http://localhost:$3/ || wget -qO- http://localhost:$3/ || nc -z localhost $3" >/dev/null 2>&1 ||
docker exec "$1" docker exec "$2" bash -c "exec 3<>/dev/tcp/localhost/$3" >/dev/null 2>&1 ||
docker exec "$1" sh -c "docker inspect -f '{{.State.Health.Status}}' $2 | grep -qx healthy" >/dev/null 2>&1
}
if [ "${SWARM_NFS_PILOT_VERBOSE:-0}" = "1" ]; then
echo "==> APP_ID=${APP_ID}"
echo " ENTITY=${ENTITY}"
echo " STACK_NAME=${STACK_NAME}"
echo " SERVICE_NAME=${SERVICE_NAME}"
echo " CUSTOM_IMAGE_REPO=${CUSTOM_IMAGE_REPO}"
echo " DB_DEP=${DB_DEP}"
echo " PRIMARY_NFS_VOLUME=${PRIMARY_NFS_VOLUME}"
echo " BACKUP_NFS_VOLUME=${BACKUP_NFS_VOLUME}"
echo " DEFAULT_PLACEMENT_MANAGER=${DEFAULT_PLACEMENT_MANAGER}"
echo " PROBE_PORT=${PROBE_PORT}"
fi