mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-08 03:38:23 +00:00
03_wait_converge built the DB service name from the engine type:
"${DB_DEP}_${DB_DEP}" -> mariadb_mariadb. That name is only correct when
the database is central. force_shared_db deliberately leaves a
manager-pinned role carrying a literal `shared: false` alone, so
web-app-magento keeps its embedded engine, and sys-svc-rdbms deploys it as
magento_database inside magento's own stack.
The name came from the engine, which is global, and was looked up with
`docker service ls` across the whole swarm instead of the role's stack, so
the `shared` flag never entered the check at any step. Both polarities were
wrong:
variant #1 - no consumer pulls the central provider in, so
mariadb_mariadb does not exist. Run 32630684501 job 97175522113 polled it
ninety times and failed the drill, while the rescue artifact shows
magento_database at 1/1 Running and the play at failed=0.
variant #0 - matomo and mailu declare a Jinja `shared`, get force-shared
and drag svc-db-mariadb in. Job 95662862942 then reported
`db(mariadb): 1/1 Running 2 hours ago` beside `magento: Running 21
minutes ago` - green on a database magento never touches.
resolve_db_service asks the deployed swarm which service carries this
role's database: <entity>_database first, else <dep>_<dep>. The two cannot
collide, because sys-svc-rdbms renders the _database sidecar only under
`lookup('database', application_id, 'local')`. Exact-line matching, not
docker's substring name filter, per 246558a13.
Only web-app-magento changes behaviour: it is the one role that is both
pinned by force_shared_db and seen by the DB_DEP grep. matrix and
bigbluebutton never reach the DB half - their entity key carries no image,
so skip_if_no_swarm_service exits first - and discourse resolves
postgres_postgres exactly as before, in every variant.
A role that declares a DB dep no service serves now says so and gates the
app half alone, instead of polling a name nobody deployed.
diagnostics.sh carried the same guess in its container filter.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
162 lines
5.2 KiB
Bash
Executable File
162 lines
5.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
_REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../.." && pwd)"
|
|
# shellcheck source=scripts/meta/env/load.sh
|
|
source "${_REPO_ROOT}/scripts/meta/env/load.sh"
|
|
|
|
if [ -s "${SWARM_DRILL_ENV:-}" ]; then
|
|
set -a
|
|
# shellcheck source=/dev/null
|
|
source "${SWARM_DRILL_ENV}"
|
|
set +a
|
|
fi
|
|
|
|
: "${APP_ID:?APP_ID required}"
|
|
|
|
ENTITY="$(PYTHONPATH="${_REPO_ROOT}" "${PYTHON}" -c "from utils.roles.entity.name import get_entity_name; print(get_entity_name('${APP_ID}'))")"
|
|
|
|
STACK_NAME="${ENTITY}"
|
|
CUSTOM_IMAGE_REPO="${ENTITY}_custom"
|
|
|
|
ROLE_DIR="${_REPO_ROOT}/roles/${APP_ID}"
|
|
|
|
PRIMARY_SERVICE_KEY="$(PYTHONPATH="${_REPO_ROOT}" "${PYTHON}" -c "
|
|
import os, yaml
|
|
key = '${ENTITY}'
|
|
svc = os.path.join('${ROLE_DIR}', 'meta', 'services.yml')
|
|
if os.path.exists(svc):
|
|
data = yaml.safe_load(open(svc)) or {}
|
|
entity = data.get('${ENTITY}') if isinstance(data, dict) else None
|
|
if isinstance(entity, dict):
|
|
k = (entity.get('proxy') or {}).get('service_key')
|
|
if isinstance(k, str) and k:
|
|
key = k
|
|
print(key)
|
|
")"
|
|
SERVICE_NAME="${STACK_NAME}_${PRIMARY_SERVICE_KEY}"
|
|
|
|
DB_DEP=none
|
|
if [ -f "${ROLE_DIR}/meta/services.yml" ]; then
|
|
if grep -qE '^mariadb:' "${ROLE_DIR}/meta/services.yml"; then
|
|
DB_DEP=mariadb
|
|
elif grep -qE '^postgres:' "${ROLE_DIR}/meta/services.yml"; then
|
|
DB_DEP=postgres
|
|
fi
|
|
fi
|
|
|
|
DEFAULT_PLACEMENT_MANAGER=false
|
|
if [ -f "${ROLE_DIR}/meta/services.yml" ] &&
|
|
grep -qE "placement:[[:space:]]*manager\b" "${ROLE_DIR}/meta/services.yml"; then
|
|
DEFAULT_PLACEMENT_MANAGER=true
|
|
fi
|
|
|
|
PROBE_PORT="$(PYTHONPATH="${_REPO_ROOT}" "${PYTHON}" -c "
|
|
import os, yaml
|
|
port = 80
|
|
svc = os.path.join('${ROLE_DIR}', 'meta', 'services.yml')
|
|
if os.path.exists(svc):
|
|
data = yaml.safe_load(open(svc)) or {}
|
|
entity = data.get('${ENTITY}') if isinstance(data, dict) else None
|
|
if isinstance(entity, dict):
|
|
try:
|
|
port = entity['ports']['internal']['http'] or 80
|
|
except Exception:
|
|
port = 80
|
|
print(port)
|
|
")"
|
|
|
|
NFS_VOLUMES=""
|
|
if [ -f "${ROLE_DIR}/meta/volumes.yml" ]; then
|
|
NFS_VOLUMES="$(python3 -c "
|
|
import sys, yaml
|
|
with open('${ROLE_DIR}/meta/volumes.yml') as f:
|
|
data = yaml.safe_load(f) or {}
|
|
if isinstance(data, list):
|
|
entries = [e for e in data if isinstance(e, dict)]
|
|
elif isinstance(data, dict):
|
|
entries = [v for v in data.values() if isinstance(v, dict)]
|
|
else:
|
|
entries = []
|
|
for v in entries:
|
|
if v.get('type') != 'volume':
|
|
continue
|
|
if v.get('nfs') is False:
|
|
continue
|
|
docker_name = v.get('docker_name') or v.get('name')
|
|
if isinstance(docker_name, str):
|
|
print(docker_name)
|
|
")"
|
|
fi
|
|
|
|
PRIMARY_NFS_VOLUME="${NFS_VOLUMES%%$'\n'*}"
|
|
|
|
NFS_CHECK_MOUNTPOINT="/mnt/nfs-check"
|
|
|
|
export APP_ID ENTITY STACK_NAME SERVICE_NAME PRIMARY_SERVICE_KEY CUSTOM_IMAGE_REPO DB_DEP NFS_VOLUMES PRIMARY_NFS_VOLUME DEFAULT_PLACEMENT_MANAGER PROBE_PORT NFS_CHECK_MOUNTPOINT
|
|
|
|
# Exits the calling chaos step (09/10/11) with success when the role is
|
|
# manager-pinned (no worker task exists to drain).
|
|
skip_chaos_if_manager_pinned() {
|
|
if [ "${DEFAULT_PLACEMENT_MANAGER}" = true ]; then
|
|
echo "SKIP: ${ENTITY} declares placement: manager (single-node on the swarm manager) — drain-a-worker reschedule chaos does not apply"
|
|
exit 0
|
|
fi
|
|
}
|
|
|
|
has_swarm_service() {
|
|
local _names=""
|
|
[ -n "${MGR:-}" ] || return 1
|
|
if ! _names="$(docker exec "${MGR}" docker stack services \
|
|
--format '{{.Name}}' "${STACK_NAME}" 2>/dev/null)"; then
|
|
_names=""
|
|
fi
|
|
printf '%s\n' "${_names}" | grep -qxF "${SERVICE_NAME}"
|
|
}
|
|
|
|
# Prints the swarm service carrying APP_ID's database. A role that pins its own
|
|
# engine runs it as <entity>_database inside its own stack, a central provider
|
|
# as <dep>_<dep> in its own (plugins/lookup/database.py).
|
|
resolve_db_service() {
|
|
local _names=""
|
|
[ "${DB_DEP}" != "none" ] || return 0
|
|
[ -n "${MGR:-}" ] || return 0
|
|
if ! _names="$(timeout 30 docker exec "${MGR}" docker service ls --format '{{.Name}}' 2>/dev/null)"; then
|
|
return 0
|
|
fi
|
|
if printf '%s\n' "${_names}" | grep -qxF "${STACK_NAME}_database"; then
|
|
printf '%s\n' "${STACK_NAME}_database"
|
|
elif printf '%s\n' "${_names}" | grep -qxF "${DB_DEP}_${DB_DEP}"; then
|
|
printf '%s\n' "${DB_DEP}_${DB_DEP}"
|
|
fi
|
|
}
|
|
|
|
# Exits the calling gate (03/05/06/07) with success when the role deploys no
|
|
# swarm service to converge or drain.
|
|
skip_if_no_swarm_service() {
|
|
if ! has_swarm_service; then
|
|
echo "SKIP: ${ENTITY} (${APP_ID}) deployed no swarm service — converge/chaos gate does not apply"
|
|
exit 0
|
|
fi
|
|
}
|
|
|
|
# Param: $1 node container, $2 app container id, $3 port
|
|
probe_app_reachable() {
|
|
docker exec "$1" docker exec "$2" sh -c \
|
|
"curl -sS http://localhost:$3/ || wget -qO- http://localhost:$3/ || nc -z localhost $3" >/dev/null 2>&1 ||
|
|
docker exec "$1" docker exec "$2" bash -c "exec 3<>/dev/tcp/localhost/$3" >/dev/null 2>&1 ||
|
|
docker exec "$1" sh -c "docker inspect -f '{{.State.Health.Status}}' $2 | grep -qx healthy" >/dev/null 2>&1
|
|
}
|
|
|
|
if [ "${SWARM_NFS_PILOT_VERBOSE:-0}" = "1" ]; then
|
|
echo "==> APP_ID=${APP_ID}"
|
|
echo " ENTITY=${ENTITY}"
|
|
echo " STACK_NAME=${STACK_NAME}"
|
|
echo " SERVICE_NAME=${SERVICE_NAME}"
|
|
echo " CUSTOM_IMAGE_REPO=${CUSTOM_IMAGE_REPO}"
|
|
echo " DB_DEP=${DB_DEP}"
|
|
echo " PRIMARY_NFS_VOLUME=${PRIMARY_NFS_VOLUME}"
|
|
echo " DEFAULT_PLACEMENT_MANAGER=${DEFAULT_PLACEMENT_MANAGER}"
|
|
echo " PROBE_PORT=${PROBE_PORT}"
|
|
fi
|