mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 19:01:54 +00:00
kevinveenbirkenbachandClaude Opus 5
d14354e892
Run 32118850138 lost the :53 listener, so the sampler counts listeners. Run 32538854067 lost DNS with the listener still bound: sockets-udp.txt showed dnsmasq holding 0.0.0.0:53 while every forwarded query timed out, and the sampler reported that hour as healthy. Each tick now also records what a glibc caller gets back — for the zone this resolver owns, derived from its own address= line, and for one name it must forward — with the latency that caller paid. The rescue collector probes the in-zone name too: probing external names alone cannot separate "forwarding is gone" from "dnsmasq is gone". The sampler now also runs on the swarm lab DNS nodes; it only ever ran in the compose path, which is why the Magento artifact carried no sampler log at all. Its location comes from group_vars FILE_DNS53_SAMPLER_LOG and is required at every call site, so a run cannot quietly collect nothing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
249 lines
13 KiB
Bash
Executable File
249 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Deploy one app on one distro: bring the stack up, init the inventory (one folder
|
|
# per meta/variants.yml round), deploy --full-cycle (sync + async per variant), then
|
|
# always remove the stack so the next distro starts fresh.
|
|
#
|
|
# Required env:
|
|
# INFINITO_DISTRO="arch|debian|ubuntu|fedora|centos"
|
|
# INFINITO_INVENTORY_DIR="/path/to/inventory"
|
|
#
|
|
# Optional env:
|
|
# PYTHON="python3"
|
|
# variant="<csv>" pin to one or more matrix rounds, e.g. "2" or "0,1,2"
|
|
# (the runner-split bundle from CI discovery); empty = all
|
|
|
|
: "${INFINITO_DISTRO:?INFINITO_DISTRO must be set (e.g. arch)}"
|
|
: "${INFINITO_INVENTORY_DIR:?INFINITO_INVENTORY_DIR must be set}"
|
|
: "${INFINITO_DOCKER_VOLUME:?INFINITO_DOCKER_VOLUME must be set}"
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
|
|
# shellcheck source=/dev/null
|
|
source <(grep -E '^INFINITO_(PLAYWRIGHT_REPORTS_BASE_DIR|RESCUE_DIAGNOSTICS_DIR|RESCUE_DIAGNOSTICS_BASE|DNS53_SAMPLER_LOG)=' "${REPO_ROOT}/.env")
|
|
: "${INFINITO_RESCUE_DIAGNOSTICS_BASE:?INFINITO_RESCUE_DIAGNOSTICS_BASE must be set}"
|
|
|
|
apps=""
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage:
|
|
INFINITO_DISTRO=<distro> INFINITO_INVENTORY_DIR=<dir> INFINITO_DOCKER_VOLUME=<abs_path> \
|
|
scripts/tests/deploy/ci/dedicated.sh \ # nocheck: self-path-reference
|
|
--apps <app_ids>
|
|
EOF
|
|
}
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--apps)
|
|
apps="${2:-}"
|
|
shift 2
|
|
;;
|
|
-h | --help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "[ERROR] Unknown argument: $1" >&2
|
|
usage
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
[[ -n "${apps}" ]] || {
|
|
echo "[ERROR] --apps is required" >&2
|
|
usage
|
|
exit 2
|
|
}
|
|
|
|
cd "${REPO_ROOT}"
|
|
|
|
echo "=== distro=${INFINITO_DISTRO} app=${apps} (debug always on) ==="
|
|
|
|
cleanup() {
|
|
rc=$?
|
|
|
|
local _playwright_host_dir="/tmp/playwright-artifacts/${INFINITO_DISTRO}/${apps}"
|
|
mkdir -p "${_playwright_host_dir}"
|
|
echo ">>> Copying Playwright artifacts from ${INFINITO_CONTAINER} to ${_playwright_host_dir}"
|
|
# nocheck: container-cp - container-to-host extraction on the CI host itself
|
|
docker cp "${INFINITO_CONTAINER}:${INFINITO_PLAYWRIGHT_REPORTS_BASE_DIR}/." \
|
|
"${_playwright_host_dir}" 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
local _rescue_host_dir="${INFINITO_RESCUE_DIAGNOSTICS_BASE}/${INFINITO_DISTRO}/${apps}"
|
|
mkdir -p "${_rescue_host_dir}"
|
|
echo ">>> Capturing rescue diagnostics inside ${INFINITO_CONTAINER} (recursive DiD snapshot) before teardown removes it"
|
|
timeout 1500 docker exec \
|
|
-e "INFINITO_RESCUE_DIAGNOSTICS_DIR=${INFINITO_RESCUE_DIAGNOSTICS_DIR}" \
|
|
-e "INFINITO_DNS53_SAMPLER_LOG=${INFINITO_DNS53_SAMPLER_LOG}" \
|
|
"${INFINITO_CONTAINER}" \
|
|
python3 /opt/src/infinito/utils/diagnostics/container.py \
|
|
"${apps}" "compose post-deploy failure" 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
echo ">>> Copying rescue diagnostics from ${INFINITO_CONTAINER} to ${_rescue_host_dir}"
|
|
docker exec "${INFINITO_CONTAINER}" \
|
|
tar -C "${INFINITO_RESCUE_DIAGNOSTICS_DIR}" -cf - . 2>/dev/null |
|
|
tar -C "${_rescue_host_dir}" -xf - 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
local _did_id
|
|
_did_id="$(docker inspect --format '{{.Id}}' "${INFINITO_CONTAINER}" 2>/dev/null)" || _did_id=""
|
|
echo ">>> Capturing rescue diagnostics on the CI host itself into ${_rescue_host_dir}/ci-host"
|
|
INFINITO_RESCUE_DIAGNOSTICS_DIR="${_rescue_host_dir}/ci-host" \
|
|
RESCUE_SEEN="${_did_id}" \
|
|
timeout 900 python3 "${REPO_ROOT}/utils/diagnostics/container.py" \
|
|
"${apps}" "ci host at compose post-deploy failure" 2>/dev/null || true # nocheck: shell-or-true -- the collector exits 1 by contract so the caller cannot mistake a snapshot for a passing deploy
|
|
|
|
bash scripts/tests/deploy/utils/rescue_index.sh "${_rescue_host_dir}"
|
|
|
|
local _inv_parent
|
|
_inv_parent="$(dirname "${INFINITO_INVENTORY_DIR}")"
|
|
echo ">>> Copying generated inventory from ${INFINITO_CONTAINER} to host ${_inv_parent}"
|
|
mkdir -p "${_inv_parent}"
|
|
# nocheck: container-cp - container-to-host extraction on the CI host itself
|
|
docker cp "${INFINITO_CONTAINER}:${_inv_parent}/." "${_inv_parent}/" 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo ">>> Removing stack for distro ${INFINITO_DISTRO} (fresh start for next distro)"
|
|
"${PYTHON}" -m cli.administration.deploy.development down || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo ">>> HARD cleanup (containers/volumes/networks/images/build-cache)"
|
|
echo ">>> Docker disk usage before HARD cleanup"
|
|
docker system df || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
_cleanup_project="${COMPOSE_PROJECT_NAME:-}"
|
|
if [[ -n "${_cleanup_project}" ]]; then
|
|
mapfile -t ids < <(docker ps -aq --filter "label=com.docker.compose.project=${_cleanup_project}" || true)
|
|
else
|
|
mapfile -t ids < <(docker ps -aq || true)
|
|
fi
|
|
if ((${#ids[@]} > 0)); then
|
|
docker rm -f "${ids[@]}" >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
fi
|
|
|
|
docker network prune -f >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
docker volume prune -f >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
docker container prune -f >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
if [[ "${INFINITO_PRESERVE_DOCKER_CACHE}" != "true" ]]; then
|
|
docker image prune -af >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
docker buildx prune -af >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
docker builder prune -af >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
else
|
|
if [[ -n "${INFINITO_IMAGE:-}" ]]; then
|
|
_image_repo="${INFINITO_IMAGE%%:*}"
|
|
mapfile -t _old_ci_images < <(
|
|
docker images --format "{{.Repository}}:{{.Tag}}" |
|
|
grep "^${_image_repo}:ci-" |
|
|
grep -vxF "${INFINITO_IMAGE}" ||
|
|
true
|
|
)
|
|
if ((${#_old_ci_images[@]} > 0)); then
|
|
echo ">>> Pruning ${#_old_ci_images[@]} stale CI image(s) for ${_image_repo}"
|
|
docker rmi "${_old_ci_images[@]}" >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
if [[ "${INFINITO_PRESERVE_DOCKER_CACHE}" == "true" ]]; then
|
|
echo ">>> INFINITO_PRESERVE_DOCKER_CACHE=true — keeping Docker root for next distro: ${INFINITO_DOCKER_VOLUME}"
|
|
elif [[ -n "${INFINITO_DOCKER_VOLUME:-}" ]]; then
|
|
if [[ "${INFINITO_DOCKER_VOLUME}" == /* ]]; then
|
|
echo ">>> CI cleanup: wiping Docker root: ${INFINITO_DOCKER_VOLUME}"
|
|
|
|
echo ">>> Pre-clean ownership/permissions (best-effort)"
|
|
ls -ld "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo ">>> Removing host docker volume dir: ${INFINITO_DOCKER_VOLUME}"
|
|
sudo rm -rf "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
sudo mkdir -vp "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
sudo chown -R "$(id -u):$(id -g)" "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo ">>> Post-clean ownership/permissions (best-effort)"
|
|
ls -ld "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
else
|
|
echo "[WARN] INFINITO_DOCKER_VOLUME is not an absolute path: '${INFINITO_DOCKER_VOLUME}' (skipping)"
|
|
fi
|
|
fi
|
|
|
|
echo ">>> Removing root-owned Python bytecode from workspace"
|
|
sudo find "${REPO_ROOT}" -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
sudo find "${REPO_ROOT}" -name "*.pyc" -delete 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo ">>> Docker disk usage after HARD cleanup"
|
|
docker system df || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
echo ">>> HARD cleanup finished"
|
|
return $rc
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
if [[ "${INFINITO_PRESERVE_DOCKER_CACHE}" == "true" ]]; then
|
|
_disk_pct=$(df --output=pcent / | tail -1 | tr -d ' %')
|
|
if [[ "${_disk_pct}" -ge 70 && -n "${INFINITO_DOCKER_VOLUME:-}" && "${INFINITO_DOCKER_VOLUME}" == /* ]]; then
|
|
echo ">>> Disk at ${_disk_pct}% — wiping full inner-Docker volume to reclaim space: ${INFINITO_DOCKER_VOLUME}"
|
|
sudo rm -rf "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
sudo mkdir -p "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
sudo chown -R "$(id -u):$(id -g)" "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
else
|
|
echo ">>> Wiping inner-Docker volumes and container state: ${INFINITO_DOCKER_VOLUME}"
|
|
sudo rm -rf "${INFINITO_DOCKER_VOLUME}/volumes" "${INFINITO_DOCKER_VOLUME}/containers" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
fi
|
|
fi
|
|
|
|
echo ">>> Ensuring stack is up for distro ${INFINITO_DISTRO}"
|
|
"${PYTHON}" -m cli.administration.deploy.development up
|
|
|
|
_up_container="${INFINITO_CONTAINER:?INFINITO_CONTAINER is not set (run make dotenv)}"
|
|
docker exec "${_up_container}" install -m 755 \
|
|
/opt/src/infinito/roles/sys-ca-selfsigned/files/shell/with-ca-trust.sh \
|
|
/usr/bin/ca-trust-wrapper 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo ">>> Starting the :53 sampler inside ${_up_container}"
|
|
docker exec -d \
|
|
-e "INFINITO_DNS53_SAMPLER_LOG=${INFINITO_DNS53_SAMPLER_LOG:?INFINITO_DNS53_SAMPLER_LOG must be set (SPOT: group_vars/all/05_paths.yml FILE_DNS53_SAMPLER_LOG); regenerate .env}" \
|
|
"${_up_container}" \
|
|
sh /opt/src/infinito/scripts/tests/deploy/ci/dns53-sampler.sh
|
|
|
|
deploy_args=(
|
|
--apps "${apps}"
|
|
--inventory-dir "${INFINITO_INVENTORY_DIR}"
|
|
--debug
|
|
)
|
|
|
|
echo ">>> DISK / DOCKER STATE BEFORE DEPLOY (distro=${INFINITO_DISTRO})"
|
|
df -h || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
docker system df || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
echo ">>> END STATE BEFORE DEPLOY"
|
|
|
|
_init_args=(
|
|
--apps "${apps}"
|
|
--inventory-dir "${INFINITO_INVENTORY_DIR}"
|
|
)
|
|
if [[ "${INFINITO_PRESERVE_DOCKER_CACHE}" == "true" ]]; then
|
|
_init_args+=(--force-storage-constrained false)
|
|
fi
|
|
|
|
if [[ "${INFINITO_TIMEOUT_MULTIPLIER}" -gt 1 ]]; then
|
|
echo ">>> Scaling Ansible retries by ${INFINITO_TIMEOUT_MULTIPLIER}x (slow hardware detected)"
|
|
docker exec \
|
|
-e "INFINITO_TIMEOUT_MULTIPLIER=${INFINITO_TIMEOUT_MULTIPLIER}" \
|
|
-e "INFINITO_REPO_ROOT=/opt/src/infinito" \
|
|
"${_up_container}" \
|
|
bash /opt/src/infinito/scripts/tests/deploy/ci/multiply-timeouts.sh
|
|
fi
|
|
|
|
echo ">>> init inventory (ASYNC_ENABLED=false baked into host_vars)"
|
|
"${PYTHON}" -m cli.administration.deploy.development init \
|
|
"${_init_args[@]}" \
|
|
--vars '{"ASYNC_ENABLED": false}'
|
|
|
|
echo ">>> deploy (PASS 1 sync + PASS 2 async per variant, --full-cycle)"
|
|
"${PYTHON}" -m cli.administration.deploy.development deploy "${deploy_args[@]}" --full-cycle
|
|
|
|
echo ">>> DISK / DOCKER STATE AFTER DEPLOY (before cleanup, distro=${INFINITO_DISTRO})"
|
|
df -h || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
docker system df || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
echo ">>> END STATE AFTER DEPLOY"
|