mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-13 06:06:38 +00:00
evangelostsak
b819931385
The role stays inert on the single CI host, which holds no mesh, and its CLI test runs at the end of the deploy.
256 lines
13 KiB
Bash
Executable File
256 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Deploy one app on one distro: bring the stack up, init the inventory (one folder
|
|
# per meta/variants.yml round), deploy --full-cycle (sync + async per variant), then
|
|
# always remove the stack so the next distro starts fresh.
|
|
#
|
|
# Required env:
|
|
# INFINITO_DISTRO="arch|debian|ubuntu|fedora|centos"
|
|
# INFINITO_INVENTORY_DIR="/path/to/inventory"
|
|
#
|
|
# Optional env:
|
|
# PYTHON="python3"
|
|
# variant="<csv>" pin to one or more matrix rounds, e.g. "2" or "0,1,2"
|
|
# (the runner-split bundle from CI discovery); empty = all
|
|
# INFINITO_COMPOSE_VPN="true" also deploy svc-net-wireguard, so its CLI test
|
|
# meshes three throwaway hosts after the app
|
|
|
|
: "${INFINITO_DISTRO:?INFINITO_DISTRO must be set (e.g. arch)}"
|
|
: "${INFINITO_INVENTORY_DIR:?INFINITO_INVENTORY_DIR must be set}"
|
|
: "${INFINITO_DOCKER_VOLUME:?INFINITO_DOCKER_VOLUME must be set}"
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
|
|
# shellcheck source=/dev/null
|
|
source <(grep -E '^INFINITO_(PLAYWRIGHT_REPORTS_BASE_DIR|RESCUE_DIAGNOSTICS_DIR|RESCUE_DIAGNOSTICS_BASE|DNS53_SAMPLER_LOG)=' "${REPO_ROOT}/.env")
|
|
: "${INFINITO_RESCUE_DIAGNOSTICS_BASE:?INFINITO_RESCUE_DIAGNOSTICS_BASE must be set}"
|
|
|
|
apps=""
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage:
|
|
INFINITO_DISTRO=<distro> INFINITO_INVENTORY_DIR=<dir> INFINITO_DOCKER_VOLUME=<abs_path> \
|
|
scripts/tests/deploy/ci/dedicated.sh \ # nocheck: self-path-reference
|
|
--apps <app_ids>
|
|
EOF
|
|
}
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--apps)
|
|
apps="${2:-}"
|
|
shift 2
|
|
;;
|
|
-h | --help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "[ERROR] Unknown argument: $1" >&2
|
|
usage
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
[[ -n "${apps}" ]] || {
|
|
echo "[ERROR] --apps is required" >&2
|
|
usage
|
|
exit 2
|
|
}
|
|
|
|
cd "${REPO_ROOT}"
|
|
|
|
echo "=== distro=${INFINITO_DISTRO} app=${apps} (debug always on) ==="
|
|
|
|
cleanup() {
|
|
rc=$?
|
|
|
|
local _playwright_host_dir="/tmp/playwright-artifacts/${INFINITO_DISTRO}/${apps}"
|
|
mkdir -p "${_playwright_host_dir}"
|
|
echo ">>> Copying Playwright artifacts from ${INFINITO_CONTAINER} to ${_playwright_host_dir}"
|
|
# nocheck: container-cp - container-to-host extraction on the CI host itself
|
|
docker cp "${INFINITO_CONTAINER}:${INFINITO_PLAYWRIGHT_REPORTS_BASE_DIR}/." \
|
|
"${_playwright_host_dir}" 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
local _rescue_host_dir="${INFINITO_RESCUE_DIAGNOSTICS_BASE}/${INFINITO_DISTRO}/${apps}"
|
|
mkdir -p "${_rescue_host_dir}"
|
|
echo ">>> Capturing rescue diagnostics inside ${INFINITO_CONTAINER} (recursive DiD snapshot) before teardown removes it"
|
|
timeout 1500 docker exec \
|
|
-e "INFINITO_RESCUE_DIAGNOSTICS_DIR=${INFINITO_RESCUE_DIAGNOSTICS_DIR}" \
|
|
-e "INFINITO_DNS53_SAMPLER_LOG=${INFINITO_DNS53_SAMPLER_LOG}" \
|
|
"${INFINITO_CONTAINER}" \
|
|
python3 /opt/src/infinito/utils/diagnostics/container.py \
|
|
"${apps}" "compose post-deploy failure" 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
echo ">>> Copying rescue diagnostics from ${INFINITO_CONTAINER} to ${_rescue_host_dir}"
|
|
docker exec "${INFINITO_CONTAINER}" \
|
|
tar -C "${INFINITO_RESCUE_DIAGNOSTICS_DIR}" -cf - . 2>/dev/null |
|
|
tar -C "${_rescue_host_dir}" -xf - 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
local _did_id
|
|
_did_id="$(docker inspect --format '{{.Id}}' "${INFINITO_CONTAINER}" 2>/dev/null)" || _did_id=""
|
|
echo ">>> Capturing rescue diagnostics on the CI host itself into ${_rescue_host_dir}/ci-host"
|
|
INFINITO_RESCUE_DIAGNOSTICS_DIR="${_rescue_host_dir}/ci-host" \
|
|
RESCUE_SEEN="${_did_id}" \
|
|
timeout 900 python3 "${REPO_ROOT}/utils/diagnostics/container.py" \
|
|
"${apps}" "ci host at compose post-deploy failure" 2>/dev/null || true # nocheck: shell-or-true -- the collector exits 1 by contract so the caller cannot mistake a snapshot for a passing deploy
|
|
|
|
bash scripts/tests/deploy/utils/rescue_index.sh "${_rescue_host_dir}"
|
|
|
|
local _inv_parent
|
|
_inv_parent="$(dirname "${INFINITO_INVENTORY_DIR}")"
|
|
echo ">>> Copying generated inventory from ${INFINITO_CONTAINER} to host ${_inv_parent}"
|
|
mkdir -p "${_inv_parent}"
|
|
# nocheck: container-cp - container-to-host extraction on the CI host itself
|
|
docker cp "${INFINITO_CONTAINER}:${_inv_parent}/." "${_inv_parent}/" 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo ">>> Removing stack for distro ${INFINITO_DISTRO} (fresh start for next distro)"
|
|
"${PYTHON}" -m cli.administration.deploy.development down || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo ">>> HARD cleanup (containers/volumes/networks/images/build-cache)"
|
|
echo ">>> Docker disk usage before HARD cleanup"
|
|
docker system df || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
_cleanup_project="${COMPOSE_PROJECT_NAME:-}"
|
|
if [[ -n "${_cleanup_project}" ]]; then
|
|
mapfile -t ids < <(docker ps -aq --filter "label=com.docker.compose.project=${_cleanup_project}" || true)
|
|
else
|
|
mapfile -t ids < <(docker ps -aq || true)
|
|
fi
|
|
if ((${#ids[@]} > 0)); then
|
|
docker rm -f "${ids[@]}" >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
fi
|
|
|
|
docker network prune -f >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
docker volume prune -f >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
docker container prune -f >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
if [[ "${INFINITO_PRESERVE_DOCKER_CACHE}" != "true" ]]; then
|
|
docker image prune -af >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
docker buildx prune -af >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
docker builder prune -af >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
else
|
|
if [[ -n "${INFINITO_IMAGE:-}" ]]; then
|
|
_image_repo="${INFINITO_IMAGE%%:*}"
|
|
mapfile -t _old_ci_images < <(
|
|
docker images --format "{{.Repository}}:{{.Tag}}" |
|
|
grep "^${_image_repo}:ci-" |
|
|
grep -vxF "${INFINITO_IMAGE}" ||
|
|
true
|
|
)
|
|
if ((${#_old_ci_images[@]} > 0)); then
|
|
echo ">>> Pruning ${#_old_ci_images[@]} stale CI image(s) for ${_image_repo}"
|
|
docker rmi "${_old_ci_images[@]}" >/dev/null 2>&1 || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
if [[ "${INFINITO_PRESERVE_DOCKER_CACHE}" == "true" ]]; then
|
|
echo ">>> INFINITO_PRESERVE_DOCKER_CACHE=true — keeping Docker root for next distro: ${INFINITO_DOCKER_VOLUME}"
|
|
elif [[ -n "${INFINITO_DOCKER_VOLUME:-}" ]]; then
|
|
if [[ "${INFINITO_DOCKER_VOLUME}" == /* ]]; then
|
|
echo ">>> CI cleanup: wiping Docker root: ${INFINITO_DOCKER_VOLUME}"
|
|
|
|
echo ">>> Pre-clean ownership/permissions (best-effort)"
|
|
ls -ld "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo ">>> Removing host docker volume dir: ${INFINITO_DOCKER_VOLUME}"
|
|
sudo rm -rf "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
sudo mkdir -vp "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
sudo chown -R "$(id -u):$(id -g)" "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo ">>> Post-clean ownership/permissions (best-effort)"
|
|
ls -ld "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
else
|
|
echo "[WARN] INFINITO_DOCKER_VOLUME is not an absolute path: '${INFINITO_DOCKER_VOLUME}' (skipping)"
|
|
fi
|
|
fi
|
|
|
|
echo ">>> Removing root-owned Python bytecode from workspace"
|
|
sudo find "${REPO_ROOT}" -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
sudo find "${REPO_ROOT}" -name "*.pyc" -delete 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo ">>> Docker disk usage after HARD cleanup"
|
|
docker system df || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
echo ">>> HARD cleanup finished"
|
|
return $rc
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
if [[ "${INFINITO_PRESERVE_DOCKER_CACHE}" == "true" ]]; then
|
|
_disk_pct=$(df --output=pcent / | tail -1 | tr -d ' %')
|
|
if [[ "${_disk_pct}" -ge 70 && -n "${INFINITO_DOCKER_VOLUME:-}" && "${INFINITO_DOCKER_VOLUME}" == /* ]]; then
|
|
echo ">>> Disk at ${_disk_pct}% — wiping full inner-Docker volume to reclaim space: ${INFINITO_DOCKER_VOLUME}"
|
|
sudo rm -rf "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
sudo mkdir -p "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
sudo chown -R "$(id -u):$(id -g)" "${INFINITO_DOCKER_VOLUME}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
else
|
|
echo ">>> Wiping inner-Docker volumes and container state: ${INFINITO_DOCKER_VOLUME}"
|
|
sudo rm -rf "${INFINITO_DOCKER_VOLUME}/volumes" "${INFINITO_DOCKER_VOLUME}/containers" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
fi
|
|
fi
|
|
|
|
echo ">>> Ensuring stack is up for distro ${INFINITO_DISTRO}"
|
|
"${PYTHON}" -m cli.administration.deploy.development up
|
|
|
|
_up_container="${INFINITO_CONTAINER:?INFINITO_CONTAINER is not set (run make dotenv)}"
|
|
docker exec "${_up_container}" install -m 755 \
|
|
/opt/src/infinito/roles/sys-ca-selfsigned/files/shell/with-ca-trust.sh \
|
|
/usr/bin/ca-trust-wrapper 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
|
|
echo ">>> Starting the :53 sampler inside ${_up_container}"
|
|
docker exec -d \
|
|
-e "INFINITO_DNS53_SAMPLER_LOG=${INFINITO_DNS53_SAMPLER_LOG:?INFINITO_DNS53_SAMPLER_LOG must be set (SPOT: group_vars/all/05_paths.yml FILE_DNS53_SAMPLER_LOG); regenerate .env}" \
|
|
"${_up_container}" \
|
|
sh /opt/src/infinito/scripts/tests/deploy/ci/dns53-sampler.sh
|
|
|
|
deploy_apps="${apps}"
|
|
if [[ "${INFINITO_COMPOSE_VPN:-}" == "true" ]]; then
|
|
deploy_apps="${apps} svc-net-wireguard"
|
|
fi
|
|
|
|
deploy_args=(
|
|
--apps "${deploy_apps}"
|
|
--inventory-dir "${INFINITO_INVENTORY_DIR}"
|
|
--debug
|
|
)
|
|
|
|
echo ">>> DISK / DOCKER STATE BEFORE DEPLOY (distro=${INFINITO_DISTRO})"
|
|
df -h || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
docker system df || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
echo ">>> END STATE BEFORE DEPLOY"
|
|
|
|
_init_args=(
|
|
--apps "${deploy_apps}"
|
|
--inventory-dir "${INFINITO_INVENTORY_DIR}"
|
|
)
|
|
if [[ "${INFINITO_PRESERVE_DOCKER_CACHE}" == "true" ]]; then
|
|
_init_args+=(--force-storage-constrained false)
|
|
fi
|
|
|
|
if [[ "${INFINITO_TIMEOUT_MULTIPLIER}" -gt 1 ]]; then
|
|
echo ">>> Scaling Ansible retries by ${INFINITO_TIMEOUT_MULTIPLIER}x (slow hardware detected)"
|
|
docker exec \
|
|
-e "INFINITO_TIMEOUT_MULTIPLIER=${INFINITO_TIMEOUT_MULTIPLIER}" \
|
|
-e "INFINITO_REPO_ROOT=/opt/src/infinito" \
|
|
"${_up_container}" \
|
|
bash /opt/src/infinito/scripts/tests/deploy/ci/multiply-timeouts.sh
|
|
fi
|
|
|
|
echo ">>> init inventory (ASYNC_ENABLED=false baked into host_vars)"
|
|
"${PYTHON}" -m cli.administration.deploy.development init \
|
|
"${_init_args[@]}" \
|
|
--vars '{"ASYNC_ENABLED": false}'
|
|
|
|
echo ">>> deploy (PASS 1 sync + PASS 2 async per variant, --full-cycle)"
|
|
"${PYTHON}" -m cli.administration.deploy.development deploy "${deploy_args[@]}" --full-cycle
|
|
|
|
echo ">>> DISK / DOCKER STATE AFTER DEPLOY (before cleanup, distro=${INFINITO_DISTRO})"
|
|
df -h || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
docker system df || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
|
echo ">>> END STATE AFTER DEPLOY"
|