docker-volume-backup/scripts/test-e2e.sh

#!/usr/bin/env bash
set -euo pipefail

# -----------------------------------------------------------------------------
# E2E runner using Docker-in-Docker (DinD) with debug-on-failure
#
# Debug toggles:
#   E2E_KEEP_ON_FAIL=1   -> keep DinD + volumes + network if tests fail
#   E2E_KEEP_VOLUMES=1   -> keep volumes even on success/cleanup
#   E2E_DEBUG_SHELL=1    -> open an interactive shell in the test container instead of running tests
#   E2E_ARTIFACTS_DIR=./artifacts
# -----------------------------------------------------------------------------

NET="${E2E_NET:-baudolo-e2e-net}"
DIND="${E2E_DIND_NAME:-baudolo-e2e-dind}"
DIND_VOL="${E2E_DIND_VOL:-baudolo-e2e-dind-data}"
E2E_TMP_VOL="${E2E_TMP_VOL:-baudolo-e2e-tmp}"

# Host-side access to the DinD daemon goes through `docker exec` (dind()
# below) instead of a host-published port: port publishing is not reachable
# from every environment (sandboxed runners, hosts with broken loopback
# publishing), while exec only needs the outer docker socket. The TCP
# listener stays for the test container inside the dedicated network.
DIND_HOST_IN_NET="${E2E_DIND_HOST_IN_NET:-tcp://${DIND}:2375}"

dind() { docker exec "${DIND}" docker "$@"; }
dind_stdin() { docker exec -i "${DIND}" docker "$@"; }

IMG="${E2E_IMAGE:-baudolo:local}"

READY_TIMEOUT_SECONDS="${E2E_READY_TIMEOUT_SECONDS:-120}"
ARTIFACTS_DIR="${E2E_ARTIFACTS_DIR:-./artifacts}"

DIND_MTU="${E2E_DIND_MTU:-1280}"

KEEP_ON_FAIL="${E2E_KEEP_ON_FAIL:-0}"
KEEP_VOLUMES="${E2E_KEEP_VOLUMES:-0}"
DEBUG_SHELL="${E2E_DEBUG_SHELL:-0}"

# Override to run a subset, e.g. E2E_TEST_PATTERN=test_e2e_postgres_empty_drop_hard.py
TEST_PATTERN="${E2E_TEST_PATTERN:-test_*.py}"

FAILED=0
TS="$(date +%Y%m%d%H%M%S)"

mkdir -p "${ARTIFACTS_DIR}"

log() { echo ">> $*"; }

dump_debug() {
  log "DEBUG: collecting diagnostics into ${ARTIFACTS_DIR}"

  {
    echo "=== Host docker version ==="
    docker version || true
    echo
    echo "=== Host docker info ==="
    docker info || true
    echo
    echo "=== DinD reachable? (docker exec ${DIND} docker version) ==="
    dind version || true
    echo
  } > "${ARTIFACTS_DIR}/debug-host-${TS}.txt" 2>&1 || true

  # DinD logs
  docker logs --tail=5000 "${DIND}" > "${ARTIFACTS_DIR}/dind-logs-${TS}.txt" 2>&1 || true

  # DinD state
  {
    echo "=== dind ps -a ==="
    dind ps -a || true
    echo
    echo "=== dind images ==="
    dind images || true
    echo
    echo "=== dind network ls ==="
    dind network ls || true
    echo
    echo "=== dind volume ls ==="
    dind volume ls || true
    echo
    echo "=== dind system df ==="
    dind system df || true
  } > "${ARTIFACTS_DIR}/debug-dind-${TS}.txt" 2>&1 || true

  # Try to capture recent events (best effort; might be noisy)
  dind events --since 10m --until 0s \
    > "${ARTIFACTS_DIR}/dind-events-${TS}.txt" 2>&1 || true

  # The shared tmp volume is mounted at /tmp inside the DinD container
  # itself, so tar it there and copy it out with the outer daemon.
  log "DEBUG: archiving shared /tmp (volume ${E2E_TMP_VOL})"
  docker exec "${DIND}" tar -czf "/tmpdump-${TS}.tar.gz" -C /tmp . >/dev/null 2>&1 || true
  docker cp "${DIND}:/tmpdump-${TS}.tar.gz" "${ARTIFACTS_DIR}/e2e-tmp-${TS}.tar.gz" >/dev/null 2>&1 || true

  log "DEBUG: artifacts written:"
  find "${ARTIFACTS_DIR}" -maxdepth 1 -mindepth 1 -print | sed 's/^/   /' || true
}

cleanup() {
  if [ "${FAILED}" -eq 1 ] && [ "${KEEP_ON_FAIL}" = "1" ]; then
    log "KEEP_ON_FAIL=1 and failure detected -> skipping cleanup."
    log "Next steps:"
    echo "   - Inspect DinD logs: docker logs ${DIND} | less"
    echo "   - Use DinD daemon:  docker exec ${DIND} docker ps -a"
    echo "   - Shared tmp vol:   docker exec ${DIND} ls -la /tmp"
    echo "   - DinD docker root: docker exec ${DIND} ls -la /var/lib/docker/volumes"
    return 0
  fi

  log "Cleanup: stopping ${DIND} and removing network ${NET}"
  docker rm -f "${DIND}" >/dev/null 2>&1 || true
  docker network rm "${NET}" >/dev/null 2>&1 || true

  if [ "${KEEP_VOLUMES}" != "1" ]; then
    docker volume rm -f "${DIND_VOL}" >/dev/null 2>&1 || true
    docker volume rm -f "${E2E_TMP_VOL}" >/dev/null 2>&1 || true
  else
    log "Keeping volumes (E2E_KEEP_VOLUMES=1): ${DIND_VOL}, ${E2E_TMP_VOL}"
  fi
}
trap cleanup EXIT INT TERM

log "(Re)creating network ${NET} with MTU ${DIND_MTU}"
docker network rm "${NET}" >/dev/null 2>&1 || true
docker network create \
  --opt com.docker.network.driver.mtu="${DIND_MTU}" \
  "${NET}" >/dev/null

log "Removing old ${DIND} (if any)"
docker rm -f "${DIND}" >/dev/null 2>&1 || true

log "(Re)creating DinD data volume ${DIND_VOL}"
docker volume rm -f "${DIND_VOL}" >/dev/null 2>&1 || true
docker volume create "${DIND_VOL}" >/dev/null

log "(Re)creating shared /tmp volume ${E2E_TMP_VOL}"
docker volume rm -f "${E2E_TMP_VOL}" >/dev/null 2>&1 || true
docker volume create "${E2E_TMP_VOL}" >/dev/null

log "Starting Docker-in-Docker daemon ${DIND}"
docker run -d --privileged \
  --name "${DIND}" \
  --network "${NET}" \
  -e DOCKER_TLS_CERTDIR="" \
  -v "${DIND_VOL}:/var/lib/docker" \
  -v "${E2E_TMP_VOL}:/tmp" \
  docker:dind \
  --host=tcp://0.0.0.0:2375 \
  --tls=false \
  --mtu="${DIND_MTU}" >/dev/null

log "Waiting for DinD to be ready..."
for i in $(seq 1 "${READY_TIMEOUT_SECONDS}"); do
  if dind version >/dev/null 2>&1; then
    log "DinD is ready."
    break
  fi
  sleep 1
  if [ "${i}" -eq "${READY_TIMEOUT_SECONDS}" ]; then
    echo "ERROR: DinD did not become ready in time"
    docker logs --tail=200 "${DIND}" || true
    FAILED=1
    dump_debug || true
    exit 1
  fi
done

log "Ensuring alpine exists in DinD (for debug helpers)"
dind pull alpine:3.20 >/dev/null

log "Loading ${IMG} image into DinD..."
docker save "${IMG}" | dind_stdin load >/dev/null

log "Running E2E tests inside DinD"
set +e
if [ "${DEBUG_SHELL}" = "1" ]; then
  log "E2E_DEBUG_SHELL=1 -> opening shell in test container"
  docker run --rm -it \
    --network "${NET}" \
    -e DOCKER_HOST="${DIND_HOST_IN_NET}" \
    -v "${DIND_VOL}:/var/lib/docker" \
    -v "${E2E_TMP_VOL}:/tmp" \
    "${IMG}" \
    bash -lc '
      set -e
      if [ ! -f /etc/machine-id ]; then
        mkdir -p /etc
        cat /proc/sys/kernel/random/uuid > /etc/machine-id
      fi
      echo ">> DOCKER_HOST=${DOCKER_HOST}"
      docker ps -a || true
      exec bash
    '
  rc=$?
else
  docker run --rm \
    --network "${NET}" \
    -e DOCKER_HOST="${DIND_HOST_IN_NET}" \
    -e E2E_TEST_PATTERN="${TEST_PATTERN}" \
    -v "${DIND_VOL}:/var/lib/docker" \
    -v "${E2E_TMP_VOL}:/tmp" \
    "${IMG}" \
    bash -lc '
    set -euo pipefail
    set -x
    export PYTHONUNBUFFERED=1

    export TMPDIR=/tmp TMP=/tmp TEMP=/tmp

    if [ ! -f /etc/machine-id ]; then
        mkdir -p /etc
        cat /proc/sys/kernel/random/uuid > /etc/machine-id
    fi

    python -m unittest discover -t . -s tests/e2e -p "${E2E_TEST_PATTERN}" -v -f
    '
  rc=$?
fi
set -e

if [ "${rc}" -ne 0 ]; then
  FAILED=1
  echo "ERROR: E2E tests failed (exit code: ${rc})"
  dump_debug || true
  exit "${rc}"
fi

log "E2E tests passed."