diff --git a/roles/svc-bkp-volume-2-local/files/test/db_probe.sh b/roles/svc-bkp-volume-2-local/files/test/db_probe.sh new file mode 100755 index 0000000000..e65086416e --- /dev/null +++ b/roles/svc-bkp-volume-2-local/files/test/db_probe.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${BKP_TEST_DATABASES_CSV:?}" + +MODE="${1:?usage: db_probe.sh seed TOKEN | verify PRE_TOKEN POST_TOKEN MANIFEST}" +PROBE_TABLE="infinito_backup_restore_probe" + +validate_token() { + if [[ ! "${1}" =~ ^[A-Za-z0-9_-]+$ ]]; then + echo "FAIL: unsafe database probe token" + exit 1 + fi +} + +engine_for_container() { + local image + image="$(container inspect -f '{{.Config.Image}}' "${1}")" + case "${image}" in + *postgres* | *postgis*) printf 'postgres\n' ;; + *mariadb* | *mysql*) printf 'mariadb\n' ;; + *) return 1 ;; + esac +} + +container_for_instance() { + local candidate engine instance="${1}" + mapfile -t candidates < <(container ps --filter "volume=${instance}_data" --format '{{.Names}}') + if ((${#candidates[@]} < 1)); then + mapfile -t candidates < <(container ps --filter "name=${instance}" --format '{{.Names}}') + fi + for candidate in "${candidates[@]}"; do + if engine="$(engine_for_container "${candidate}")"; then + printf '%s;%s\n' "${candidate}" "${engine}" + return 0 + fi + done + return 1 +} + +container_for_volume() { + local candidate engine volume="${1}" expected_engine="${2}" + while IFS= read -r candidate; do + [[ -n "${candidate}" ]] || continue + if engine="$(engine_for_container "${candidate}")" && [[ "${engine}" == "${expected_engine}" ]]; then + printf '%s\n' "${candidate}" + return 0 + fi + done < <(container ps --filter "volume=${volume}" --format '{{.Names}}') + return 1 +} + +run_seed() { + local container database engine instance password resolved token="${1}" user + local seeded=0 + validate_token "${token}" + if [[ ! -f "${BKP_TEST_DATABASES_CSV}" ]]; then + echo "FAIL: databases csv missing at ${BKP_TEST_DATABASES_CSV}" + exit 1 + fi + + while IFS=';' read -r instance database user password; do + [[ -n "${instance}" ]] || continue + [[ "${database}" != "*" ]] || continue + if ! resolved="$(container_for_instance "${instance}")"; then + echo "SKIP: no running database container found for instance '${instance}'" + continue + fi + IFS=';' read -r container engine <<<"${resolved}" + case "${engine}" in + postgres) + container exec -e "PGPASSWORD=${password}" "${container}" \ + psql -X -v ON_ERROR_STOP=1 -U "${user}" -d "${database}" -c \ + "CREATE TABLE IF NOT EXISTS ${PROBE_TABLE} (token text PRIMARY KEY); INSERT INTO ${PROBE_TABLE} (token) VALUES ('${token}') ON CONFLICT (token) DO NOTHING;" >/dev/null + ;; + mariadb) + container exec -e "MYSQL_PWD=${password}" "${container}" \ + mariadb --batch --skip-column-names -u "${user}" "${database}" -e \ + "CREATE TABLE IF NOT EXISTS ${PROBE_TABLE} (token varchar(128) PRIMARY KEY); INSERT IGNORE INTO ${PROBE_TABLE} (token) VALUES ('${token}');" >/dev/null + ;; + esac + echo "OK: seeded database restore probe in ${database} (${engine})" + seeded=$((seeded + 1)) + done < <(tail -n +2 "${BKP_TEST_DATABASES_CSV}") + echo "OK: seeded ${seeded} database restore probe(s)" +} + +probe_count() { + local container="${1}" database="${2}" engine="${3}" password="${4}" token="${5}" user="${6}" + case "${engine}" in + postgres) + container exec -e "PGPASSWORD=${password}" "${container}" \ + psql -X -Atq -v ON_ERROR_STOP=1 -U "${user}" -d "${database}" -c \ + "SELECT COUNT(*) FROM ${PROBE_TABLE} WHERE token = '${token}';" | tr -d '[:space:]' + ;; + mariadb) + container exec -e "MYSQL_PWD=${password}" "${container}" \ + mariadb --batch --skip-column-names -u "${user}" "${database}" -e \ + "SELECT COUNT(*) FROM ${PROBE_TABLE} WHERE token = '${token}';" | tr -d '[:space:]' + ;; + *) + echo "FAIL: unsupported database engine '${engine}'" + exit 1 + ;; + esac +} + +run_verify() { + local container database engine manifest="${3}" password post_count pre_count + local pre_token="${1}" post_token="${2}" row user verified=0 volume + validate_token "${pre_token}" + validate_token "${post_token}" + if [[ ! -s "${manifest}" ]]; then + echo "FAIL: restored database manifest missing or empty at ${manifest}" + exit 1 + fi + + while IFS=';' read -r engine volume database; do + [[ -n "${engine}" && -n "${volume}" && -n "${database}" ]] || continue + row="$(awk -F';' -v db="${database}" 'NR > 1 && $2 == db { print; exit }' "${BKP_TEST_DATABASES_CSV}")" + if [[ -z "${row}" ]]; then + echo "FAIL: no databases.csv row for restored database '${database}'" + exit 1 + fi + IFS=';' read -r _instance _database user password <<<"${row}" + if ! container="$(container_for_volume "${volume}" "${engine}")"; then + echo "FAIL: no running ${engine} container mounts restored volume ${volume}" + exit 1 + fi + pre_count="$(probe_count "${container}" "${database}" "${engine}" "${password}" "${pre_token}" "${user}")" + post_count="$(probe_count "${container}" "${database}" "${engine}" "${password}" "${post_token}" "${user}")" + if [[ "${pre_count}" != "1" ]] || [[ "${post_count}" != "0" ]]; then + echo "FAIL: ${database} restore probe mismatch (pre=${pre_count}, post=${post_count})" + exit 1 + fi + echo "OK: ${database} contains the pre-backup probe and excludes the post-backup probe" + verified=$((verified + 1)) + done <"${manifest}" + echo "OK: verified ${verified} destructively restored database(s)" +} + +case "${MODE}" in +seed) + run_seed "${2:?seed token required}" + ;; +verify) + run_verify "${2:?pre token required}" "${3:?post token required}" "${4:?manifest required}" + ;; +*) + echo "FAIL: unsupported db_probe mode '${MODE}'" + exit 1 + ;; +esac diff --git a/roles/svc-bkp-volume-2-local/files/test/db_restore.sh b/roles/svc-bkp-volume-2-local/files/test/db_restore.sh index dbccad12ce..6707639384 100755 --- a/roles/svc-bkp-volume-2-local/files/test/db_restore.sh +++ b/roles/svc-bkp-volume-2-local/files/test/db_restore.sh @@ -16,6 +16,11 @@ set -euo pipefail : "${NEWEST_GENERATION:?}" GEN_DIR="${REPO_DIR}/${NEWEST_GENERATION}" +RESTORED_DATABASES_FILE="${BKP_TEST_RESTORED_DATABASES_FILE:-}" + +if [[ -n "${RESTORED_DATABASES_FILE}" ]]; then + : >"${RESTORED_DATABASES_FILE}" +fi if [[ ! -f "${BKP_TEST_DATABASES_CSV}" ]]; then echo "FAIL: databases csv missing at ${BKP_TEST_DATABASES_CSV}" @@ -71,6 +76,9 @@ for sql_file in "${SQL_FILES[@]}"; do --db-password "${db_password}" \ --empty echo "OK: replayed ${engine} dump '${db_name}' into ${container}" + if [[ -n "${RESTORED_DATABASES_FILE}" ]]; then + printf '%s;%s;%s\n' "${engine}" "${volume}" "${db_name}" >>"${RESTORED_DATABASES_FILE}" + fi RESTORED=$((RESTORED + 1)) done diff --git a/roles/svc-bkp-volume-2-local/files/test/test.sh b/roles/svc-bkp-volume-2-local/files/test/test.sh index 1266cc6a9d..fcb2f23128 100755 --- a/roles/svc-bkp-volume-2-local/files/test/test.sh +++ b/roles/svc-bkp-volume-2-local/files/test/test.sh @@ -1,14 +1,4 @@ #!/usr/bin/env bash -# E2E orchestrator for svc-bkp-volume-2-local. -# Waits for the backup service to terminate, then: -# sync pass (ASYNC_ENABLED=false): verify the first backup generation is -# stored, then run the destructive restore drill (stop containers, wipe -# the backed-up volumes, restore, require everything healthy again) and -# replay the sql dumps. -# async pass (ASYNC_ENABLED=true): require two differential backup -# generations, triggering one extra service run when only one exists; -# no destructive steps. -# Variables sourced from test.env.j2 by test-e2e-cli. set -euo pipefail : "${ASYNC_ENABLED:?}" @@ -17,6 +7,7 @@ set -euo pipefail : "${BKP_TEST_SERVICE:?}" : "${BKP_TEST_RESTORE_BIN:?}" : "${BKP_TEST_HEALTH_TIMEOUT:?}" +: "${BKP_TEST_SWARM_DRILL:?}" if [[ "${BKP_TEST_IS_STACK_HOST}" != "true" ]]; then echo "SKIP: not the stack host; svc-bkp-volume-2-local only deploys there" @@ -57,13 +48,35 @@ count_generations() { wait_service_terminated +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SWARM_ACTIVE=false +if [[ "$(container info --format '{{.Swarm.LocalNodeState}}' 2>/dev/null)" == "active" ]]; then + SWARM_ACTIVE=true +fi + +MACHINE_HASH="$(sha256sum /etc/machine-id | cut -c1-64)" +HANDOFF_PENDING="${DIR}/swarm-restore.pending" +HANDOFF_COMPLETE="${DIR}/swarm-restore.complete" +HANDOFF_MANIFEST="${DIR}/swarm-restore.manifest" + +if [[ "${ASYNC_ENABLED}" != "true" ]] && [[ "${SWARM_ACTIVE}" == "true" ]]; then + if [[ "${BKP_TEST_SWARM_DRILL}" != "true" ]]; then + echo "FAIL: refusing to replay database dumps on an active Swarm outside the orchestrated CI DR drill" + exit 1 + fi + rm -f "${HANDOFF_PENDING}" "${HANDOFF_COMPLETE}" "${HANDOFF_MANIFEST}" + PROBE_BASE="infinito_${MACHINE_HASH:0:12}_$(date +%s)_${BASHPID}" + PROBE_PRE_TOKEN="${PROBE_BASE}_pre" + PROBE_POST_TOKEN="${PROBE_BASE}_post" + bash "${DIR}/db_probe.sh" seed "${PROBE_PRE_TOKEN}" +fi + echo "Forcing a post-deploy backup run (the service-loader pre-state backup can predate the app volumes and be empty)" if ! timeout "${BKP_TEST_HEALTH_TIMEOUT}" systemctl start "${BKP_TEST_SERVICE}"; then echo "backup unit start returned non-zero; inspecting result" fi wait_service_terminated -MACHINE_HASH="$(sha256sum /etc/machine-id | cut -c1-64)" MACHINE_DIR="${BKP_TEST_BACKUPS_DIR%/}/${MACHINE_HASH}" if [[ ! -d "${MACHINE_DIR}" ]]; then @@ -86,7 +99,6 @@ if (( COUNT < 1 )); then exit 1 fi -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" export MACHINE_HASH REPO_DIR REPO_NAME if [[ "${ASYNC_ENABLED}" == "true" ]]; then @@ -111,6 +123,29 @@ fi export NEWEST_GENERATION="${GENERATIONS[-1]}" bash "${DIR}/verify_backup.sh" + +if [[ "${SWARM_ACTIVE}" == "true" ]]; then + mapfile -t SQL_FILES < <(find "${REPO_DIR}/${NEWEST_GENERATION}" -mindepth 3 -maxdepth 3 \ + -type f -path '*/sql/*.backup.sql' ! -name '*.cluster.backup.sql' | sort) + if (( ${#SQL_FILES[@]} < 1 )); then + echo "SWARM SYNC PASS COMPLETE: backup verified; no single-database dumps require a restore" + exit 0 + fi + + bash "${DIR}/db_probe.sh" seed "${PROBE_POST_TOKEN}" + umask 077 + { + printf 'MACHINE_HASH=%q\n' "${MACHINE_HASH}" + printf 'REPO_NAME=%q\n' "${REPO_NAME}" + printf 'NEWEST_GENERATION=%q\n' "${NEWEST_GENERATION}" + printf 'PROBE_PRE_TOKEN=%q\n' "${PROBE_PRE_TOKEN}" + printf 'PROBE_POST_TOKEN=%q\n' "${PROBE_POST_TOKEN}" + } >"${HANDOFF_PENDING}.tmp" + mv "${HANDOFF_PENDING}.tmp" "${HANDOFF_PENDING}" + echo "SWARM SYNC PASS COMPLETE: destructive database restore handed to the orchestrated CI DR drill" + exit 0 +fi + bash "${DIR}/restore_cycle.sh" bash "${DIR}/db_restore.sh" echo "SYNC PASS COMPLETE: backup verified and restore cycle succeeded" diff --git a/roles/svc-bkp-volume-2-local/templates/test.env.j2 b/roles/svc-bkp-volume-2-local/templates/test.env.j2 index ac13b5f88d..1363d0e6d8 100644 --- a/roles/svc-bkp-volume-2-local/templates/test.env.j2 +++ b/roles/svc-bkp-volume-2-local/templates/test.env.j2 @@ -5,3 +5,4 @@ BKP_TEST_HEALTH_TIMEOUT={{ lookup('timeout', 1200) | int }} BKP_TEST_IS_STACK_HOST={{ IS_STACK_HOST | bool | lower }} BKP_TEST_RESTORE_BIN=baudolo-restore BKP_TEST_SERVICE={{ SYS_SERVICE_BACKUP_VOLUME_2_LOCAL }} +BKP_TEST_SWARM_DRILL={{ TEST_E2E_SWARM_DRILL | default(false) | bool | lower }} diff --git a/scripts/tests/deploy/swarm/routine/backup/04_quiesce_database_writers.sh b/scripts/tests/deploy/swarm/routine/backup/04_quiesce_database_writers.sh new file mode 100755 index 0000000000..9cbc9f2d1b --- /dev/null +++ b/scripts/tests/deploy/swarm/routine/backup/04_quiesce_database_writers.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail + +MODE="${1:?usage: 04_quiesce_database_writers.sh manager|worker}" + +is_infrastructure_image() { + local image="${1,,}" repository + repository="${image##*/}" + repository="${repository%%:*}" + repository="${repository%%@*}" + case "${repository}" in + postgres | postgres_* | postgis | postgis_* | mariadb | mariadb_* | mysql | mysql_* | registry) return 0 ;; + *) return 1 ;; + esac +} + +if [[ "${MODE}" == "manager" ]]; then + mapfile -t STACKS < <(docker stack ls --format '{{.Name}}' | sort) + REMOVED=() + for stack in "${STACKS[@]}"; do + mapfile -t IMAGES < <(docker stack services "${stack}" --format '{{.Image}}') + keep=true + if ((${#IMAGES[@]} < 1)); then + keep=false + fi + for image in "${IMAGES[@]}"; do + is_infrastructure_image "${image}" || keep=false + done + if [[ "${keep}" == "true" ]]; then + echo "KEEP: infrastructure stack ${stack} (${IMAGES[*]})" + continue + fi + docker stack rm "${stack}" + REMOVED+=("${stack}") + done + + deadline=$(($(date +%s) + 180)) + for stack in "${REMOVED[@]}"; do + while [[ -n "$(docker service ls -q --filter "label=com.docker.stack.namespace=${stack}")" ]]; do + if (($(date +%s) >= deadline)); then + echo "FAIL: stack ${stack} still has services after quiesce timeout" + exit 1 + fi + sleep 2 + done + echo "OK: removed writer stack ${stack}" + done +elif [[ "${MODE}" != "worker" ]]; then + echo "FAIL: unsupported quiesce mode '${MODE}'" + exit 1 +fi + +STOP_CONTAINERS=() +while IFS=$'\t' read -r container_id image; do + [[ -n "${container_id}" ]] || continue + if ! is_infrastructure_image "${image}"; then + STOP_CONTAINERS+=("${container_id}") + fi +done < <(docker ps --format '{{.ID}}\t{{.Image}}') +if ((${#STOP_CONTAINERS[@]} > 0)); then + docker stop "${STOP_CONTAINERS[@]}" +fi +echo "OK: ${MODE} runs only database engines and the local registry" diff --git a/scripts/tests/deploy/swarm/routine/backup/06_restore_databases.sh b/scripts/tests/deploy/swarm/routine/backup/06_restore_databases.sh new file mode 100755 index 0000000000..9598d02b82 --- /dev/null +++ b/scripts/tests/deploy/swarm/routine/backup/06_restore_databases.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +TEST_DIR="${1:?role test directory required}" +BACKUPS_ROOT="${2:?backup root required}" +PENDING="${TEST_DIR}/swarm-restore.pending" +COMPLETE="${TEST_DIR}/swarm-restore.complete" +MANIFEST="${TEST_DIR}/swarm-restore.manifest" + +if [[ ! -f "${PENDING}" ]]; then + echo "FAIL: database restore handoff missing at ${PENDING}" + exit 1 +fi +# shellcheck disable=SC1091 +. "${TEST_DIR}/test.env" +# shellcheck disable=SC1090 +. "${PENDING}" + +if [[ "${BKP_TEST_SWARM_DRILL}" != "true" ]]; then + echo "FAIL: database restore handoff was not created by the Swarm CI drill" + exit 1 +fi + +BKP_TEST_BACKUPS_DIR="${BACKUPS_ROOT%/}" +REPO_DIR="${BKP_TEST_BACKUPS_DIR}/${MACHINE_HASH}/${REPO_NAME}" +BKP_TEST_RESTORED_DATABASES_FILE="${MANIFEST}" +export BKP_TEST_BACKUPS_DIR REPO_DIR BKP_TEST_RESTORED_DATABASES_FILE +export MACHINE_HASH REPO_NAME NEWEST_GENERATION + +if [[ ! -d "${REPO_DIR}/${NEWEST_GENERATION}" ]]; then + echo "FAIL: handed-off database backup generation missing at ${REPO_DIR}/${NEWEST_GENERATION}" + exit 1 +fi + +bash "${TEST_DIR}/db_restore.sh" +if [[ ! -s "${MANIFEST}" ]]; then + echo "FAIL: database restore completed without a restored-database manifest" + exit 1 +fi +mv "${PENDING}" "${COMPLETE}" +echo "OK: consumed Swarm database restore handoff" diff --git a/scripts/tests/deploy/swarm/routine/backup/07_verify_databases.sh b/scripts/tests/deploy/swarm/routine/backup/07_verify_databases.sh new file mode 100755 index 0000000000..087eb1014f --- /dev/null +++ b/scripts/tests/deploy/swarm/routine/backup/07_verify_databases.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +TEST_DIR="${1:?role test directory required}" +COMPLETE="${TEST_DIR}/swarm-restore.complete" +MANIFEST="${TEST_DIR}/swarm-restore.manifest" + +if [[ ! -f "${COMPLETE}" ]]; then + echo "FAIL: completed database restore handoff missing at ${COMPLETE}" + exit 1 +fi +# shellcheck disable=SC1091 +. "${TEST_DIR}/test.env" +# shellcheck disable=SC1090 +. "${COMPLETE}" + +if [[ "${BKP_TEST_SWARM_DRILL}" != "true" ]]; then + echo "FAIL: completed database restore was not produced by the Swarm CI drill" + exit 1 +fi + +bash "${TEST_DIR}/db_probe.sh" verify \ + "${PROBE_PRE_TOKEN}" "${PROBE_POST_TOKEN}" "${MANIFEST}" diff --git a/scripts/tests/deploy/swarm/routine/backup/README.md b/scripts/tests/deploy/swarm/routine/backup/README.md index 4ac4917b5d..3647cf366f 100644 --- a/scripts/tests/deploy/swarm/routine/backup/README.md +++ b/scripts/tests/deploy/swarm/routine/backup/README.md @@ -6,6 +6,28 @@ the live instance; the drill passes only when every marker survives the whole loop. `base.sh` runs the nine steps below; the numbers in both diagrams are its `[n/9]` log markers. +## Schema + +The CI drill includes destructive database recovery. The disposable test +cluster keeps only the database engines and local registry available while +every database-writing workload, including shared dependencies such as +Discourse, is stopped. The normal matrix update remains the single source of +truth for bringing the workloads back. + +```mermaid +flowchart LR + seed["Seed pre-backup database probes"] + backup["Run deployed backup units"] + mutate["Write post-backup probes"] + quiesce["Remove all database writers
target stack + shared dependencies"] + files["Recover NFS, volumes and secrets"] + database["Restore SQL dumps into
the real CI databases"] + redeploy["Matrix update redeploys
the desired state"] + verify["Health + Playwright
pre-backup probes present
post-backup probes absent"] + + seed --> backup --> mutate --> quiesce --> files --> database --> redeploy --> verify +``` + ## Step sequence (who does what) ```mermaid @@ -13,10 +35,13 @@ sequenceDiagram autonumber participant N as nfs-server participant M as manager + participant D as database engines participant B as backup host participant U as LUKS "USB" (loop img on B) - Note over N,M: [1/9] seed markers + Note over M,D: role test seeds pre-backup DB probes, backs up, then seeds post-backup probes + + Note over N,M: [1/9] seed file markers N->>N: marker → live NFS volume dir M->>M: marker → host secrets dir @@ -34,11 +59,11 @@ sequenceDiagram Note over B,U: [5/9] plug LUKS device, sync via local-2-device unit B->>U: hard-linked snapshot - Note over M: [6/9] full disaster teardown - alt stack workload - M->>M: docker stack rm + wait - else node-local workload - M->>M: stop compose project on every node + Note over M,D: [6/9] quiesce writers + alt database handoff exists + M->>M: remove all non-DB/non-registry stacks and containers + else file-only drill + M->>M: remove the target stack or node-local compose project end Note over B,U: [7/9] recover device → local root @@ -53,7 +78,11 @@ sequenceDiagram B-->>M: volume generation → recover CLI B-->>M: secrets generation → recover CLI - Note over M,N: matrix update pass redeploys, then verify_recovered_marker.sh asserts the live marker + Note over M,D: [9b/9] restore selected SQL generation + B-->>M: device-recovered volume backup repository + M->>D: replay dumps while all writers remain stopped + + Note over M,N: matrix update redeploys; health, Playwright, file markers and DB before/after probes must pass ``` ## Data flow (what is proven) @@ -64,33 +93,39 @@ flowchart TB direction TB vol["📂 [1] Live NFS volume + docker volume
marker seeded"]:::live sec["🔑 [1] Host secrets on the manager
(secrets, CA, ACME, node identity) marker seeded"]:::live + dbpre["🗃️ Pre-backup DB probe
included in the selected SQL generation"]:::live localbkp["🗄️ [2-3] Local backups on manager + NFS server
volume / nfs / secrets generation snapshots"]:::store pulled["🗄️ [4] Backup host
pulled generations (remote-2-local unit, rsync over ssh, pull-only)"]:::store usb["🔒 [5] Encrypted USB (LUKS)
hard-linked snapshot (local-2-device unit)"]:::device vol -->|backup units| localbkp sec -->|secrets unit| localbkp + dbpre -->|SQL dump| localbkp localbkp -->|remote pull| pulled -->|device sync| usb end - teardown["🛑 [6] Full disaster:
stack rm (stack workload) or
compose stop on every node (node-local workload),
then export wiped"]:::dead + dbpost["✍️ Post-backup DB probe
must disappear after restore"]:::dead + teardown["🛑 [6] Full disaster:
all DB writers removed, including shared dependencies;
only DB engines + registry remain,
then export wiped"]:::dead - subgraph recovery["♻️ Recovery (dashed: the recover CLI, newest generation only)"] + subgraph recovery["♻️ Recovery (dashed: newest file snapshots + handed-off SQL generation)"] direction TB restored["📁 [7] Restore root on the backup host
newest snapshot after luksOpen (pulled tree + loop img freed)"]:::store export["📂 [8] Live NFS export
volume subtree rebuilt, ganesha + client mounts refreshed"]:::live volume["📦 [9] Docker volume
rebuilt on the manager"]:::live secrets["🔑 [9] Host secrets
rebuilt on the manager (marker cleared first)"]:::live - verified["✅ Update pass redeploys
markers verified on the live instance"]:::ok + database["🗃️ [9b] Live databases
selected SQL generation replayed without writers"]:::live + verified["✅ Update pass redeploys
health + Playwright pass;
file markers and DB before/after probes verified"]:::ok usb -.->|decrypt + recover| restored restored -.->|restore export| export restored -.->|restore volume| volume restored -.->|restore secrets| secrets + restored -.->|restore SQL dumps| database export -.-> verified volume -.-> verified secrets -.-> verified + database -.-> verified end - forward --> teardown --> recovery + forward --> dbpost --> teardown --> recovery classDef live fill:#f1f8f6,stroke:#d5e7e2,color:#455a64; classDef store fill:#f3f7fb,stroke:#d6e2ee,color:#455a64; @@ -103,13 +138,16 @@ flowchart TB ## Applicability -- The drill runs whenever the app declares an NFS-flagged volume - (`PRIMARY_NFS_VOLUME`); apps without one skip it, since there is nothing - to prove a restore against. -- The workload form only changes step 6: stack workloads get - `docker stack rm`, `workload: node-local` roles get their compose - project stopped on every node. Every other step operates on volumes, - the NFS export and the deployed backup units, independent of the - orchestrator. -- Runs once per app, in the variant-0 CI job, between the matrix's first - deploy and its update pass. +- The first matrix round runs the complete file chain whenever the app + declares an NFS-flagged volume (`PRIMARY_NFS_VOLUME`). A database handoff + is destructive even without NFS: all writers are quiesced and the selected + manager-local generation is restored. +- Later variant rounds skip the expensive file chain. If their role test + hands off SQL dumps, they still run the destructive database restore and + post-update probe verification; no Swarm restore is replaced by a guard. +- In a file-only drill, the workload form changes step 6: stack workloads + get `docker stack rm`, while `workload: node-local` roles have their + Compose project stopped on every node. A database handoff instead removes + every non-database workload in either form. +- Runs between the sync and update pass. The full NFS/device chain runs once + per app; database recovery runs in every round that produced a handoff. diff --git a/scripts/tests/deploy/swarm/routine/backup/base.sh b/scripts/tests/deploy/swarm/routine/backup/base.sh index 7bc7972338..09052674f4 100755 --- a/scripts/tests/deploy/swarm/routine/backup/base.sh +++ b/scripts/tests/deploy/swarm/routine/backup/base.sh @@ -1,21 +1,4 @@ #!/usr/bin/env bash -# Disaster-recovery drill for the swarm test cluster: proves the full -# backup chain volume + secrets + nfs -> remote -> device through the -# DEPLOYED systemd units on every host (the backup host runs the real -# svc-bkp-remote-2-local and svc-bkp-local-2-device roles; the drill only -# installs the ssh pull identity and simulates the USB plug via a LUKS -# loop mount), then tears the stack down completely and recovers the same -# data back through the recover CLI (cli.administration.recover: device -> -# local root -> nfs export, docker volume and host secrets). The matrix update -# pass then boots the stack onto the recovered export and -# verify_recovered_marker.sh asserts the live marker, so no dedicated redeploy -# runs here. Runs once, between the matrix's first and second round, against -# the already-converged round-1 stack. Per-host -# routines live next to this file and execute in-node from the repo copy -# under INFINITO_NODE_SRC_DIR (one docker exec per routine). Marker probes -# are scoped per repo (volume/nfs/secrets share DR_MARKER; an unscoped -# ${MID} glob would cross-select). Device paths (mount/target) come from -# the same extras SPOT that configures the deployed role (write_extras). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -45,14 +28,64 @@ USB_IMG="/var/lib/infinito-usb.img" USB_MAPPER="usbdrill" USB_PASS="drillpass" RESTORE_ROOT="/var/tmp/dr-device-restored" +ROLE_TEST_DIR="/tmp/test-e2e-cli/svc-bkp-volume-2-local" +DB_PENDING_FILE="${ROLE_TEST_DIR}/swarm-restore.pending" +DB_RESTORE_ROOT="/var/tmp/dr-database-restore-${APP_ID}" +DRILL_FULL_CHAIN="${DRILL_FULL_CHAIN:-true}" DEV_MOUNT="$(python3 -c "import sys, yaml; print(yaml.safe_load(open(sys.argv[1]))['applications']['svc-bkp-local-2-device']['services']['local-2-device']['mount'])" "${DRILL_EXTRAS}")" DEV_TARGET="$(python3 -c "import sys, yaml; print(yaml.safe_load(open(sys.argv[1]))['applications']['svc-bkp-local-2-device']['services']['local-2-device']['target'])" "${DRILL_EXTRAS}")" DEV_DEST="${DEV_MOUNT}${DEV_TARGET}" -if [ -z "${PRIMARY_NFS_VOLUME}" ]; then - echo "SKIP drill: ${APP_ID} declares no NFS-flagged volume — nothing to prove a restore against" +case "${DRILL_FULL_CHAIN}" in +true | false) ;; +*) + echo "FAILURE: DRILL_FULL_CHAIN must be true or false" + exit 1 + ;; +esac + +DB_PENDING=false +if docker exec "${MGR}" test -f "${DB_PENDING_FILE}"; then + DB_PENDING=true +fi +MGR_MID="$(docker exec "${MGR}" sha256sum /etc/machine-id | cut -c1-64)" + +quiesce_database_writers() { + docker exec "${MGR}" bash "${BKP_IN_NODE}/04_quiesce_database_writers.sh" manager + for _node in "${WRK1}" "${WRK2}"; do + docker exec "${_node}" bash "${BKP_IN_NODE}/04_quiesce_database_writers.sh" worker + done +} + +restore_databases() { + local backups_root="${1}" + docker exec "${MGR}" bash "${BKP_IN_NODE}/06_restore_databases.sh" \ + "${ROLE_TEST_DIR}" "${backups_root}" +} + +write_verify_env() { + local db_verify="${1}" nfs_verify="${2}" + printf 'DR_DB_VERIFY=%s\nDR_NFS_VERIFY=%s\nMGR=%s\nROLE_TEST_DIR=%s\nBKP_IN_NODE=%s\n' \ + "${db_verify}" "${nfs_verify}" "${MGR}" "${ROLE_TEST_DIR}" "${BKP_IN_NODE}" >"${DR_VERIFY_ENV}" + if [ "${nfs_verify}" = true ]; then + printf 'DR_TOKEN=%s\nDR_MARKER=%s\nNFS_SERVER=%s\nNFS_VOL_DIR=%s\n' \ + "${DR_TOKEN}" "${DR_MARKER}" "${NFS_SERVER}" "${NFS_VOL_DIR}" >>"${DR_VERIFY_ENV}" + fi +} + +if [ "${DRILL_FULL_CHAIN}" = false ] || [ -z "${PRIMARY_NFS_VOLUME}" ]; then + if [ "${DB_PENDING}" != true ]; then + echo "SKIP drill: no database handoff and no full NFS recovery requested for ${APP_ID}" + exit 0 + fi + echo "==> destructive database-only DR drill for ${APP_ID}" + quiesce_database_writers + restore_databases "${DIR_BACKUPS}" + write_verify_env true false + echo "==> database recovery complete; the matrix update redeploys and verifies the restored probes" exit 0 fi + if ! docker inspect "${BACKUP_NODE}" >/dev/null 2>&1; then echo "FAILURE: backup host ${BACKUP_NODE} is not running (01_bootstrap.sh did not run?)" exit 1 @@ -63,7 +96,6 @@ if [ ! -f "${BACKUP_KEY_PATH}" ]; then fi NFS_VOL_DIR="${NFS_STATE_PATH}/${PRIMARY_NFS_VOLUME}" -MGR_MID="$(docker exec "${MGR}" sha256sum /etc/machine-id | cut -c1-64)" NFS_MID="$(docker exec "${NFS_SERVER}" sha256sum /etc/machine-id | cut -c1-64)" echo "==> DR drill for ${APP_ID} (volume '${PRIMARY_NFS_VOLUME}')" @@ -152,8 +184,10 @@ if ! docker exec "${BACKUP_NODE}" find "${DEV_DEST}" -name "${DR_MARKER}" 2>/dev fi echo " marker present on encrypted USB" -echo "==> [6/9] tear the stack down completely (full disaster) before recovery" -if [ "${HAS_SWARM_SERVICE}" = true ]; then +echo "==> [6/9] quiesce workloads before recovery" +if [ "${DB_PENDING}" = true ]; then + quiesce_database_writers +elif [ "${HAS_SWARM_SERVICE}" = true ]; then docker exec "${MGR}" bash "${BKP_IN_NODE}/04_stack_rm_wait.sh" "${STACK_NAME}" else for _node in "${MGR}" "${WRK1}" "${WRK2}"; do @@ -251,7 +285,18 @@ else echo " secrets recover skipped: svc-bkp-secrets-2-local not installed on ${MGR}" fi -echo "==> recovery complete: device -> nfs export -> volume -> secrets restored via the recover CLI" -echo " the matrix update pass boots the stack onto the recovered export; verify_recovered_marker.sh asserts the live marker there" -printf 'DR_TOKEN=%s\nDR_MARKER=%s\nNFS_SERVER=%s\nNFS_VOL_DIR=%s\n' \ - "${DR_TOKEN}" "${DR_MARKER}" "${NFS_SERVER}" "${NFS_VOL_DIR}" >"${DR_VERIFY_ENV}" +if [ "${DB_PENDING}" = true ]; then + echo "==> [9b/9] restore the handed-off SQL generation from the device-recovered tree" + if ! docker exec "${BACKUP_NODE}" test -d "${RESTORE_ROOT}/${MGR_MID}/${VOLUME_REPO}"; then + echo "FAILURE: ${VOLUME_REPO} missing from the device-recovered manager tree" + exit 1 + fi + docker exec "${MGR}" sh -c "rm -rf '${DB_RESTORE_ROOT}'; mkdir -p '${DB_RESTORE_ROOT}'" + docker exec "${BACKUP_NODE}" tar -C "${RESTORE_ROOT}" -cf - "${MGR_MID}/${VOLUME_REPO}" | + docker exec -i "${MGR}" tar --numeric-owner -C "${DB_RESTORE_ROOT}" -xf - + restore_databases "${DB_RESTORE_ROOT}" +fi + +echo "==> recovery complete: device -> nfs export -> volume -> secrets -> databases restored" +echo " the matrix update pass redeploys every workload and verifies the recovered markers and database probes" +write_verify_env "${DB_PENDING}" true diff --git a/scripts/tests/deploy/swarm/routine/backup/verify_recovered_marker.sh b/scripts/tests/deploy/swarm/routine/backup/verify_recovered_marker.sh index d5b1ef7418..56873375b2 100755 --- a/scripts/tests/deploy/swarm/routine/backup/verify_recovered_marker.sh +++ b/scripts/tests/deploy/swarm/routine/backup/verify_recovered_marker.sh @@ -1,8 +1,4 @@ #!/usr/bin/env bash -# Runs on the controller after the matrix update pass. Asserts the DR-drill -# marker (seeded pre-backup) survived recovery plus the update-pass bring-up -# onto the recovered NFS export. Skips when the sentinel is absent (the drill -# recovered nothing: the app declares no NFS-flagged volume). set -euo pipefail DR_VERIFY_ENV="/tmp/dr-drill-verify-${APP_ID:?APP_ID required (matrix sets it)}.env" @@ -13,8 +9,15 @@ fi # shellcheck disable=SC1090 . "${DR_VERIFY_ENV}" -if ! docker exec "${NFS_SERVER}" grep -qF "${DR_TOKEN}" "${NFS_VOL_DIR}/${DR_MARKER}" 2>/dev/null; then - echo "FAILURE: DR-drill marker missing on the live volume after recover + update pass" - exit 1 +if [ "${DR_NFS_VERIFY:?}" = true ]; then + if ! docker exec "${NFS_SERVER}" grep -qF "${DR_TOKEN}" "${NFS_VOL_DIR}/${DR_MARKER}" 2>/dev/null; then + echo "FAILURE: DR-drill marker missing on the live volume after recover + update pass" + exit 1 + fi fi -echo "==> DR drill PASSED: backup -> teardown -> recover device->nfs->volume->secrets -> live marker verified after the update pass" + +if [ "${DR_DB_VERIFY:?}" = true ]; then + docker exec "${MGR}" bash "${BKP_IN_NODE}/07_verify_databases.sh" "${ROLE_TEST_DIR}" +fi + +echo "==> DR drill PASSED: recovered NFS=${DR_NFS_VERIFY}, databases=${DR_DB_VERIFY}; post-update probes verified" diff --git a/tests/unit/utils/tests/swarm/test_backup_drill.py b/tests/unit/utils/tests/swarm/test_backup_drill.py new file mode 100644 index 0000000000..618fa9997e --- /dev/null +++ b/tests/unit/utils/tests/swarm/test_backup_drill.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils import PROJECT_ROOT + + +BACKUP_SCRIPTS = PROJECT_ROOT / "scripts/tests/deploy/swarm/routine/backup" +ROLE_TEST_SCRIPTS = PROJECT_ROOT / "roles/svc-bkp-volume-2-local/files/test" + + +class TestDatabaseRestoreHandoff(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.test_dir = self.root / "role-test" + self.test_dir.mkdir() + (self.test_dir / "test.env").write_text( + "BKP_TEST_SWARM_DRILL=true\nBKP_TEST_DATABASES_CSV=/unused\n" + ) + + def test_restore_consumes_handoff_only_after_manifest_is_written(self): + backups = self.root / "backups" + (backups / "machine" / "repo" / "generation").mkdir(parents=True) + (self.test_dir / "swarm-restore.pending").write_text( + "MACHINE_HASH=machine\n" + "REPO_NAME=repo\n" + "NEWEST_GENERATION=generation\n" + "PROBE_PRE_TOKEN=before\n" + "PROBE_POST_TOKEN=after\n" + ) + restore = self.test_dir / "db_restore.sh" + restore.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + ": \"${BKP_TEST_BACKUPS_DIR:?}\" \"${REPO_DIR:?}\"\n" + "printf 'postgres;postgres_data;app\\n' >\"${BKP_TEST_RESTORED_DATABASES_FILE}\"\n" + ) + restore.chmod(0o755) + + subprocess.run( + [ + "bash", + str(BACKUP_SCRIPTS / "06_restore_databases.sh"), + str(self.test_dir), + str(backups), + ], + check=True, + capture_output=True, + text=True, + ) + + self.assertFalse((self.test_dir / "swarm-restore.pending").exists()) + self.assertTrue((self.test_dir / "swarm-restore.complete").is_file()) + self.assertTrue((self.test_dir / "swarm-restore.manifest").is_file()) + + def test_verify_passes_completed_probe_tokens_to_role_helper(self): + (self.test_dir / "swarm-restore.complete").write_text( + "PROBE_PRE_TOKEN=before\nPROBE_POST_TOKEN=after\n" + ) + manifest = self.test_dir / "swarm-restore.manifest" + manifest.write_text("postgres;postgres_data;app\n") + verified = self.root / "probe-verified" + probe = self.test_dir / "db_probe.sh" + probe.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "test \"$1\" = verify\n" + "test \"$2\" = before\n" + "test \"$3\" = after\n" + f"test \"$4\" = {manifest}\n" + "touch \"${PROBE_VERIFIED:?}\"\n" + ) + probe.chmod(0o755) + env = os.environ.copy() + env["PROBE_VERIFIED"] = str(verified) + + subprocess.run( + [ + "bash", + str(BACKUP_SCRIPTS / "07_verify_databases.sh"), + str(self.test_dir), + ], + check=True, + capture_output=True, + text=True, + env=env, + ) + + self.assertTrue(verified.is_file()) + + +class TestDatabaseRestoreProbe(unittest.TestCase): + def test_seeds_and_verifies_postgres_and_mariadb(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + csv_file = root / "databases.csv" + csv_file.write_text( + "instance;database;username;password\n" + "postgres;app_pg;user_pg;pw_pg\n" + "mariadb;app_my;user_my;pw_my\n" + ) + manifest = root / "manifest" + manifest.write_text( + "postgres;postgres_data;app_pg\n" + "mariadb;mariadb_data;app_my\n" + ) + fake_bin = root / "bin" + fake_bin.mkdir() + container = fake_bin / "container" + container.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "case \"$1\" in\n" + " ps)\n" + " case \"$*\" in\n" + " *postgres_data*) printf 'postgres.1\\n' ;;\n" + " *mariadb_data*) printf 'mariadb.1\\n' ;;\n" + " esac\n" + " ;;\n" + " inspect)\n" + " case \"${*: -1}\" in\n" + " postgres.1) printf 'postgres_custom\\n' ;;\n" + " mariadb.1) printf 'mariadb_custom\\n' ;;\n" + " esac\n" + " ;;\n" + " exec)\n" + " case \"$*\" in\n" + " *SELECT*before*)\n" + " test -f \"${DB_STATE:?}/seeded-${4}\" && printf '1\\n' || printf '0\\n'\n" + " ;;\n" + " *SELECT*after*) printf '0\\n' ;;\n" + " *) touch \"${DB_STATE:?}/seeded-${4}\" ;;\n" + " esac\n" + " ;;\n" + "esac\n" + ) + container.chmod(0o755) + env = os.environ.copy() + env["PATH"] = f"{fake_bin}:{env['PATH']}" + env["BKP_TEST_DATABASES_CSV"] = str(csv_file) + env["DB_STATE"] = str(root) + + subprocess.run( + ["bash", str(ROLE_TEST_SCRIPTS / "db_probe.sh"), "seed", "before"], + check=True, + capture_output=True, + text=True, + env=env, + ) + self.assertTrue((root / "seeded-postgres.1").is_file()) + self.assertTrue((root / "seeded-mariadb.1").is_file()) + + subprocess.run( + [ + "bash", + str(ROLE_TEST_SCRIPTS / "db_probe.sh"), + "verify", + "before", + "after", + str(manifest), + ], + check=True, + capture_output=True, + text=True, + env=env, + ) + + +class TestDatabaseWriterQuiesce(unittest.TestCase): + def test_keeps_only_database_registry_and_stops_discourse(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + fake_bin = root / "bin" + fake_bin.mkdir() + docker = fake_bin / "docker" + docker.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "case \"$1 $2\" in\n" + " 'stack ls') printf 'docker\\npostgres\\nwordpress\\n' ;;\n" + " 'stack services')\n" + " case \"$3\" in\n" + " docker) printf 'registry:2\\n' ;;\n" + " postgres) printf 'localhost:5000/postgres_custom:17\\n' ;;\n" + " wordpress) printf 'wordpress:latest\\n' ;;\n" + " esac\n" + " ;;\n" + " 'stack rm') touch \"${DOCKER_STATE:?}/removed-$3\" ;;\n" + " 'service ls') : ;;\n" + " 'ps --format')\n" + " printf 'db\\tlocalhost:5000/postgres_custom:17\\n'\n" + " printf 'registry\\tregistry:2\\n'\n" + " printf 'discourse\\tlocal_discourse/discourse:latest\\n'\n" + " printf 'wordpress-task\\twordpress:latest\\n'\n" + " ;;\n" + " 'stop discourse')\n" + " touch \"${DOCKER_STATE:?}/stopped-$2\" \"${DOCKER_STATE:?}/stopped-$3\"\n" + " ;;\n" + " *) printf 'unexpected docker call: %s\\n' \"$*\" >&2; exit 1 ;;\n" + "esac\n" + ) + docker.chmod(0o755) + env = os.environ.copy() + env["PATH"] = f"{fake_bin}:{env['PATH']}" + env["DOCKER_STATE"] = str(root) + + subprocess.run( + [ + "bash", + str(BACKUP_SCRIPTS / "04_quiesce_database_writers.sh"), + "manager", + ], + check=True, + capture_output=True, + text=True, + env=env, + ) + + self.assertTrue((root / "removed-wordpress").is_file()) + self.assertTrue((root / "stopped-discourse").is_file()) + self.assertTrue((root / "stopped-wordpress-task").is_file()) + self.assertFalse((root / "removed-postgres").exists()) + self.assertFalse((root / "removed-docker").exists()) + self.assertFalse((root / "stopped-db").exists()) + self.assertFalse((root / "stopped-registry").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/tests/swarm/matrix.py b/utils/tests/swarm/matrix.py index 094ddc345c..4466b4e31f 100644 --- a/utils/tests/swarm/matrix.py +++ b/utils/tests/swarm/matrix.py @@ -139,6 +139,8 @@ def _deploy( f"@{extras_path}", "-e", f"VARIANT_INDEX={json.dumps(round_index)}", + "-e", + "TEST_E2E_SWARM_DRILL=true", ] pass_label = ( f"matrix-deploy: round {round_index + 1}/{total} " @@ -192,11 +194,14 @@ def _converge_and_verify(*, app_id: str) -> int: ) -def _backup_restore_drill(*, app_id: str, inv_dir: str, extras_path: str) -> int: +def _backup_restore_drill( + *, app_id: str, inv_dir: str, extras_path: str, full_chain: bool +) -> int: env = os.environ.copy() env["APP_ID"] = app_id env["INFINITO_INVENTORY_DIR"] = inv_dir env["DRILL_EXTRAS"] = extras_path + env["DRILL_FULL_CHAIN"] = str(full_chain).lower() return _run( ["bash", str(_SWARM_SCRIPTS / "backup" / "base.sh")], env=env, @@ -339,15 +344,18 @@ def main(argv: list[str] | None = None) -> int: ) if rc == 0: rc = _converge_and_verify(app_id=app_id) - if rc == 0 and round_index == 0: + if rc == 0 and plan_index == 0: rc = _deploy_backup_host( app_id=app_id, inv_dir=inv_root, extras_path=f"{inv_root}/swarm-nfs-extras.deploy.yml", ) - if rc == 0 and round_index == 0: + if rc == 0: rc = _backup_restore_drill( - app_id=app_id, inv_dir=inv_root, extras_path=extras_path + app_id=app_id, + inv_dir=inv_root, + extras_path=extras_path, + full_chain=plan_index == 0, ) if rc == 0: rc = _deploy( @@ -360,7 +368,7 @@ def main(argv: list[str] | None = None) -> int: ) if rc == 0: rc = _converge_and_verify(app_id=app_id) - if rc == 0 and round_index == 0: + if rc == 0: rc = _verify_recovered_marker(app_id=app_id) if rc != 0: return rc