fix(backup): quiesce swarm writers for restore

Defer Swarm SQL replay to the disposable DR drill so shared writers such as Discourse are stopped before the destructive restore. Verify the selected generation with before/after database probes while preserving the Compose restore path.

roles/svc-bkp-volume-2-local/files/test/db_probe.sh

@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${BKP_TEST_DATABASES_CSV:?}"
+
+MODE="${1:?usage: db_probe.sh seed TOKEN | verify PRE_TOKEN POST_TOKEN MANIFEST}"
+PROBE_TABLE="infinito_backup_restore_probe"
+
+validate_token() {
+	if [[ ! "${1}" =~ ^[A-Za-z0-9_-]+$ ]]; then
+		echo "FAIL: unsafe database probe token"
+		exit 1
+	fi
+}
+
+engine_for_container() {
+	local image
+	image="$(container inspect -f '{{.Config.Image}}' "${1}")"
+	case "${image}" in
+	*postgres* | *postgis*) printf 'postgres\n' ;;
+	*mariadb* | *mysql*) printf 'mariadb\n' ;;
+	*) return 1 ;;
+	esac
+}
+
+container_for_instance() {
+	local candidate engine instance="${1}"
+	mapfile -t candidates < <(container ps --filter "volume=${instance}_data" --format '{{.Names}}')
+	if ((${#candidates[@]} < 1)); then
+		mapfile -t candidates < <(container ps --filter "name=${instance}" --format '{{.Names}}')
+	fi
+	for candidate in "${candidates[@]}"; do
+		if engine="$(engine_for_container "${candidate}")"; then
+			printf '%s;%s\n' "${candidate}" "${engine}"
+			return 0
+		fi
+	done
+	return 1
+}
+
+container_for_volume() {
+	local candidate engine volume="${1}" expected_engine="${2}"
+	while IFS= read -r candidate; do
+		[[ -n "${candidate}" ]] || continue
+		if engine="$(engine_for_container "${candidate}")" && [[ "${engine}" == "${expected_engine}" ]]; then
+			printf '%s\n' "${candidate}"
+			return 0
+		fi
+	done < <(container ps --filter "volume=${volume}" --format '{{.Names}}')
+	return 1
+}
+
+run_seed() {
+	local container database engine instance password resolved token="${1}" user
+	local seeded=0
+	validate_token "${token}"
+	if [[ ! -f "${BKP_TEST_DATABASES_CSV}" ]]; then
+		echo "FAIL: databases csv missing at ${BKP_TEST_DATABASES_CSV}"
+		exit 1
+	fi
+
+	while IFS=';' read -r instance database user password; do
+		[[ -n "${instance}" ]] || continue
+		[[ "${database}" != "*" ]] || continue
+		if ! resolved="$(container_for_instance "${instance}")"; then
+			echo "SKIP: no running database container found for instance '${instance}'"
+			continue
+		fi
+		IFS=';' read -r container engine <<<"${resolved}"
+		case "${engine}" in
+		postgres)
+			container exec -e "PGPASSWORD=${password}" "${container}" \
+				psql -X -v ON_ERROR_STOP=1 -U "${user}" -d "${database}" -c \
+				"CREATE TABLE IF NOT EXISTS ${PROBE_TABLE} (token text PRIMARY KEY); INSERT INTO ${PROBE_TABLE} (token) VALUES ('${token}') ON CONFLICT (token) DO NOTHING;" >/dev/null
+			;;
+		mariadb)
+			container exec -e "MYSQL_PWD=${password}" "${container}" \
+				mariadb --batch --skip-column-names -u "${user}" "${database}" -e \
+				"CREATE TABLE IF NOT EXISTS ${PROBE_TABLE} (token varchar(128) PRIMARY KEY); INSERT IGNORE INTO ${PROBE_TABLE} (token) VALUES ('${token}');" >/dev/null
+			;;
+		esac
+		echo "OK: seeded database restore probe in ${database} (${engine})"
+		seeded=$((seeded + 1))
+	done < <(tail -n +2 "${BKP_TEST_DATABASES_CSV}")
+	echo "OK: seeded ${seeded} database restore probe(s)"
+}
+
+probe_count() {
+	local container="${1}" database="${2}" engine="${3}" password="${4}" token="${5}" user="${6}"
+	case "${engine}" in
+	postgres)
+		container exec -e "PGPASSWORD=${password}" "${container}" \
+			psql -X -Atq -v ON_ERROR_STOP=1 -U "${user}" -d "${database}" -c \
+			"SELECT COUNT(*) FROM ${PROBE_TABLE} WHERE token = '${token}';" | tr -d '[:space:]'
+		;;
+	mariadb)
+		container exec -e "MYSQL_PWD=${password}" "${container}" \
+			mariadb --batch --skip-column-names -u "${user}" "${database}" -e \
+			"SELECT COUNT(*) FROM ${PROBE_TABLE} WHERE token = '${token}';" | tr -d '[:space:]'
+		;;
+	*)
+		echo "FAIL: unsupported database engine '${engine}'"
+		exit 1
+		;;
+	esac
+}
+
+run_verify() {
+	local container database engine manifest="${3}" password post_count pre_count
+	local pre_token="${1}" post_token="${2}" row user verified=0 volume
+	validate_token "${pre_token}"
+	validate_token "${post_token}"
+	if [[ ! -s "${manifest}" ]]; then
+		echo "FAIL: restored database manifest missing or empty at ${manifest}"
+		exit 1
+	fi
+
+	while IFS=';' read -r engine volume database; do
+		[[ -n "${engine}" && -n "${volume}" && -n "${database}" ]] || continue
+		row="$(awk -F';' -v db="${database}" 'NR > 1 && $2 == db { print; exit }' "${BKP_TEST_DATABASES_CSV}")"
+		if [[ -z "${row}" ]]; then
+			echo "FAIL: no databases.csv row for restored database '${database}'"
+			exit 1
+		fi
+		IFS=';' read -r _instance _database user password <<<"${row}"
+		if ! container="$(container_for_volume "${volume}" "${engine}")"; then
+			echo "FAIL: no running ${engine} container mounts restored volume ${volume}"
+			exit 1
+		fi
+		pre_count="$(probe_count "${container}" "${database}" "${engine}" "${password}" "${pre_token}" "${user}")"
+		post_count="$(probe_count "${container}" "${database}" "${engine}" "${password}" "${post_token}" "${user}")"
+		if [[ "${pre_count}" != "1" ]] || [[ "${post_count}" != "0" ]]; then
+			echo "FAIL: ${database} restore probe mismatch (pre=${pre_count}, post=${post_count})"
+			exit 1
+		fi
+		echo "OK: ${database} contains the pre-backup probe and excludes the post-backup probe"
+		verified=$((verified + 1))
+	done <"${manifest}"
+	echo "OK: verified ${verified} destructively restored database(s)"
+}
+
+case "${MODE}" in
+seed)
+	run_seed "${2:?seed token required}"
+	;;
+verify)
+	run_verify "${2:?pre token required}" "${3:?post token required}" "${4:?manifest required}"
+	;;
+*)
+	echo "FAIL: unsupported db_probe mode '${MODE}'"
+	exit 1
+	;;
+esac

roles/svc-bkp-volume-2-local/files/test/db_restore.sh

@@ -16,6 +16,11 @@ set -euo pipefail
 : "${NEWEST_GENERATION:?}"
 
 GEN_DIR="${REPO_DIR}/${NEWEST_GENERATION}"
+RESTORED_DATABASES_FILE="${BKP_TEST_RESTORED_DATABASES_FILE:-}"
+
+if [[ -n "${RESTORED_DATABASES_FILE}" ]]; then
+    : >"${RESTORED_DATABASES_FILE}"
+fi
 
 if [[ ! -f "${BKP_TEST_DATABASES_CSV}" ]]; then
     echo "FAIL: databases csv missing at ${BKP_TEST_DATABASES_CSV}"
@@ -71,6 +76,9 @@ for sql_file in "${SQL_FILES[@]}"; do
         --db-password "${db_password}" \
         --empty
     echo "OK: replayed ${engine} dump '${db_name}' into ${container}"
+    if [[ -n "${RESTORED_DATABASES_FILE}" ]]; then
+        printf '%s;%s;%s\n' "${engine}" "${volume}" "${db_name}" >>"${RESTORED_DATABASES_FILE}"
+    fi
     RESTORED=$((RESTORED + 1))
 done
 

roles/svc-bkp-volume-2-local/files/test/test.sh

@@ -1,14 +1,4 @@
 #!/usr/bin/env bash
-# E2E orchestrator for svc-bkp-volume-2-local.
-# Waits for the backup service to terminate, then:
-#   sync pass (ASYNC_ENABLED=false): verify the first backup generation is
-#     stored, then run the destructive restore drill (stop containers, wipe
-#     the backed-up volumes, restore, require everything healthy again) and
-#     replay the sql dumps.
-#   async pass (ASYNC_ENABLED=true): require two differential backup
-#     generations, triggering one extra service run when only one exists;
-#     no destructive steps.
-# Variables sourced from test.env.j2 by test-e2e-cli.
 set -euo pipefail
 
 : "${ASYNC_ENABLED:?}"
@@ -17,6 +7,7 @@ set -euo pipefail
 : "${BKP_TEST_SERVICE:?}"
 : "${BKP_TEST_RESTORE_BIN:?}"
 : "${BKP_TEST_HEALTH_TIMEOUT:?}"
+: "${BKP_TEST_SWARM_DRILL:?}"
 
 if [[ "${BKP_TEST_IS_STACK_HOST}" != "true" ]]; then
     echo "SKIP: not the stack host; svc-bkp-volume-2-local only deploys there"
@@ -57,13 +48,35 @@ count_generations() {
 
 wait_service_terminated
 
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SWARM_ACTIVE=false
+if [[ "$(container info --format '{{.Swarm.LocalNodeState}}' 2>/dev/null)" == "active" ]]; then
+    SWARM_ACTIVE=true
+fi
+
+MACHINE_HASH="$(sha256sum /etc/machine-id | cut -c1-64)"
+HANDOFF_PENDING="${DIR}/swarm-restore.pending"
+HANDOFF_COMPLETE="${DIR}/swarm-restore.complete"
+HANDOFF_MANIFEST="${DIR}/swarm-restore.manifest"
+
+if [[ "${ASYNC_ENABLED}" != "true" ]] && [[ "${SWARM_ACTIVE}" == "true" ]]; then
+    if [[ "${BKP_TEST_SWARM_DRILL}" != "true" ]]; then
+        echo "FAIL: refusing to replay database dumps on an active Swarm outside the orchestrated CI DR drill"
+        exit 1
+    fi
+    rm -f "${HANDOFF_PENDING}" "${HANDOFF_COMPLETE}" "${HANDOFF_MANIFEST}"
+    PROBE_BASE="infinito_${MACHINE_HASH:0:12}_$(date +%s)_${BASHPID}"
+    PROBE_PRE_TOKEN="${PROBE_BASE}_pre"
+    PROBE_POST_TOKEN="${PROBE_BASE}_post"
+    bash "${DIR}/db_probe.sh" seed "${PROBE_PRE_TOKEN}"
+fi
+
 echo "Forcing a post-deploy backup run (the service-loader pre-state backup can predate the app volumes and be empty)"
 if ! timeout "${BKP_TEST_HEALTH_TIMEOUT}" systemctl start "${BKP_TEST_SERVICE}"; then
     echo "backup unit start returned non-zero; inspecting result"
 fi
 wait_service_terminated
 
-MACHINE_HASH="$(sha256sum /etc/machine-id | cut -c1-64)"
 MACHINE_DIR="${BKP_TEST_BACKUPS_DIR%/}/${MACHINE_HASH}"
 
 if [[ ! -d "${MACHINE_DIR}" ]]; then
@@ -86,7 +99,6 @@ if (( COUNT < 1 )); then
     exit 1
 fi
 
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 export MACHINE_HASH REPO_DIR REPO_NAME
 
 if [[ "${ASYNC_ENABLED}" == "true" ]]; then
@@ -111,6 +123,29 @@ fi
 
 export NEWEST_GENERATION="${GENERATIONS[-1]}"
 bash "${DIR}/verify_backup.sh"
+
+if [[ "${SWARM_ACTIVE}" == "true" ]]; then
+    mapfile -t SQL_FILES < <(find "${REPO_DIR}/${NEWEST_GENERATION}" -mindepth 3 -maxdepth 3 \
+        -type f -path '*/sql/*.backup.sql' ! -name '*.cluster.backup.sql' | sort)
+    if (( ${#SQL_FILES[@]} < 1 )); then
+        echo "SWARM SYNC PASS COMPLETE: backup verified; no single-database dumps require a restore"
+        exit 0
+    fi
+
+    bash "${DIR}/db_probe.sh" seed "${PROBE_POST_TOKEN}"
+    umask 077
+    {
+        printf 'MACHINE_HASH=%q\n' "${MACHINE_HASH}"
+        printf 'REPO_NAME=%q\n' "${REPO_NAME}"
+        printf 'NEWEST_GENERATION=%q\n' "${NEWEST_GENERATION}"
+        printf 'PROBE_PRE_TOKEN=%q\n' "${PROBE_PRE_TOKEN}"
+        printf 'PROBE_POST_TOKEN=%q\n' "${PROBE_POST_TOKEN}"
+    } >"${HANDOFF_PENDING}.tmp"
+    mv "${HANDOFF_PENDING}.tmp" "${HANDOFF_PENDING}"
+    echo "SWARM SYNC PASS COMPLETE: destructive database restore handed to the orchestrated CI DR drill"
+    exit 0
+fi
+
 bash "${DIR}/restore_cycle.sh"
 bash "${DIR}/db_restore.sh"
 echo "SYNC PASS COMPLETE: backup verified and restore cycle succeeded"

roles/svc-bkp-volume-2-local/templates/test.env.j2

@@ -5,3 +5,4 @@ BKP_TEST_HEALTH_TIMEOUT={{ lookup('timeout', 1200) | int }}
 BKP_TEST_IS_STACK_HOST={{ IS_STACK_HOST | bool | lower }}
 BKP_TEST_RESTORE_BIN=baudolo-restore
 BKP_TEST_SERVICE={{ SYS_SERVICE_BACKUP_VOLUME_2_LOCAL }}
+BKP_TEST_SWARM_DRILL={{ TEST_E2E_SWARM_DRILL | default(false) | bool | lower }}

scripts/tests/deploy/swarm/routine/backup/04_quiesce_database_writers.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MODE="${1:?usage: 04_quiesce_database_writers.sh manager|worker}"
+
+is_infrastructure_image() {
+	local image="${1,,}" repository
+	repository="${image##*/}"
+	repository="${repository%%:*}"
+	repository="${repository%%@*}"
+	case "${repository}" in
+	postgres | postgres_* | postgis | postgis_* | mariadb | mariadb_* | mysql | mysql_* | registry) return 0 ;;
+	*) return 1 ;;
+	esac
+}
+
+if [[ "${MODE}" == "manager" ]]; then
+	mapfile -t STACKS < <(docker stack ls --format '{{.Name}}' | sort)
+	REMOVED=()
+	for stack in "${STACKS[@]}"; do
+		mapfile -t IMAGES < <(docker stack services "${stack}" --format '{{.Image}}')
+		keep=true
+		if ((${#IMAGES[@]} < 1)); then
+			keep=false
+		fi
+		for image in "${IMAGES[@]}"; do
+			is_infrastructure_image "${image}" || keep=false
+		done
+		if [[ "${keep}" == "true" ]]; then
+			echo "KEEP: infrastructure stack ${stack} (${IMAGES[*]})"
+			continue
+		fi
+		docker stack rm "${stack}"
+		REMOVED+=("${stack}")
+	done
+
+	deadline=$(($(date +%s) + 180))
+	for stack in "${REMOVED[@]}"; do
+		while [[ -n "$(docker service ls -q --filter "label=com.docker.stack.namespace=${stack}")" ]]; do
+			if (($(date +%s) >= deadline)); then
+				echo "FAIL: stack ${stack} still has services after quiesce timeout"
+				exit 1
+			fi
+			sleep 2
+		done
+		echo "OK: removed writer stack ${stack}"
+	done
+elif [[ "${MODE}" != "worker" ]]; then
+	echo "FAIL: unsupported quiesce mode '${MODE}'"
+	exit 1
+fi
+
+STOP_CONTAINERS=()
+while IFS=$'\t' read -r container_id image; do
+	[[ -n "${container_id}" ]] || continue
+	if ! is_infrastructure_image "${image}"; then
+		STOP_CONTAINERS+=("${container_id}")
+	fi
+done < <(docker ps --format '{{.ID}}\t{{.Image}}')
+if ((${#STOP_CONTAINERS[@]} > 0)); then
+	docker stop "${STOP_CONTAINERS[@]}"
+fi
+echo "OK: ${MODE} runs only database engines and the local registry"

scripts/tests/deploy/swarm/routine/backup/06_restore_databases.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+TEST_DIR="${1:?role test directory required}"
+BACKUPS_ROOT="${2:?backup root required}"
+PENDING="${TEST_DIR}/swarm-restore.pending"
+COMPLETE="${TEST_DIR}/swarm-restore.complete"
+MANIFEST="${TEST_DIR}/swarm-restore.manifest"
+
+if [[ ! -f "${PENDING}" ]]; then
+	echo "FAIL: database restore handoff missing at ${PENDING}"
+	exit 1
+fi
+# shellcheck disable=SC1091
+. "${TEST_DIR}/test.env"
+# shellcheck disable=SC1090
+. "${PENDING}"
+
+if [[ "${BKP_TEST_SWARM_DRILL}" != "true" ]]; then
+	echo "FAIL: database restore handoff was not created by the Swarm CI drill"
+	exit 1
+fi
+
+BKP_TEST_BACKUPS_DIR="${BACKUPS_ROOT%/}"
+REPO_DIR="${BKP_TEST_BACKUPS_DIR}/${MACHINE_HASH}/${REPO_NAME}"
+BKP_TEST_RESTORED_DATABASES_FILE="${MANIFEST}"
+export BKP_TEST_BACKUPS_DIR REPO_DIR BKP_TEST_RESTORED_DATABASES_FILE
+export MACHINE_HASH REPO_NAME NEWEST_GENERATION
+
+if [[ ! -d "${REPO_DIR}/${NEWEST_GENERATION}" ]]; then
+	echo "FAIL: handed-off database backup generation missing at ${REPO_DIR}/${NEWEST_GENERATION}"
+	exit 1
+fi
+
+bash "${TEST_DIR}/db_restore.sh"
+if [[ ! -s "${MANIFEST}" ]]; then
+	echo "FAIL: database restore completed without a restored-database manifest"
+	exit 1
+fi
+mv "${PENDING}" "${COMPLETE}"
+echo "OK: consumed Swarm database restore handoff"

scripts/tests/deploy/swarm/routine/backup/07_verify_databases.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+TEST_DIR="${1:?role test directory required}"
+COMPLETE="${TEST_DIR}/swarm-restore.complete"
+MANIFEST="${TEST_DIR}/swarm-restore.manifest"
+
+if [[ ! -f "${COMPLETE}" ]]; then
+	echo "FAIL: completed database restore handoff missing at ${COMPLETE}"
+	exit 1
+fi
+# shellcheck disable=SC1091
+. "${TEST_DIR}/test.env"
+# shellcheck disable=SC1090
+. "${COMPLETE}"
+
+if [[ "${BKP_TEST_SWARM_DRILL}" != "true" ]]; then
+	echo "FAIL: completed database restore was not produced by the Swarm CI drill"
+	exit 1
+fi
+
+bash "${TEST_DIR}/db_probe.sh" verify \
+	"${PROBE_PRE_TOKEN}" "${PROBE_POST_TOKEN}" "${MANIFEST}"

scripts/tests/deploy/swarm/routine/backup/README.md

@@ -6,6 +6,28 @@ the live instance; the drill passes only when every marker survives the
 whole loop. `base.sh` runs the nine steps below; the numbers in both
 diagrams are its `[n/9]` log markers.
 
+## Schema
+
+The CI drill includes destructive database recovery. The disposable test
+cluster keeps only the database engines and local registry available while
+every database-writing workload, including shared dependencies such as
+Discourse, is stopped. The normal matrix update remains the single source of
+truth for bringing the workloads back.
+
+```mermaid
+flowchart LR
+    seed["Seed pre-backup database probes"]
+    backup["Run deployed backup units"]
+    mutate["Write post-backup probes"]
+    quiesce["Remove all database writers<br/>target stack + shared dependencies"]
+    files["Recover NFS, volumes and secrets"]
+    database["Restore SQL dumps into<br/>the real CI databases"]
+    redeploy["Matrix update redeploys<br/>the desired state"]
+    verify["Health + Playwright<br/>pre-backup probes present<br/>post-backup probes absent"]
+
+    seed --> backup --> mutate --> quiesce --> files --> database --> redeploy --> verify
+```
+
 ## Step sequence (who does what)
 
 ```mermaid
@@ -13,10 +35,13 @@ sequenceDiagram
     autonumber
     participant N as nfs-server
     participant M as manager
+    participant D as database engines
     participant B as backup host
     participant U as LUKS "USB" (loop img on B)
 
-    Note over N,M: [1/9] seed markers
+    Note over M,D: role test seeds pre-backup DB probes, backs up, then seeds post-backup probes
+
+    Note over N,M: [1/9] seed file markers
     N->>N: marker → live NFS volume dir
     M->>M: marker → host secrets dir
 
@@ -34,11 +59,11 @@ sequenceDiagram
     Note over B,U: [5/9] plug LUKS device, sync via local-2-device unit
     B->>U: hard-linked snapshot
 
-    Note over M: [6/9] full disaster teardown
-    alt stack workload
-        M->>M: docker stack rm + wait
-    else node-local workload
-        M->>M: stop compose project on every node
+    Note over M,D: [6/9] quiesce writers
+    alt database handoff exists
+        M->>M: remove all non-DB/non-registry stacks and containers
+    else file-only drill
+        M->>M: remove the target stack or node-local compose project
     end
 
     Note over B,U: [7/9] recover device → local root
@@ -53,7 +78,11 @@ sequenceDiagram
     B-->>M: volume generation → recover CLI
     B-->>M: secrets generation → recover CLI
 
-    Note over M,N: matrix update pass redeploys, then verify_recovered_marker.sh asserts the live marker
+    Note over M,D: [9b/9] restore selected SQL generation
+    B-->>M: device-recovered volume backup repository
+    M->>D: replay dumps while all writers remain stopped
+
+    Note over M,N: matrix update redeploys; health, Playwright, file markers and DB before/after probes must pass
 ```
 
 ## Data flow (what is proven)
@@ -64,33 +93,39 @@ flowchart TB
         direction TB
         vol["📂 [1] Live NFS volume + docker volume<br/>marker seeded"]:::live
         sec["🔑 [1] Host secrets on the manager<br/>(secrets, CA, ACME, node identity) marker seeded"]:::live
+        dbpre["🗃️ Pre-backup DB probe<br/>included in the selected SQL generation"]:::live
         localbkp["🗄️ [2-3] Local backups on manager + NFS server<br/>volume / nfs / secrets generation snapshots"]:::store
         pulled["🗄️ [4] Backup host<br/>pulled generations (remote-2-local unit, rsync over ssh, pull-only)"]:::store
         usb["🔒 [5] Encrypted USB (LUKS)<br/>hard-linked snapshot (local-2-device unit)"]:::device
         vol -->|backup units| localbkp
         sec -->|secrets unit| localbkp
+        dbpre -->|SQL dump| localbkp
         localbkp -->|remote pull| pulled -->|device sync| usb
     end
 
-    teardown["🛑 [6] Full disaster:<br/>stack rm (stack workload) or<br/>compose stop on every node (node-local workload),<br/>then export wiped"]:::dead
+    dbpost["✍️ Post-backup DB probe<br/>must disappear after restore"]:::dead
+    teardown["🛑 [6] Full disaster:<br/>all DB writers removed, including shared dependencies;<br/>only DB engines + registry remain,<br/>then export wiped"]:::dead
 
-    subgraph recovery["♻️ Recovery (dashed: the recover CLI, newest generation only)"]
+    subgraph recovery["♻️ Recovery (dashed: newest file snapshots + handed-off SQL generation)"]
         direction TB
         restored["📁 [7] Restore root on the backup host<br/>newest snapshot after luksOpen (pulled tree + loop img freed)"]:::store
         export["📂 [8] Live NFS export<br/>volume subtree rebuilt, ganesha + client mounts refreshed"]:::live
         volume["📦 [9] Docker volume<br/>rebuilt on the manager"]:::live
         secrets["🔑 [9] Host secrets<br/>rebuilt on the manager (marker cleared first)"]:::live
-        verified["✅ Update pass redeploys<br/>markers verified on the live instance"]:::ok
+        database["🗃️ [9b] Live databases<br/>selected SQL generation replayed without writers"]:::live
+        verified["✅ Update pass redeploys<br/>health + Playwright pass;<br/>file markers and DB before/after probes verified"]:::ok
         usb -.->|decrypt + recover| restored
         restored -.->|restore export| export
         restored -.->|restore volume| volume
         restored -.->|restore secrets| secrets
+        restored -.->|restore SQL dumps| database
         export -.-> verified
         volume -.-> verified
         secrets -.-> verified
+        database -.-> verified
     end
 
-    forward --> teardown --> recovery
+    forward --> dbpost --> teardown --> recovery
 
     classDef live fill:#f1f8f6,stroke:#d5e7e2,color:#455a64;
     classDef store fill:#f3f7fb,stroke:#d6e2ee,color:#455a64;
@@ -103,13 +138,16 @@ flowchart TB
 
 ## Applicability
 
-- The drill runs whenever the app declares an NFS-flagged volume
-  (`PRIMARY_NFS_VOLUME`); apps without one skip it, since there is nothing
-  to prove a restore against.
-- The workload form only changes step 6: stack workloads get
-  `docker stack rm`, `workload: node-local` roles get their compose
-  project stopped on every node. Every other step operates on volumes,
-  the NFS export and the deployed backup units, independent of the
-  orchestrator.
-- Runs once per app, in the variant-0 CI job, between the matrix's first
-  deploy and its update pass.
+- The first matrix round runs the complete file chain whenever the app
+  declares an NFS-flagged volume (`PRIMARY_NFS_VOLUME`). A database handoff
+  is destructive even without NFS: all writers are quiesced and the selected
+  manager-local generation is restored.
+- Later variant rounds skip the expensive file chain. If their role test
+  hands off SQL dumps, they still run the destructive database restore and
+  post-update probe verification; no Swarm restore is replaced by a guard.
+- In a file-only drill, the workload form changes step 6: stack workloads
+  get `docker stack rm`, while `workload: node-local` roles have their
+  Compose project stopped on every node. A database handoff instead removes
+  every non-database workload in either form.
+- Runs between the sync and update pass. The full NFS/device chain runs once
+  per app; database recovery runs in every round that produced a handoff.

scripts/tests/deploy/swarm/routine/backup/base.sh

@@ -1,21 +1,4 @@
 #!/usr/bin/env bash
-# Disaster-recovery drill for the swarm test cluster: proves the full
-# backup chain volume + secrets + nfs -> remote -> device through the
-# DEPLOYED systemd units on every host (the backup host runs the real
-# svc-bkp-remote-2-local and svc-bkp-local-2-device roles; the drill only
-# installs the ssh pull identity and simulates the USB plug via a LUKS
-# loop mount), then tears the stack down completely and recovers the same
-# data back through the recover CLI (cli.administration.recover: device ->
-# local root -> nfs export, docker volume and host secrets). The matrix update
-# pass then boots the stack onto the recovered export and
-# verify_recovered_marker.sh asserts the live marker, so no dedicated redeploy
-# runs here. Runs once, between the matrix's first and second round, against
-# the already-converged round-1 stack. Per-host
-# routines live next to this file and execute in-node from the repo copy
-# under INFINITO_NODE_SRC_DIR (one docker exec per routine). Marker probes
-# are scoped per repo (volume/nfs/secrets share DR_MARKER; an unscoped
-# ${MID} glob would cross-select). Device paths (mount/target) come from
-# the same extras SPOT that configures the deployed role (write_extras).
 set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
@@ -45,14 +28,64 @@ USB_IMG="/var/lib/infinito-usb.img"
 USB_MAPPER="usbdrill"
 USB_PASS="drillpass"
 RESTORE_ROOT="/var/tmp/dr-device-restored"
+ROLE_TEST_DIR="/tmp/test-e2e-cli/svc-bkp-volume-2-local"
+DB_PENDING_FILE="${ROLE_TEST_DIR}/swarm-restore.pending"
+DB_RESTORE_ROOT="/var/tmp/dr-database-restore-${APP_ID}"
+DRILL_FULL_CHAIN="${DRILL_FULL_CHAIN:-true}"
 DEV_MOUNT="$(python3 -c "import sys, yaml; print(yaml.safe_load(open(sys.argv[1]))['applications']['svc-bkp-local-2-device']['services']['local-2-device']['mount'])" "${DRILL_EXTRAS}")"
 DEV_TARGET="$(python3 -c "import sys, yaml; print(yaml.safe_load(open(sys.argv[1]))['applications']['svc-bkp-local-2-device']['services']['local-2-device']['target'])" "${DRILL_EXTRAS}")"
 DEV_DEST="${DEV_MOUNT}${DEV_TARGET}"
 
-if [ -z "${PRIMARY_NFS_VOLUME}" ]; then
-	echo "SKIP drill: ${APP_ID} declares no NFS-flagged volume — nothing to prove a restore against"
+case "${DRILL_FULL_CHAIN}" in
+true | false) ;;
+*)
+	echo "FAILURE: DRILL_FULL_CHAIN must be true or false"
+	exit 1
+	;;
+esac
+
+DB_PENDING=false
+if docker exec "${MGR}" test -f "${DB_PENDING_FILE}"; then
+	DB_PENDING=true
+fi
+MGR_MID="$(docker exec "${MGR}" sha256sum /etc/machine-id | cut -c1-64)"
+
+quiesce_database_writers() {
+	docker exec "${MGR}" bash "${BKP_IN_NODE}/04_quiesce_database_writers.sh" manager
+	for _node in "${WRK1}" "${WRK2}"; do
+		docker exec "${_node}" bash "${BKP_IN_NODE}/04_quiesce_database_writers.sh" worker
+	done
+}
+
+restore_databases() {
+	local backups_root="${1}"
+	docker exec "${MGR}" bash "${BKP_IN_NODE}/06_restore_databases.sh" \
+		"${ROLE_TEST_DIR}" "${backups_root}"
+}
+
+write_verify_env() {
+	local db_verify="${1}" nfs_verify="${2}"
+	printf 'DR_DB_VERIFY=%s\nDR_NFS_VERIFY=%s\nMGR=%s\nROLE_TEST_DIR=%s\nBKP_IN_NODE=%s\n' \
+		"${db_verify}" "${nfs_verify}" "${MGR}" "${ROLE_TEST_DIR}" "${BKP_IN_NODE}" >"${DR_VERIFY_ENV}"
+	if [ "${nfs_verify}" = true ]; then
+		printf 'DR_TOKEN=%s\nDR_MARKER=%s\nNFS_SERVER=%s\nNFS_VOL_DIR=%s\n' \
+			"${DR_TOKEN}" "${DR_MARKER}" "${NFS_SERVER}" "${NFS_VOL_DIR}" >>"${DR_VERIFY_ENV}"
+	fi
+}
+
+if [ "${DRILL_FULL_CHAIN}" = false ] || [ -z "${PRIMARY_NFS_VOLUME}" ]; then
+	if [ "${DB_PENDING}" != true ]; then
+		echo "SKIP drill: no database handoff and no full NFS recovery requested for ${APP_ID}"
+		exit 0
+	fi
+	echo "==> destructive database-only DR drill for ${APP_ID}"
+	quiesce_database_writers
+	restore_databases "${DIR_BACKUPS}"
+	write_verify_env true false
+	echo "==> database recovery complete; the matrix update redeploys and verifies the restored probes"
 	exit 0
 fi
+
 if ! docker inspect "${BACKUP_NODE}" >/dev/null 2>&1; then
 	echo "FAILURE: backup host ${BACKUP_NODE} is not running (01_bootstrap.sh did not run?)"
 	exit 1
@@ -63,7 +96,6 @@ if [ ! -f "${BACKUP_KEY_PATH}" ]; then
 fi
 
 NFS_VOL_DIR="${NFS_STATE_PATH}/${PRIMARY_NFS_VOLUME}"
-MGR_MID="$(docker exec "${MGR}" sha256sum /etc/machine-id | cut -c1-64)"
 NFS_MID="$(docker exec "${NFS_SERVER}" sha256sum /etc/machine-id | cut -c1-64)"
 echo "==> DR drill for ${APP_ID} (volume '${PRIMARY_NFS_VOLUME}')"
 
@@ -152,8 +184,10 @@ if ! docker exec "${BACKUP_NODE}" find "${DEV_DEST}" -name "${DR_MARKER}" 2>/dev
 fi
 echo "    marker present on encrypted USB"
 
-echo "==> [6/9] tear the stack down completely (full disaster) before recovery"
-if [ "${HAS_SWARM_SERVICE}" = true ]; then
+echo "==> [6/9] quiesce workloads before recovery"
+if [ "${DB_PENDING}" = true ]; then
+	quiesce_database_writers
+elif [ "${HAS_SWARM_SERVICE}" = true ]; then
 	docker exec "${MGR}" bash "${BKP_IN_NODE}/04_stack_rm_wait.sh" "${STACK_NAME}"
 else
 	for _node in "${MGR}" "${WRK1}" "${WRK2}"; do
@@ -251,7 +285,18 @@ else
 	echo "    secrets recover skipped: svc-bkp-secrets-2-local not installed on ${MGR}"
 fi
 
-echo "==> recovery complete: device -> nfs export -> volume -> secrets restored via the recover CLI"
-echo "    the matrix update pass boots the stack onto the recovered export; verify_recovered_marker.sh asserts the live marker there"
-printf 'DR_TOKEN=%s\nDR_MARKER=%s\nNFS_SERVER=%s\nNFS_VOL_DIR=%s\n' \
-	"${DR_TOKEN}" "${DR_MARKER}" "${NFS_SERVER}" "${NFS_VOL_DIR}" >"${DR_VERIFY_ENV}"
+if [ "${DB_PENDING}" = true ]; then
+	echo "==> [9b/9] restore the handed-off SQL generation from the device-recovered tree"
+	if ! docker exec "${BACKUP_NODE}" test -d "${RESTORE_ROOT}/${MGR_MID}/${VOLUME_REPO}"; then
+		echo "FAILURE: ${VOLUME_REPO} missing from the device-recovered manager tree"
+		exit 1
+	fi
+	docker exec "${MGR}" sh -c "rm -rf '${DB_RESTORE_ROOT}'; mkdir -p '${DB_RESTORE_ROOT}'"
+	docker exec "${BACKUP_NODE}" tar -C "${RESTORE_ROOT}" -cf - "${MGR_MID}/${VOLUME_REPO}" |
+		docker exec -i "${MGR}" tar --numeric-owner -C "${DB_RESTORE_ROOT}" -xf -
+	restore_databases "${DB_RESTORE_ROOT}"
+fi
+
+echo "==> recovery complete: device -> nfs export -> volume -> secrets -> databases restored"
+echo "    the matrix update pass redeploys every workload and verifies the recovered markers and database probes"
+write_verify_env "${DB_PENDING}" true

scripts/tests/deploy/swarm/routine/backup/verify_recovered_marker.sh

@@ -1,8 +1,4 @@
 #!/usr/bin/env bash
-# Runs on the controller after the matrix update pass. Asserts the DR-drill
-# marker (seeded pre-backup) survived recovery plus the update-pass bring-up
-# onto the recovered NFS export. Skips when the sentinel is absent (the drill
-# recovered nothing: the app declares no NFS-flagged volume).
 set -euo pipefail
 
 DR_VERIFY_ENV="/tmp/dr-drill-verify-${APP_ID:?APP_ID required (matrix sets it)}.env"
@@ -13,8 +9,15 @@ fi
 # shellcheck disable=SC1090
 . "${DR_VERIFY_ENV}"
 
-if ! docker exec "${NFS_SERVER}" grep -qF "${DR_TOKEN}" "${NFS_VOL_DIR}/${DR_MARKER}" 2>/dev/null; then
-	echo "FAILURE: DR-drill marker missing on the live volume after recover + update pass"
-	exit 1
+if [ "${DR_NFS_VERIFY:?}" = true ]; then
+	if ! docker exec "${NFS_SERVER}" grep -qF "${DR_TOKEN}" "${NFS_VOL_DIR}/${DR_MARKER}" 2>/dev/null; then
+		echo "FAILURE: DR-drill marker missing on the live volume after recover + update pass"
+		exit 1
+	fi
 fi
-echo "==> DR drill PASSED: backup -> teardown -> recover device->nfs->volume->secrets -> live marker verified after the update pass"
+
+if [ "${DR_DB_VERIFY:?}" = true ]; then
+	docker exec "${MGR}" bash "${BKP_IN_NODE}/07_verify_databases.sh" "${ROLE_TEST_DIR}"
+fi
+
+echo "==> DR drill PASSED: recovered NFS=${DR_NFS_VERIFY}, databases=${DR_DB_VERIFY}; post-update probes verified"

tests/unit/utils/tests/swarm/test_backup_drill.py

@@ -0,0 +1,235 @@
+from __future__ import annotations
+
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+from utils import PROJECT_ROOT
+
+
+BACKUP_SCRIPTS = PROJECT_ROOT / "scripts/tests/deploy/swarm/routine/backup"
+ROLE_TEST_SCRIPTS = PROJECT_ROOT / "roles/svc-bkp-volume-2-local/files/test"
+
+
+class TestDatabaseRestoreHandoff(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.root = Path(self.tmp.name)
+        self.test_dir = self.root / "role-test"
+        self.test_dir.mkdir()
+        (self.test_dir / "test.env").write_text(
+            "BKP_TEST_SWARM_DRILL=true\nBKP_TEST_DATABASES_CSV=/unused\n"
+        )
+
+    def test_restore_consumes_handoff_only_after_manifest_is_written(self):
+        backups = self.root / "backups"
+        (backups / "machine" / "repo" / "generation").mkdir(parents=True)
+        (self.test_dir / "swarm-restore.pending").write_text(
+            "MACHINE_HASH=machine\n"
+            "REPO_NAME=repo\n"
+            "NEWEST_GENERATION=generation\n"
+            "PROBE_PRE_TOKEN=before\n"
+            "PROBE_POST_TOKEN=after\n"
+        )
+        restore = self.test_dir / "db_restore.sh"
+        restore.write_text(
+            "#!/usr/bin/env bash\n"
+            "set -euo pipefail\n"
+            ": \"${BKP_TEST_BACKUPS_DIR:?}\" \"${REPO_DIR:?}\"\n"
+            "printf 'postgres;postgres_data;app\\n' >\"${BKP_TEST_RESTORED_DATABASES_FILE}\"\n"
+        )
+        restore.chmod(0o755)
+
+        subprocess.run(
+            [
+                "bash",
+                str(BACKUP_SCRIPTS / "06_restore_databases.sh"),
+                str(self.test_dir),
+                str(backups),
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+
+        self.assertFalse((self.test_dir / "swarm-restore.pending").exists())
+        self.assertTrue((self.test_dir / "swarm-restore.complete").is_file())
+        self.assertTrue((self.test_dir / "swarm-restore.manifest").is_file())
+
+    def test_verify_passes_completed_probe_tokens_to_role_helper(self):
+        (self.test_dir / "swarm-restore.complete").write_text(
+            "PROBE_PRE_TOKEN=before\nPROBE_POST_TOKEN=after\n"
+        )
+        manifest = self.test_dir / "swarm-restore.manifest"
+        manifest.write_text("postgres;postgres_data;app\n")
+        verified = self.root / "probe-verified"
+        probe = self.test_dir / "db_probe.sh"
+        probe.write_text(
+            "#!/usr/bin/env bash\n"
+            "set -euo pipefail\n"
+            "test \"$1\" = verify\n"
+            "test \"$2\" = before\n"
+            "test \"$3\" = after\n"
+            f"test \"$4\" = {manifest}\n"
+            "touch \"${PROBE_VERIFIED:?}\"\n"
+        )
+        probe.chmod(0o755)
+        env = os.environ.copy()
+        env["PROBE_VERIFIED"] = str(verified)
+
+        subprocess.run(
+            [
+                "bash",
+                str(BACKUP_SCRIPTS / "07_verify_databases.sh"),
+                str(self.test_dir),
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+            env=env,
+        )
+
+        self.assertTrue(verified.is_file())
+
+
+class TestDatabaseRestoreProbe(unittest.TestCase):
+    def test_seeds_and_verifies_postgres_and_mariadb(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            csv_file = root / "databases.csv"
+            csv_file.write_text(
+                "instance;database;username;password\n"
+                "postgres;app_pg;user_pg;pw_pg\n"
+                "mariadb;app_my;user_my;pw_my\n"
+            )
+            manifest = root / "manifest"
+            manifest.write_text(
+                "postgres;postgres_data;app_pg\n"
+                "mariadb;mariadb_data;app_my\n"
+            )
+            fake_bin = root / "bin"
+            fake_bin.mkdir()
+            container = fake_bin / "container"
+            container.write_text(
+                "#!/usr/bin/env bash\n"
+                "set -euo pipefail\n"
+                "case \"$1\" in\n"
+                "  ps)\n"
+                "    case \"$*\" in\n"
+                "      *postgres_data*) printf 'postgres.1\\n' ;;\n"
+                "      *mariadb_data*) printf 'mariadb.1\\n' ;;\n"
+                "    esac\n"
+                "    ;;\n"
+                "  inspect)\n"
+                "    case \"${*: -1}\" in\n"
+                "      postgres.1) printf 'postgres_custom\\n' ;;\n"
+                "      mariadb.1) printf 'mariadb_custom\\n' ;;\n"
+                "    esac\n"
+                "    ;;\n"
+                "  exec)\n"
+                "    case \"$*\" in\n"
+                "      *SELECT*before*)\n"
+                "        test -f \"${DB_STATE:?}/seeded-${4}\" && printf '1\\n' || printf '0\\n'\n"
+                "        ;;\n"
+                "      *SELECT*after*) printf '0\\n' ;;\n"
+                "      *) touch \"${DB_STATE:?}/seeded-${4}\" ;;\n"
+                "    esac\n"
+                "    ;;\n"
+                "esac\n"
+            )
+            container.chmod(0o755)
+            env = os.environ.copy()
+            env["PATH"] = f"{fake_bin}:{env['PATH']}"
+            env["BKP_TEST_DATABASES_CSV"] = str(csv_file)
+            env["DB_STATE"] = str(root)
+
+            subprocess.run(
+                ["bash", str(ROLE_TEST_SCRIPTS / "db_probe.sh"), "seed", "before"],
+                check=True,
+                capture_output=True,
+                text=True,
+                env=env,
+            )
+            self.assertTrue((root / "seeded-postgres.1").is_file())
+            self.assertTrue((root / "seeded-mariadb.1").is_file())
+
+            subprocess.run(
+                [
+                    "bash",
+                    str(ROLE_TEST_SCRIPTS / "db_probe.sh"),
+                    "verify",
+                    "before",
+                    "after",
+                    str(manifest),
+                ],
+                check=True,
+                capture_output=True,
+                text=True,
+                env=env,
+            )
+
+
+class TestDatabaseWriterQuiesce(unittest.TestCase):
+    def test_keeps_only_database_registry_and_stops_discourse(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            fake_bin = root / "bin"
+            fake_bin.mkdir()
+            docker = fake_bin / "docker"
+            docker.write_text(
+                "#!/usr/bin/env bash\n"
+                "set -euo pipefail\n"
+                "case \"$1 $2\" in\n"
+                "  'stack ls') printf 'docker\\npostgres\\nwordpress\\n' ;;\n"
+                "  'stack services')\n"
+                "    case \"$3\" in\n"
+                "      docker) printf 'registry:2\\n' ;;\n"
+                "      postgres) printf 'localhost:5000/postgres_custom:17\\n' ;;\n"
+                "      wordpress) printf 'wordpress:latest\\n' ;;\n"
+                "    esac\n"
+                "    ;;\n"
+                "  'stack rm') touch \"${DOCKER_STATE:?}/removed-$3\" ;;\n"
+                "  'service ls') : ;;\n"
+                "  'ps --format')\n"
+                "    printf 'db\\tlocalhost:5000/postgres_custom:17\\n'\n"
+                "    printf 'registry\\tregistry:2\\n'\n"
+                "    printf 'discourse\\tlocal_discourse/discourse:latest\\n'\n"
+                "    printf 'wordpress-task\\twordpress:latest\\n'\n"
+                "    ;;\n"
+                "  'stop discourse')\n"
+                "    touch \"${DOCKER_STATE:?}/stopped-$2\" \"${DOCKER_STATE:?}/stopped-$3\"\n"
+                "    ;;\n"
+                "  *) printf 'unexpected docker call: %s\\n' \"$*\" >&2; exit 1 ;;\n"
+                "esac\n"
+            )
+            docker.chmod(0o755)
+            env = os.environ.copy()
+            env["PATH"] = f"{fake_bin}:{env['PATH']}"
+            env["DOCKER_STATE"] = str(root)
+
+            subprocess.run(
+                [
+                    "bash",
+                    str(BACKUP_SCRIPTS / "04_quiesce_database_writers.sh"),
+                    "manager",
+                ],
+                check=True,
+                capture_output=True,
+                text=True,
+                env=env,
+            )
+
+            self.assertTrue((root / "removed-wordpress").is_file())
+            self.assertTrue((root / "stopped-discourse").is_file())
+            self.assertTrue((root / "stopped-wordpress-task").is_file())
+            self.assertFalse((root / "removed-postgres").exists())
+            self.assertFalse((root / "removed-docker").exists())
+            self.assertFalse((root / "stopped-db").exists())
+            self.assertFalse((root / "stopped-registry").exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

utils/tests/swarm/matrix.py

@@ -139,6 +139,8 @@ def _deploy(
         f"@{extras_path}",
         "-e",
         f"VARIANT_INDEX={json.dumps(round_index)}",
+        "-e",
+        "TEST_E2E_SWARM_DRILL=true",
     ]
     pass_label = (
         f"matrix-deploy: round {round_index + 1}/{total} "
@@ -192,11 +194,14 @@ def _converge_and_verify(*, app_id: str) -> int:
     )
 
 
-def _backup_restore_drill(*, app_id: str, inv_dir: str, extras_path: str) -> int:
+def _backup_restore_drill(
+    *, app_id: str, inv_dir: str, extras_path: str, full_chain: bool
+) -> int:
     env = os.environ.copy()
     env["APP_ID"] = app_id
     env["INFINITO_INVENTORY_DIR"] = inv_dir
     env["DRILL_EXTRAS"] = extras_path
+    env["DRILL_FULL_CHAIN"] = str(full_chain).lower()
     return _run(
         ["bash", str(_SWARM_SCRIPTS / "backup" / "base.sh")],
         env=env,
@@ -339,15 +344,18 @@ def main(argv: list[str] | None = None) -> int:
             )
         if rc == 0:
             rc = _converge_and_verify(app_id=app_id)
-        if rc == 0 and round_index == 0:
+        if rc == 0 and plan_index == 0:
             rc = _deploy_backup_host(
                 app_id=app_id,
                 inv_dir=inv_root,
                 extras_path=f"{inv_root}/swarm-nfs-extras.deploy.yml",
             )
-        if rc == 0 and round_index == 0:
+        if rc == 0:
             rc = _backup_restore_drill(
-                app_id=app_id, inv_dir=inv_root, extras_path=extras_path
+                app_id=app_id,
+                inv_dir=inv_root,
+                extras_path=extras_path,
+                full_chain=plan_index == 0,
             )
         if rc == 0:
             rc = _deploy(
@@ -360,7 +368,7 @@ def main(argv: list[str] | None = None) -> int:
             )
         if rc == 0:
             rc = _converge_and_verify(app_id=app_id)
-        if rc == 0 and round_index == 0:
+        if rc == 0:
             rc = _verify_recovered_marker(app_id=app_id)
         if rc != 0:
             return rc