fix(svc-bkp-volume-2-local): fail the drill on a database that was never dumped

The payload check accepted a files/ tree OR an sql dump, so a database whose dump never happened and which was copied as raw engine files passed it. That is exactly what occurred: Discourse's postgres landed as PG_VERSION/base/pg_wal and the drill reported green, while restoring that replays a torn cluster rather than a dump.

The generation's manifest is now the authority on which volumes held a database and which of those carry a dump; generations written before baudolo recorded that are judged by the engine's on-disk footprint instead. Demonstrated against both: the manifest branch catches a volume the footprint heuristic cannot see, and leaves a non-database volume alone.

seed/marker.py reads dumps and therefore needs baudolo, but ran under a bare python3, which Ansible's PATH resolves to the sealed project venv while sys-pip-install installs into the system interpreter - so it died at import and the entire restore half of the drill never ran. It now uses BKP_TEST_PYTHON, the interpreter the role installed into.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

roles/svc-bkp-volume-2-local/files/test/restore_cycle.sh

@@ -22,6 +22,7 @@
 set -euo pipefail
 
 : "${BKP_TEST_HEALTH_TIMEOUT:?}"
+: "${BKP_TEST_PYTHON:?}"
 : "${BKP_TEST_REPO_ROOT:?}"
 : "${BKP_TEST_SCRATCH_IMAGE:?}"
 : "${DR_TOKEN:?}"
@@ -220,7 +221,7 @@ for volume in "${RESTORABLE[@]-}"; do
     fi
 done
 
-python3 "${DIR}/seed/marker.py" blank "${GEN_DIR}" "${DR_TOKEN}"
+"${BKP_TEST_PYTHON}" "${DIR}/seed/marker.py" blank "${GEN_DIR}" "${DR_TOKEN}"
 
 echo "Restoring ${#VOLUME_DIRS[@]} volume(s) from generation ${NEWEST_GENERATION}"
 for volume_dir in "${VOLUME_DIRS[@]}"; do
@@ -285,7 +286,7 @@ if (( ${#DB_PROJECTS[@]} > 0 )); then
     recover database "${GEN_DIR}"
 fi
 
-python3 "${DIR}/seed/marker.py" verify "${GEN_DIR}" "${DR_TOKEN}"
+"${BKP_TEST_PYTHON}" "${DIR}/seed/marker.py" verify "${GEN_DIR}" "${DR_TOKEN}"
 
 DB_SERVICE=()
 echo "Starting every project to completion..."

roles/svc-bkp-volume-2-local/files/test/test.sh

@@ -19,6 +19,7 @@ set -euo pipefail
 : "${BKP_TEST_SERVICE:?}"
 : "${BKP_TEST_HEALTH_SCRIPT:?}"
 : "${BKP_TEST_HEALTH_TIMEOUT:?}"
+: "${BKP_TEST_PYTHON:?}"
 
 if [[ "${BKP_TEST_IS_STACK_HOST}" != "true" ]]; then
     echo "SKIP: not the stack host; svc-bkp-volume-2-local only deploys there"
@@ -119,8 +120,8 @@ bash "${DIR}/verify/backup.sh"
 DR_TOKEN="dr-$(date +%s)-$$"
 SEEDED_GENERATION="${REPO_DIR}/${NEWEST_GENERATION}"
 export DR_TOKEN
-python3 "${DIR}/seed/marker.py" seed "${SEEDED_GENERATION}" "${DR_TOKEN}"
-trap 'python3 "${DIR}/seed/marker.py" clean "${REPO_DIR}/${NEWEST_GENERATION}" "${DR_TOKEN}"' EXIT
+"${BKP_TEST_PYTHON}" "${DIR}/seed/marker.py" seed "${SEEDED_GENERATION}" "${DR_TOKEN}"
+trap '"${BKP_TEST_PYTHON}" "${DIR}/seed/marker.py" clean "${REPO_DIR}/${NEWEST_GENERATION}" "${DR_TOKEN}"' EXIT
 
 echo "Backing up once more so the generation under test carries the marker"
 if ! timeout "${BKP_TEST_HEALTH_TIMEOUT}" systemctl start "${BKP_TEST_SERVICE}"; then
@@ -139,7 +140,7 @@ if ! diff <(subjects "${SEEDED_GENERATION}") <(subjects "${REPO_DIR}/${NEWEST_GE
     exit 1
 fi
 
-python3 "${DIR}/seed/marker.py" captured "${REPO_DIR}/${NEWEST_GENERATION}" "${DR_TOKEN}"
+"${BKP_TEST_PYTHON}" "${DIR}/seed/marker.py" captured "${REPO_DIR}/${NEWEST_GENERATION}" "${DR_TOKEN}"
 
 if [[ "$(container info --format '{{.Swarm.LocalNodeState}}' 2>/dev/null)" == "active" ]]; then
     echo "SKIP: swarm node; the restore half needs compose lifecycle control, so it is drilled by scripts/tests/deploy/swarm/routine/backup"

roles/svc-bkp-volume-2-local/files/test/verify/backup.sh

@@ -1,11 +1,16 @@
 #!/usr/bin/env bash
-# Verify the newest backup generation is stored and non-empty (a volume counts
-# as payload with a files/ tree or a non-empty sql dump).
-# Requires REPO_DIR/NEWEST_GENERATION from test.sh.
+# Verify the newest backup generation is stored, non-empty, and dumped where a
+# dump was owed: a volume counts as payload with a files/ tree or a non-empty
+# sql dump, and a volume the run recorded as a database must carry the dump.
+# The generation's own manifest is the authority; a generation written before
+# baudolo wrote one is judged by the engine's on-disk footprint instead.
+# Requires REPO_DIR/NEWEST_GENERATION/BKP_TEST_REPO_ROOT from test.sh.
 set -euo pipefail
 
 : "${REPO_DIR:?}"
 : "${NEWEST_GENERATION:?}"
+: "${BKP_TEST_REPO_ROOT:?}"
+: "${BKP_TEST_PYTHON:?}"
 
 GEN_DIR="${REPO_DIR}/${NEWEST_GENERATION}"
 
@@ -16,17 +21,64 @@ if (( ${#VOLUME_DIRS[@]} < 1 )); then
 fi
 echo "OK: generation ${NEWEST_GENERATION} stores ${#VOLUME_DIRS[@]} volume(s)"
 
+DB_ENGINE_MARKERS=(
+    "postgres:PG_VERSION"
+    "mariadb:ibdata1"
+    "mariadb:mysql/user.frm"
+    "mariadb:mysql/user.ibd"
+)
+
+engine_of() {
+    local files_dir="$1" marker
+    for marker in "${DB_ENGINE_MARKERS[@]}"; do
+        if [[ -e "${files_dir}/${marker#*:}" ]]; then
+            echo "${marker%%:*}"
+            return 0
+        fi
+    done
+    return 1
+}
+
+MANIFEST_LINES=""
+if MANIFEST_LINES="$(PYTHONPATH="${BKP_TEST_REPO_ROOT}" "${BKP_TEST_PYTHON}" -m utils.recovery.manifest "${GEN_DIR}")"; then
+    FROM_MANIFEST=true
+    echo "OK: generation ${NEWEST_GENERATION} carries a manifest; reading its verdict"
+else
+    FROM_MANIFEST=false
+    echo "NOTE: generation ${NEWEST_GENERATION} predates the manifest; judging by the engine footprint"
+fi
+
 EMPTY=0
+UNDUMPED=()
 for vol_dir in "${VOLUME_DIRS[@]}"; do
     has_files="$(find "${vol_dir}/files" -mindepth 1 -print -quit 2>/dev/null || true)"
     has_sql="$(find "${vol_dir}/sql" -mindepth 1 -type f -name '*.backup.sql' -size +0 -print -quit 2>/dev/null || true)"
     if [[ -z "${has_files}" ]] && [[ -z "${has_sql}" ]]; then
         echo "WARN: ${vol_dir##*/} backed up empty"
         EMPTY=$((EMPTY + 1))
+        continue
+    fi
+    if [[ "${FROM_MANIFEST}" == "false" ]] \
+        && engine="$(engine_of "${vol_dir}/files")" \
+        && [[ -z "${has_sql}" ]]; then
+        UNDUMPED+=("${vol_dir##*/} (${engine})")
     fi
 done
+
+if [[ "${FROM_MANIFEST}" == "true" ]] && [[ -n "${MANIFEST_LINES}" ]]; then
+    while IFS=$'\t' read -r volume engine; do
+        if [[ -n "${volume}" ]]; then
+            UNDUMPED+=("${volume} (${engine})")
+        fi
+    done <<< "${MANIFEST_LINES}"
+fi
 if (( EMPTY == ${#VOLUME_DIRS[@]} )); then
     echo "FAIL: every backed-up volume in ${NEWEST_GENERATION} is empty"
     exit 1
 fi
-echo "OK: backup payload present ($(( ${#VOLUME_DIRS[@]} - EMPTY )) non-empty volume(s))"
+if (( ${#UNDUMPED[@]} > 0 )); then
+    echo "FAIL: ${#UNDUMPED[@]} volume(s) hold a live database data directory but no sql dump: ${UNDUMPED[*]}"
+    echo "      Usual cause: no row in the databases.csv baudolo reads, so it had no credentials to dump with."
+    exit 1
+fi
+echo "OK: backup payload present ($(( ${#VOLUME_DIRS[@]} - EMPTY )) non-empty volume(s)), every database volume dumped"

roles/svc-bkp-volume-2-local/templates/test.env.j2

@@ -3,6 +3,7 @@ BKP_TEST_BACKUPS_DIR={{ DIR_BACKUPS }}
 BKP_TEST_HEALTH_SCRIPT={{ [ DIR_SCRIPTS_SYSTEMCTL, 'sys-ctl-hlth-container', 'script.sh' ] | path_join }}
 BKP_TEST_HEALTH_TIMEOUT={{ lookup('timeout', 1200) | int }}
 BKP_TEST_IS_STACK_HOST={{ IS_STACK_HOST | bool | lower }}
+BKP_TEST_PYTHON={{ ansible_python_interpreter }}
 BKP_TEST_REPO_ROOT={{ playbook_dir }}
 BKP_TEST_SCRATCH_IMAGE={{ lookup('config', application_id, 'services.test.image') }}:{{ lookup('config', application_id, 'services.test.version') }}
 BKP_TEST_SERVICE={{ SYS_SERVICE_BACKUP_VOLUME_2_LOCAL }}