diff --git a/roles/web-app-bigbluebutton/meta/services.yml b/roles/web-app-bigbluebutton/meta/services.yml index 270cadb869..8d48254395 100644 --- a/roles/web-app-bigbluebutton/meta/services.yml +++ b/roles/web-app-bigbluebutton/meta/services.yml @@ -29,6 +29,7 @@ bigbluebutton: bond: 1 enabled: true shared: true + workload: node-local repository: https://github.com/bigbluebutton/docker.git ref: v3.0.23 recording: diff --git a/scripts/tests/deploy/swarm/routine/backup/README.md b/scripts/tests/deploy/swarm/routine/backup/README.md index 7beb4249a5..4ac4917b5d 100644 --- a/scripts/tests/deploy/swarm/routine/backup/README.md +++ b/scripts/tests/deploy/swarm/routine/backup/README.md @@ -3,29 +3,84 @@ Marker files seeded into the live NFS volume and into the manager's host secrets travel the full backup chain forward and are recovered back onto the live instance; the drill passes only when every marker survives the -whole loop. +whole loop. `base.sh` runs the nine steps below; the numbers in both +diagrams are its `[n/9]` log markers. + +## Step sequence (who does what) + +```mermaid +sequenceDiagram + autonumber + participant N as nfs-server + participant M as manager + participant B as backup host + participant U as LUKS "USB" (loop img on B) + + Note over N,M: [1/9] seed markers + N->>N: marker → live NFS volume dir + M->>M: marker → host secrets dir + + Note over N,M: [2/9] trigger deployed backup units + M->>M: svc-bkp-volume-2-local + secrets-2-local + N->>N: svc-bkp-nfs-2-local + + Note over N,M: [3/9] locate the generation holding the marker + N-->>M: find marker in files/*// (volume-scoped) + + Note over B: [4/9] pull via remote-2-local unit (rsync over ssh) + B->>N: pull nfs generations + B->>M: pull volume + secrets generations + + Note over B,U: [5/9] plug LUKS device, sync via local-2-device unit + B->>U: hard-linked snapshot + + Note over M: [6/9] full disaster teardown + alt stack workload + M->>M: docker stack rm + wait + else node-local workload + M->>M: stop compose project on every node + end + + Note over B,U: [7/9] recover device → local root + U-->>B: luksOpen + recover CLI → /var/tmp restore root + B->>B: drop pulled tree + loop image (free disk) + + Note over N: [8/9] recover local root → live NFS export + B-->>N: tar the volume subtree → recover CLI + N->>N: [8b] restart ganesha, remount clients, coherence probe + + Note over M: [9/9] recover docker volume + host secrets + B-->>M: volume generation → recover CLI + B-->>M: secrets generation → recover CLI + + Note over M,N: matrix update pass redeploys, then verify_recovered_marker.sh asserts the live marker +``` + +## Data flow (what is proven) ```mermaid flowchart TB subgraph forward["💾 Backup (solid: the deployed systemd units end to end)"] direction TB - vol["📂 Live NFS volume + docker volume
marker seeded"]:::live - sec["🔑 Host secrets on the manager
(secrets, CA, ACME, node identity) marker seeded"]:::live - localbkp["🗄️ Local backups on manager + NFS server
volume / nfs / secrets generation snapshots"]:::store - pulled["🗄️ Backup host
pulled generations (remote-2-local unit, rsync over ssh, pull-only)"]:::store - usb["🔒 Encrypted USB (LUKS)
hard-linked snapshot (local-2-device unit)"]:::device + vol["📂 [1] Live NFS volume + docker volume
marker seeded"]:::live + sec["🔑 [1] Host secrets on the manager
(secrets, CA, ACME, node identity) marker seeded"]:::live + localbkp["🗄️ [2-3] Local backups on manager + NFS server
volume / nfs / secrets generation snapshots"]:::store + pulled["🗄️ [4] Backup host
pulled generations (remote-2-local unit, rsync over ssh, pull-only)"]:::store + usb["🔒 [5] Encrypted USB (LUKS)
hard-linked snapshot (local-2-device unit)"]:::device vol -->|backup units| localbkp sec -->|secrets unit| localbkp localbkp -->|remote pull| pulled -->|device sync| usb end - subgraph recovery["♻️ Recovery (stack stopped, volume wiped first)"] + teardown["🛑 [6] Full disaster:
stack rm (stack workload) or
compose stop on every node (node-local workload),
then export wiped"]:::dead + + subgraph recovery["♻️ Recovery (dashed: the recover CLI, newest generation only)"] direction TB - restored["📁 Restore root on the backup host
newest snapshot after luksOpen"]:::store - export["📂 Live NFS export
rebuilt from the restored generation"]:::live - volume["📦 Docker volume
rebuilt on the manager"]:::live - secrets["🔑 Host secrets
rebuilt on the manager (marker cleared first)"]:::live - verified["✅ Stack redeployed
markers verified on the live instance"]:::ok + restored["📁 [7] Restore root on the backup host
newest snapshot after luksOpen (pulled tree + loop img freed)"]:::store + export["📂 [8] Live NFS export
volume subtree rebuilt, ganesha + client mounts refreshed"]:::live + volume["📦 [9] Docker volume
rebuilt on the manager"]:::live + secrets["🔑 [9] Host secrets
rebuilt on the manager (marker cleared first)"]:::live + verified["✅ Update pass redeploys
markers verified on the live instance"]:::ok usb -.->|decrypt + recover| restored restored -.->|restore export| export restored -.->|restore volume| volume @@ -35,10 +90,26 @@ flowchart TB secrets -.-> verified end + forward --> teardown --> recovery + classDef live fill:#f1f8f6,stroke:#d5e7e2,color:#455a64; classDef store fill:#f3f7fb,stroke:#d6e2ee,color:#455a64; classDef device fill:#fbf3f6,stroke:#eed9e2,color:#455a64; + classDef dead fill:#fdf2f2,stroke:#eddada,color:#455a64; classDef ok fill:#f3f9f4,stroke:#d9ecdc,color:#455a64; style forward fill:#ffffff,stroke:#e6e6e6; style recovery fill:#ffffff,stroke:#e6e6e6; ``` + +## Applicability + +- The drill runs whenever the app declares an NFS-flagged volume + (`PRIMARY_NFS_VOLUME`); apps without one skip it, since there is nothing + to prove a restore against. +- The workload form only changes step 6: stack workloads get + `docker stack rm`, `workload: node-local` roles get their compose + project stopped on every node. Every other step operates on volumes, + the NFS export and the deployed backup units, independent of the + orchestrator. +- Runs once per app, in the variant-0 CI job, between the matrix's first + deploy and its update pass. diff --git a/scripts/tests/deploy/swarm/routine/backup/base.sh b/scripts/tests/deploy/swarm/routine/backup/base.sh index 53b4854d1b..7bc7972338 100755 --- a/scripts/tests/deploy/swarm/routine/backup/base.sh +++ b/scripts/tests/deploy/swarm/routine/backup/base.sh @@ -49,10 +49,6 @@ DEV_MOUNT="$(python3 -c "import sys, yaml; print(yaml.safe_load(open(sys.argv[1] DEV_TARGET="$(python3 -c "import sys, yaml; print(yaml.safe_load(open(sys.argv[1]))['applications']['svc-bkp-local-2-device']['services']['local-2-device']['target'])" "${DRILL_EXTRAS}")" DEV_DEST="${DEV_MOUNT}${DEV_TARGET}" -if [ "${HAS_SWARM_SERVICE}" != true ]; then - echo "SKIP drill: ${APP_ID} deploys no swarm service" - exit 0 -fi if [ -z "${PRIMARY_NFS_VOLUME}" ]; then echo "SKIP drill: ${APP_ID} declares no NFS-flagged volume — nothing to prove a restore against" exit 0 @@ -157,7 +153,15 @@ fi echo " marker present on encrypted USB" echo "==> [6/9] tear the stack down completely (full disaster) before recovery" -docker exec "${MGR}" bash "${BKP_IN_NODE}/04_stack_rm_wait.sh" "${STACK_NAME}" +if [ "${HAS_SWARM_SERVICE}" = true ]; then + docker exec "${MGR}" bash "${BKP_IN_NODE}/04_stack_rm_wait.sh" "${STACK_NAME}" +else + for _node in "${MGR}" "${WRK1}" "${WRK2}"; do + docker exec "${_node}" sh -c \ + "ids=\$(docker ps -q --filter label=com.docker.compose.project=${ENTITY}); [ -z \"\$ids\" ] || docker stop \$ids" + done + echo " node-local workload: stopped compose project '${ENTITY}' on every node instead of a stack rm" +fi echo "==> [7/9] recover device -> local root via the recover CLI (full LUKS open)" docker exec "${BACKUP_NODE}" sh -c \ diff --git a/scripts/tests/deploy/swarm/routine/backup/verify_recovered_marker.sh b/scripts/tests/deploy/swarm/routine/backup/verify_recovered_marker.sh index b0d2018c6d..d5b1ef7418 100755 --- a/scripts/tests/deploy/swarm/routine/backup/verify_recovered_marker.sh +++ b/scripts/tests/deploy/swarm/routine/backup/verify_recovered_marker.sh @@ -2,7 +2,7 @@ # Runs on the controller after the matrix update pass. Asserts the DR-drill # marker (seeded pre-backup) survived recovery plus the update-pass bring-up # onto the recovered NFS export. Skips when the sentinel is absent (the drill -# recovered nothing: no NFS volume or no swarm service). +# recovered nothing: the app declares no NFS-flagged volume). set -euo pipefail DR_VERIFY_ENV="/tmp/dr-drill-verify-${APP_ID:?APP_ID required (matrix sets it)}.env"