feat(backup): capture volumes from a filesystem snapshot

Backing up a live volume with rsync copies a moving target: a database
written to mid-copy lands on disk in a state no engine ever committed.
Stopping the container avoids that at the cost of downtime.

A snapshot removes both. `--snapshot {btrfs,zfs}` with `--snapshot-subject`
freezes the docker root once per run, and every volume copy is then read
from that frozen tree while the containers keep serving. A restore of such
a copy is an ordinary crash recovery, which every supported engine performs
on its own at startup.

An unsupported filesystem or an unknown snapshot kind fails loudly rather
than degrading to a live copy, since a silent fallback would return exactly
the torn backup the mode exists to prevent. `--shutdown` is rejected
alongside `--snapshot` instead of being ignored: under a snapshot no
container is ever stopped, so accepting the flag would promise downtime
semantics the run does not deliver.

Copies out of a snapshot skip rsync's --checksum verification. The source
is immutable for the lifetime of the copy, so size-and-mtime cannot race,
and dropping the second full read roughly halves the I/O per volume.

backup/app.py grew past what one module could carry and is split into
layout, policy and dumps along the lines it already had internally.

Tests: unit coverage for the new snapshot, layout, policy, volume and cli
units; e2e cases drive real btrfs, zfs and ext4 filesystems on loop devices
in a privileged container, including a MariaDB that is written to across
the snapshot and must recover from the restored copy without losing a
committed row. CI installs zfs and sets E2E_REQUIRE_FILESYSTEMS so a
missing kernel module fails the build instead of silently skipping a
filesystem.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

tests/e2e/helpers/__init__.py

@@ -0,0 +1,4 @@
+"""Shared e2e helpers, re-exported so tests import one name."""
+
+from .fixtures import *  # noqa: F401,F403
+from .process import *  # noqa: F401,F403

tests/e2e/helpers/fixtures.py

@@ -0,0 +1,114 @@
+"""Fixtures and paths the e2e suite builds its scenarios from."""
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+
+from .process import machine_hash, run
+
+# postgres 18+ mounts at /var/lib/postgresql, not /var/lib/postgresql/data.
+POSTGRES_IMAGE = "postgres:alpine"
+POSTGRES_DATA_DIR = "/var/lib/postgresql"
+MARIADB_IMAGE = "mariadb:latest"
+MARIADB_DATA_DIR = "/var/lib/mysql"
+
+
+def backup_run(
+    *,
+    backups_dir: str,
+    repo_name: str,
+    compose_dir: str,
+    databases_csv: str,
+    database_containers: list[str],
+    images_no_stop_required: list[str],
+    images_no_backup_required: list[str] | None = None,
+    dump_only_sql: bool = False,
+) -> None:
+    cmd = [
+        "baudolo",
+        "--compose-dir",
+        compose_dir,
+        "--hard-restart-projects",
+        "mailu",
+        "--repo-name",
+        repo_name,
+        "--databases-csv",
+        databases_csv,
+        "--backups-dir",
+        backups_dir,
+        "--database-containers",
+        *database_containers,
+        "--images-no-stop-required",
+        *images_no_stop_required,
+    ]
+    if images_no_backup_required:
+        cmd += ["--images-no-backup-required", *images_no_backup_required]
+    if dump_only_sql:
+        cmd += ["--dump-only-sql"]
+
+    try:
+        run(cmd, capture=True, check=True)
+    except subprocess.CalledProcessError as e:
+        print(">>> baudolo failed (exit code:", e.returncode, ")")
+        if e.stdout:
+            print(">>> baudolo STDOUT:\n" + e.stdout)
+        if e.stderr:
+            print(">>> baudolo STDERR:\n" + e.stderr)
+        raise
+
+
+def latest_version_dir(backups_dir: str, repo_name: str) -> tuple[str, str]:
+    """
+    Returns (hash, version) for the latest backup.
+    """
+    h = machine_hash()
+    root = Path(backups_dir) / h / repo_name
+    if not root.is_dir():
+        raise FileNotFoundError(str(root))
+
+    versions = sorted([p.name for p in root.iterdir() if p.is_dir()])
+    if not versions:
+        raise RuntimeError(f"No versions found under {root}")
+    return h, versions[-1]
+
+
+def backup_path(backups_dir: str, repo_name: str, version: str, volume: str) -> Path:
+    h = machine_hash()
+    return Path(backups_dir) / h / repo_name / version / volume
+
+
+def create_minimal_compose_dir(base: str) -> str:
+    """
+    baudolo requires --compose-dir. Create an empty dir with one non-compose subdir.
+    """
+    p = Path(base) / "compose-root"
+    p.mkdir(parents=True, exist_ok=True)
+    (p / "noop").mkdir(parents=True, exist_ok=True)
+    return str(p)
+
+
+def write_databases_csv(path: str, rows: list[tuple[str, str, str, str]]) -> None:
+    """
+    rows: (instance, database, username, password)
+    database may be '' (empty) to trigger pg_dumpall behavior if you want, but here we use db name.
+    """
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        f.write("instance;database;username;password\n")
+        for inst, db, user, pw in rows:
+            f.write(f"{inst};{db};{user};{pw}\n")
+
+
+def cleanup_docker(*, containers: list[str], volumes: list[str]) -> None:
+    for c in containers:
+        run(["docker", "rm", "-f", c], capture=True, check=False)
+    for v in volumes:
+        run(["docker", "volume", "rm", "-f", v], capture=True, check=False)
+
+
+def ensure_empty_dir(path: str) -> None:
+    p = Path(path)
+    if p.exists():
+        shutil.rmtree(p)
+    p.mkdir(parents=True, exist_ok=True)

tests/e2e/helpers/process.py

@@ -1,19 +1,9 @@
-# tests/e2e/helpers.py
+"""Process, docker and readiness helpers for the e2e suite."""
 from __future__ import annotations
 
-import shutil
 import subprocess
 import time
 import uuid
-from pathlib import Path
-
-# SPOT for the database images and their in-container data dirs the e2e
-# suite runs against. postgres:alpine tracks latest (18+), which mounts at
-# /var/lib/postgresql (not /var/lib/postgresql/data); bump here only.
-POSTGRES_IMAGE = "postgres:alpine"
-POSTGRES_DATA_DIR = "/var/lib/postgresql"
-MARIADB_IMAGE = "mariadb:latest"
-MARIADB_DATA_DIR = "/var/lib/mysql"
 
 
 def run(
@@ -163,103 +153,3 @@ def wait_for_mariadb_sql(
     raise TimeoutError(
         f"Timed out waiting for MariaDB SQL login readiness in container {container}"
     )
-
-
-def backup_run(
-    *,
-    backups_dir: str,
-    repo_name: str,
-    compose_dir: str,
-    databases_csv: str,
-    database_containers: list[str],
-    images_no_stop_required: list[str],
-    images_no_backup_required: list[str] | None = None,
-    dump_only_sql: bool = False,
-) -> None:
-    cmd = [
-        "baudolo",
-        "--compose-dir",
-        compose_dir,
-        "--hard-restart-projects",
-        "mailu",
-        "--repo-name",
-        repo_name,
-        "--databases-csv",
-        databases_csv,
-        "--backups-dir",
-        backups_dir,
-        "--database-containers",
-        *database_containers,
-        "--images-no-stop-required",
-        *images_no_stop_required,
-    ]
-    if images_no_backup_required:
-        cmd += ["--images-no-backup-required", *images_no_backup_required]
-    if dump_only_sql:
-        cmd += ["--dump-only-sql"]
-
-    try:
-        run(cmd, capture=True, check=True)
-    except subprocess.CalledProcessError as e:
-        print(">>> baudolo failed (exit code:", e.returncode, ")")
-        if e.stdout:
-            print(">>> baudolo STDOUT:\n" + e.stdout)
-        if e.stderr:
-            print(">>> baudolo STDERR:\n" + e.stderr)
-        raise
-
-
-def latest_version_dir(backups_dir: str, repo_name: str) -> tuple[str, str]:
-    """
-    Returns (hash, version) for the latest backup.
-    """
-    h = machine_hash()
-    root = Path(backups_dir) / h / repo_name
-    if not root.is_dir():
-        raise FileNotFoundError(str(root))
-
-    versions = sorted([p.name for p in root.iterdir() if p.is_dir()])
-    if not versions:
-        raise RuntimeError(f"No versions found under {root}")
-    return h, versions[-1]
-
-
-def backup_path(backups_dir: str, repo_name: str, version: str, volume: str) -> Path:
-    h = machine_hash()
-    return Path(backups_dir) / h / repo_name / version / volume
-
-
-def create_minimal_compose_dir(base: str) -> str:
-    """
-    baudolo requires --compose-dir. Create an empty dir with one non-compose subdir.
-    """
-    p = Path(base) / "compose-root"
-    p.mkdir(parents=True, exist_ok=True)
-    (p / "noop").mkdir(parents=True, exist_ok=True)
-    return str(p)
-
-
-def write_databases_csv(path: str, rows: list[tuple[str, str, str, str]]) -> None:
-    """
-    rows: (instance, database, username, password)
-    database may be '' (empty) to trigger pg_dumpall behavior if you want, but here we use db name.
-    """
-    Path(path).parent.mkdir(parents=True, exist_ok=True)
-    with open(path, "w", encoding="utf-8") as f:
-        f.write("instance;database;username;password\n")
-        for inst, db, user, pw in rows:
-            f.write(f"{inst};{db};{user};{pw}\n")
-
-
-def cleanup_docker(*, containers: list[str], volumes: list[str]) -> None:
-    for c in containers:
-        run(["docker", "rm", "-f", c], capture=True, check=False)
-    for v in volumes:
-        run(["docker", "volume", "rm", "-f", v], capture=True, check=False)
-
-
-def ensure_empty_dir(path: str) -> None:
-    p = Path(path)
-    if p.exists():
-        shutil.rmtree(p)
-    p.mkdir(parents=True, exist_ok=True)

tests/e2e/snapshot_db_driver.py

@@ -0,0 +1,41 @@
+"""Copy a live database's volume out of a snapshot, using the real backup path.
+
+Runs inside the privileged container built by test_e2e_snapshot_db.py, where a
+database is mid-write on a btrfs subvolume. Exercises volume_snapshot and
+backup_volume exactly as a backup run would.
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+
+sys.path.insert(0, "/src")
+
+from baudolo.backup.snapshot import SnapshotError, volume_snapshot  # noqa: E402
+from baudolo.backup.volume import backup_volume  # noqa: E402
+
+SUBJECT = "/subject/docker"
+VOLUME = "mariadb_data"
+DATADIR = f"{SUBJECT}/volumes/{VOLUME}/_data"
+VERSIONS = "/backups"
+GENERATION = f"{VERSIONS}/20260731"
+
+
+def shell(command: str) -> list[str]:
+    proc = subprocess.run(command, shell=True, capture_output=True, text=True)
+    if proc.returncode != 0:
+        raise SnapshotError(f"{command} exited {proc.returncode}: {proc.stderr.strip()}")
+    return proc.stdout.splitlines()
+
+
+with volume_snapshot("btrfs", SUBJECT, "dbtest", run=shell) as resolve:
+    backup_volume(
+        VERSIONS,
+        VOLUME,
+        f"{GENERATION}/{VOLUME}",
+        authoritative=False,
+        source=resolve(DATADIR) + "/",
+    )
+
+print("SNAPSHOT COPY DONE", flush=True)

tests/e2e/snapshot_driver.py

@@ -0,0 +1,59 @@
+"""Exercise volume_snapshot against a real filesystem, from inside a container.
+
+Runs where loop devices exist. Prints one PASS/FAIL line per assertion and exits
+non-zero on the first failure, so the calling test can surface the reason.
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+sys.path.insert(0, "/src")
+
+from baudolo.backup.snapshot import SnapshotError, volume_snapshot  # noqa: E402
+
+KIND = sys.argv[1]
+SUBJECT = sys.argv[2]
+EXPECT = sys.argv[3]
+
+
+def shell(command: str) -> list[str]:
+    proc = subprocess.run(command, shell=True, capture_output=True, text=True)
+    if proc.returncode != 0:
+        raise SnapshotError(f"{command} exited {proc.returncode}: {proc.stderr.strip()}")
+    return proc.stdout.splitlines()
+
+
+def check(label: str, condition: bool) -> None:
+    print(f"{'PASS' if condition else 'FAIL'} {label}", flush=True)
+    if not condition:
+        sys.exit(1)
+
+
+volume = Path(SUBJECT) / "volumes" / "demo" / "_data"
+volume.mkdir(parents=True, exist_ok=True)
+(volume / "state").write_text("before\n")
+
+if EXPECT == "unsupported":
+    try:
+        with volume_snapshot(KIND, SUBJECT, "e2e", run=shell):
+            check("snapshot on an unsupported filesystem must not succeed", False)
+    except SnapshotError as exc:
+        check(f"refused loudly: {str(exc)[:60]}", True)
+    sys.exit(0)
+
+with volume_snapshot(KIND, SUBJECT, "e2e", run=shell) as resolve:
+    frozen = Path(resolve(str(volume))) / "state"
+    check("the snapshot exposes the volume", frozen.is_file())
+    check("the snapshot carries the content", frozen.read_text() == "before\n")
+
+    (volume / "state").write_text("after\n")
+    check("a later write does not reach the snapshot", frozen.read_text() == "before\n")
+    check("the live tree did change", (volume / "state").read_text() == "after\n")
+
+    root = Path(resolve(SUBJECT))
+
+check("the snapshot is removed afterwards", not root.exists() or not (root / "volumes").exists())
+print("ALL OK", flush=True)

tests/e2e/test_e2e_snapshot.py

@@ -0,0 +1,141 @@
+"""Snapshot capture against real filesystems.
+
+Loop devices are only available to a privileged container, so each case builds
+its filesystem inside one and drives snapshot_driver.py there. A filesystem
+without snapshot support must fail loudly rather than degrade to a live copy,
+which is the property that makes the mode safe to offer at all.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import unittest
+from pathlib import Path
+
+from .helpers import require_docker, run, unique
+
+REPO_SRC = Path(__file__).resolve().parents[2] / "src"
+DRIVER = Path(__file__).resolve().parent / "snapshot_driver.py"
+IMAGE = "alpine:3.20"
+PACKAGES = "apk add -q btrfs-progs e2fsprogs zfs python3 util-linux"
+ATTACH = (
+    "LOOP=$(losetup -f | awk '{print $1}') "
+    '&& { [ -b "$LOOP" ] || mknod "$LOOP" b 7 "${LOOP#/dev/loop}"; }; '
+    'losetup "$LOOP" /img'
+)
+LOOP_FS = {
+    "btrfs": "btrfs subvolume create /subject/docker >/dev/null",
+    "ext4": "mkdir -p /subject/docker",
+}
+# A container carries no /lib/modules, so modprobe fails even on a loaded module.
+ZFS_READY = '{ [ -c /dev/zfs ] || modprobe zfs 2>/dev/null; }; [ -c /dev/zfs ]'
+
+
+def mount_script(fstype: str) -> str:
+    """Build a filesystem on a loop device and carve out the snapshot subject."""
+    if fstype == "zfs":
+        return (
+            f"{PACKAGES} && {ZFS_READY} && truncate -s 400M /img "
+            "&& zpool create -m none baudolo /img "
+            "&& zfs create -o mountpoint=/subject/docker baudolo/docker"
+        )
+    return (
+        f"{PACKAGES} && truncate -s 400M /img && mkfs.{fstype} -q /img "
+        f'&& mkdir -p /subject && {ATTACH} && mount -t {fstype} "$LOOP" /subject '
+        f"&& {LOOP_FS[fstype]}"
+    )
+
+
+def zfs_usable() -> bool:
+    """Whether this host's kernel can serve zfs to a privileged container."""
+    proc = run(
+        [
+            "docker", "run", "--rm", "--privileged", IMAGE, "sh", "-lc",
+            f"apk add -q zfs >/dev/null 2>&1 && {ZFS_READY}",
+        ],
+        capture=True,
+        check=False,
+    )
+    return proc.returncode == 0
+
+
+def required(fstype: str) -> bool:
+    """Whether this run must cover ``fstype`` instead of skipping it.
+
+    CI sets E2E_REQUIRE_FILESYSTEMS so a missing kernel module fails the build
+    rather than passing it with a filesystem silently untested.
+    """
+    demanded = os.environ.get("E2E_REQUIRE_FILESYSTEMS", "")
+    return fstype in demanded.replace(",", " ").split()
+
+
+def stage() -> Path:
+    """Copy source and driver under /tmp, the only path the DinD daemon shares."""
+    staged = Path("/tmp") / unique("baudolo-e2e-snapshot")
+    shutil.copytree(REPO_SRC, staged / "src")
+    shutil.copy(DRIVER, staged / "driver.py")
+    return staged
+
+
+def drive(fstype: str, kind: str, expect: str) -> str:
+    staged = stage()
+    script = (
+        f"set -e; {mount_script(fstype)}; "
+        f"python3 /driver.py {kind} /subject/docker {expect}"
+    )
+    try:
+        proc = run(
+            [
+                "docker", "run", "--rm", "--privileged",
+                "--name", staged.name,
+                "-v", f"{staged / 'src'}:/src:ro",
+                "-v", f"{staged / 'driver.py'}:/driver.py:ro",
+                IMAGE, "sh", "-lc", script,
+            ],
+            capture=True,
+            check=False,
+        )
+    finally:
+        shutil.rmtree(staged, ignore_errors=True)
+    if proc.returncode != 0:
+        raise AssertionError(f"{fstype}/{kind} driver failed:\n{proc.stdout}\n{proc.stderr}")
+    return proc.stdout
+
+
+class TestE2ESnapshot(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        require_docker()
+
+    def assert_freezes(self, fstype: str) -> None:
+        output = drive(fstype, fstype, "supported")
+        self.assertIn("PASS the snapshot exposes the volume", output)
+        self.assertIn("PASS a later write does not reach the snapshot", output)
+        self.assertIn("PASS the snapshot is removed afterwards", output)
+        self.assertIn("ALL OK", output)
+
+    def test_btrfs_snapshot_freezes_the_volume(self) -> None:
+        self.assert_freezes("btrfs")
+
+    def test_zfs_snapshot_freezes_the_volume(self) -> None:
+        if not zfs_usable():
+            if required("zfs"):
+                self.fail(
+                    "E2E_REQUIRE_FILESYSTEMS demands zfs, but this kernel provides no "
+                    "zfs module; load it before running the suite"
+                )
+            self.skipTest("this kernel provides no zfs module, so no pool can be created")
+        self.assert_freezes("zfs")
+
+    def test_ext4_has_no_snapshot_and_says_so(self) -> None:
+        output = drive("ext4", "btrfs", "unsupported")
+        self.assertIn("PASS refused loudly", output)
+
+    def test_an_unknown_kind_is_refused_before_touching_the_filesystem(self) -> None:
+        output = drive("ext4", "lvm", "unsupported")
+        self.assertIn("PASS refused loudly", output)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/e2e/test_e2e_snapshot_db.py

@@ -0,0 +1,103 @@
+"""A live database survives being captured from a snapshot.
+
+The database is written to while the snapshot is taken and keeps writing
+afterwards, so the copy can only be a point in time - never a clean shutdown.
+A second server is then started on that copy: it must recover on its own and
+still hold every row committed before the snapshot.
+"""
+
+from __future__ import annotations
+
+import shutil
+import unittest
+from pathlib import Path
+
+from .helpers import require_docker, run, unique
+
+REPO_SRC = Path(__file__).resolve().parents[2] / "src"
+DRIVER = Path(__file__).resolve().parent / "snapshot_db_driver.py"
+IMAGE = "alpine:3.20"
+SOCKET = "/tmp/live.sock"
+RESTORED_SOCKET = "/tmp/restored.sock"
+DATADIR = "/subject/docker/volumes/mariadb_data/_data"
+RESTORED = "/restored"
+
+SCRIPT = f"""set -e
+apk add -q btrfs-progs util-linux python3 mariadb mariadb-client rsync
+truncate -s 900M /img
+mkfs.btrfs -q /img
+mkdir -p /subject
+LOOP=$(losetup -f | awk '{{print $1}}')
+{{ [ -b "$LOOP" ] || mknod "$LOOP" b 7 "${{LOOP#/dev/loop}}"; }}
+losetup "$LOOP" /img
+mount -t btrfs "$LOOP" /subject
+btrfs subvolume create /subject/docker >/dev/null
+mkdir -p {DATADIR}
+
+mariadb-install-db --user=root --datadir={DATADIR} >/dev/null 2>&1
+mariadbd --user=root --datadir={DATADIR} --socket={SOCKET} --skip-networking &
+for i in $(seq 1 60); do mariadb-admin --socket={SOCKET} ping >/dev/null 2>&1 && break; sleep 1; done
+
+mariadb --socket={SOCKET} -e "CREATE DATABASE demo;
+  CREATE TABLE demo.t (id INT PRIMARY KEY, v VARCHAR(32)) ENGINE=InnoDB;
+  INSERT INTO demo.t VALUES (1,'committed'),(2,'committed');"
+
+mariadb --socket={SOCKET} -e "INSERT INTO demo.t VALUES (3,'committed');"
+python3 /driver.py
+mariadb --socket={SOCKET} -e "INSERT INTO demo.t VALUES (4,'after-snapshot');"
+
+mkdir -p {RESTORED}
+rsync -a /backups/20260731/mariadb_data/files/ {RESTORED}/
+mariadbd --user=root --datadir={RESTORED} --socket={RESTORED_SOCKET} --skip-networking &
+for i in $(seq 1 60); do mariadb-admin --socket={RESTORED_SOCKET} ping >/dev/null 2>&1 && break; sleep 1; done
+
+echo "RESTORED_ROWS=$(mariadb --socket={RESTORED_SOCKET} -N -B -e 'SELECT COUNT(*) FROM demo.t;')"
+echo "RESTORED_AFTER=$(mariadb --socket={RESTORED_SOCKET} -N -B -e \\
+  "SELECT COUNT(*) FROM demo.t WHERE v='after-snapshot';")"
+echo DB_OK
+"""
+
+
+class TestE2ESnapshotDatabase(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        require_docker()
+        staged = Path("/tmp") / unique("baudolo-e2e-snapshot-db")
+        shutil.copytree(REPO_SRC, staged / "src")
+        shutil.copy(DRIVER, staged / "driver.py")
+        try:
+            proc = run(
+                [
+                    "docker", "run", "--rm", "--privileged",
+                    "--name", staged.name,
+                    "-v", f"{staged / 'src'}:/src:ro",
+                    "-v", f"{staged / 'driver.py'}:/driver.py:ro",
+                    IMAGE, "sh", "-lc", SCRIPT,
+                ],
+                capture=True,
+                check=False,
+            )
+        finally:
+            shutil.rmtree(staged, ignore_errors=True)
+        cls.output = proc.stdout + proc.stderr
+        cls.returncode = proc.returncode
+
+    def test_the_run_completed(self) -> None:
+        self.assertEqual(self.returncode, 0, self.output)
+        self.assertIn("DB_OK", self.output)
+
+    def test_the_backup_came_from_the_snapshot(self) -> None:
+        self.assertIn("SNAPSHOT COPY DONE", self.output)
+
+    def test_the_restored_server_recovered_on_its_own(self) -> None:
+        self.assertIn("RESTORED_ROWS=3", self.output)
+
+    def test_writes_after_the_snapshot_are_absent(self) -> None:
+        self.assertIn("RESTORED_AFTER=0", self.output)
+
+    def test_the_copy_was_an_unclean_state_the_engine_had_to_repair(self) -> None:
+        self.assertRegex(self.output, r"(?i)crash recovery|rolling back|log sequence")
+
+
+if __name__ == "__main__":
+    unittest.main()