kevinveenbirkenbach/docker-volume-backup
1
0
Fork 0
mirror of https://github.com/kevinveenbirkenbach/docker-volume-backup.git synced 2026-08-01 12:34:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(backup): capture volumes from a filesystem snapshot

Browse Source 
Backing up a live volume with rsync copies a moving target: a database
written to mid-copy lands on disk in a state no engine ever committed.
Stopping the container avoids that at the cost of downtime.

A snapshot removes both. `--snapshot {btrfs,zfs}` with `--snapshot-subject`
freezes the docker root once per run, and every volume copy is then read
from that frozen tree while the containers keep serving. A restore of such
a copy is an ordinary crash recovery, which every supported engine performs
on its own at startup.

An unsupported filesystem or an unknown snapshot kind fails loudly rather
than degrading to a live copy, since a silent fallback would return exactly
the torn backup the mode exists to prevent. `--shutdown` is rejected
alongside `--snapshot` instead of being ignored: under a snapshot no
container is ever stopped, so accepting the flag would promise downtime
semantics the run does not deliver.

Copies out of a snapshot skip rsync's --checksum verification. The source
is immutable for the lifetime of the copy, so size-and-mtime cannot race,
and dropping the second full read roughly halves the I/O per volume.

backup/app.py grew past what one module could carry and is split into
layout, policy and dumps along the lines it already had internally.

Tests: unit coverage for the new snapshot, layout, policy, volume and cli
units; e2e cases drive real btrfs, zfs and ext4 filesystems on loop devices
in a privileged container, including a MariaDB that is written to across
the snapshot and must recover from the restored copy without losing a
committed row. CI installs zfs and sets E2E_REQUIRE_FILESYSTEMS so a
missing kernel module fails the build instead of silently skipping a
filesystem.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-Birkenbach
2026-07-31 13:01:43 +02:00
parent ec3d1a5046
commit d4317827bd
26 changed files with 1329 additions and 430 deletions
Download Patch File Download Diff File Expand all files Collapse all files

282
src/baudolo/backup/app.py
View File

@@ -1,171 +1,29 @@
"""Back up every Docker volume of a host into a timestamped generation."""
from __future__ import annotations
import os
import pathlib
import sys
from contextlib import ExitStack
from datetime import datetime
import pandas
from dirval import create_stamp_file
from pandas.errors import EmptyDataError
from .cli import parse_args
from .compose import handle_docker_compose_services
from .db import backup_database
from .docker import (
change_containers_status,
containers_using_volume,
docker_volume_names,
filter_stoppable,
get_image_info,
has_image,
is_swarm_task,
)
from .shell import execute_shell_command
from .volume import backup_volume
def get_machine_id() -> str:
return execute_shell_command("sha256sum /etc/machine-id")[0][0:64]
def stamp_directory(version_dir: str) -> None:
"""
Use dirval as a Python library to stamp the directory (no CLI dependency).
"""
create_stamp_file(version_dir)
def create_version_directory(versions_dir: str, backup_time: str) -> str:
version_dir = os.path.join(versions_dir, backup_time)
pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
return version_dir
def create_volume_directory(version_dir: str, volume_name: str) -> str:
path = os.path.join(version_dir, volume_name)
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
return path
def is_image_ignored(container: str, images_no_backup_required: list[str]) -> bool:
if not images_no_backup_required:
return False
img = get_image_info(container)
return img in images_no_backup_required
def volume_is_fully_ignored(
containers: list[str], images_no_backup_required: list[str]
) -> bool:
"""
Skip file backup only if all containers linked to the volume are ignored.
"""
if not containers:
return False
return all(is_image_ignored(c, images_no_backup_required) for c in containers)
def requires_stop(containers: list[str], images_no_stop_required: list[str]) -> bool:
"""
Stop is required if ANY stoppable container image is NOT in the exact
image whitelist. Swarm task containers never count: baudolo must
not cycle them (see docker.is_swarm_task).
"""
for c in containers:
if is_swarm_task(c):
continue
img = get_image_info(c)
if img not in images_no_stop_required:
return True
return False
def backup_mariadb_or_postgres(
*,
container: str,
volume_dir: str,
databases_df: "pandas.DataFrame",
database_containers: list[str],
) -> tuple[bool, bool]:
"""
Returns (is_db_container, dumped_any)
"""
for img in ["mariadb", "postgres"]:
if has_image(container, img):
dumped = backup_database(
container=container,
volume_dir=volume_dir,
db_type=img,
databases_df=databases_df,
database_containers=database_containers,
)
return True, dumped
return False, False
def _empty_databases_df() -> "pandas.DataFrame":
"""
Create an empty DataFrame with the expected schema for databases.csv.
This allows the backup to continue without DB dumps when the CSV is missing
or empty (pandas EmptyDataError).
"""
return pandas.DataFrame(columns=["instance", "database", "username", "password"])
def _load_databases_df(csv_path: str) -> "pandas.DataFrame":
"""
Load databases.csv robustly.
- Missing file -> warn, continue with empty df
- Empty file -> warn, continue with empty df
- Valid CSV -> return dataframe
"""
try:
return pandas.read_csv(csv_path, sep=";", keep_default_na=False, dtype=str)
except FileNotFoundError:
print(
f"WARNING: databases.csv not found: {csv_path}. Continuing without database dumps.",
file=sys.stderr,
flush=True,
)
return _empty_databases_df()
except EmptyDataError:
print(
f"WARNING: databases.csv exists but is empty: {csv_path}. Continuing without database dumps.",
file=sys.stderr,
flush=True,
)
return _empty_databases_df()
def _backup_dumps_for_volume(
*,
containers: list[str],
vol_dir: str,
databases_df: "pandas.DataFrame",
database_containers: list[str],
) -> tuple[bool, bool]:
"""
Returns (found_db_container, dumped_any)
"""
found_db = False
dumped_any = False
for c in containers:
is_db, dumped = backup_mariadb_or_postgres(
container=c,
volume_dir=vol_dir,
databases_df=databases_df,
database_containers=database_containers,
)
if is_db:
found_db = True
if dumped:
dumped_any = True
return found_db, dumped_any
from .dumps import backup_dumps_for_volume, load_databases_df
from .layout import (
create_version_directory,
create_volume_directory,
get_machine_id,
stamp_directory,
)
from .policy import requires_stop, volume_is_fully_ignored
from .snapshot import volume_snapshot
from .volume import backup_volume, get_storage_path
def main() -> int:
@@ -183,66 +41,80 @@ def main() -> int:
#
# Robust behavior:
# - if the file is missing or empty, we continue without DB dumps.
databases_df = _load_databases_df(args.databases_csv)
databases_df = load_databases_df(args.databases_csv)
print("💾 Start volume backups...", flush=True)
for volume_name in docker_volume_names():
print(f"Start backup routine for volume: {volume_name}", flush=True)
containers = containers_using_volume(volume_name)
# EARLY SKIP: if all linked containers are ignored, do not create any dirs
if volume_is_fully_ignored(containers, args.images_no_backup_required):
print(
f"Skipping volume '{volume_name}' entirely (all linked containers are ignored).",
flush=True,
with ExitStack() as stack:
resolve_source = None
if args.snapshot:
resolve_source = stack.enter_context(
volume_snapshot(args.snapshot, args.snapshot_subject, backup_time)
)
continue
vol_dir = create_volume_directory(version_dir, volume_name)
for volume_name in docker_volume_names():
print(f"Start backup routine for volume: {volume_name}", flush=True)
containers = containers_using_volume(volume_name)
found_db, dumped_any = _backup_dumps_for_volume(
containers=containers,
vol_dir=vol_dir,
databases_df=databases_df,
database_containers=args.database_containers,
)
if volume_is_fully_ignored(containers, args.images_no_backup_required):
print(
f"Skipping volume '{volume_name}' entirely (all linked containers are ignored).",
flush=True,
)
continue
# dump-only-sql logic:
if args.dump_only_sql:
if found_db:
if not dumped_any:
print(
f"WARNING: dump-only-sql requested but no DB dump was produced for DB volume '{volume_name}'. "
"Falling back to file backup.",
flush=True,
)
# fall through to file backup below
else:
# DB volume successfully dumped -> skip file backup
continue
# Non-DB volume -> always do file backup (fall through)
vol_dir = create_volume_directory(version_dir, volume_name)
if args.everything:
# "everything": always do pre-rsync, then stop + rsync again
stoppable = filter_stoppable(containers)
backup_volume(versions_dir, volume_name, vol_dir, authoritative=False)
change_containers_status(stoppable, "stop")
backup_volume(versions_dir, volume_name, vol_dir, authoritative=True)
if not args.shutdown:
change_containers_status(stoppable, "start")
continue
found_db, dumped_any = backup_dumps_for_volume(
containers=containers,
vol_dir=vol_dir,
databases_df=databases_df,
database_containers=args.database_containers,
)
# default: rsync, and if needed stop + rsync
backup_volume(versions_dir, volume_name, vol_dir, authoritative=False)
if requires_stop(containers, args.images_no_stop_required):
stoppable = filter_stoppable(containers)
change_containers_status(stoppable, "stop")
backup_volume(versions_dir, volume_name, vol_dir, authoritative=True)
if not args.shutdown:
change_containers_status(stoppable, "start")
if args.dump_only_sql:
if found_db:
if not dumped_any:
print(
f"WARNING: dump-only-sql requested but no DB dump was produced for DB volume '{volume_name}'. "
"Falling back to file backup.",
flush=True,
)
else:
continue
live_source = get_storage_path(volume_name)
def copy(*, authoritative: bool, source: str = live_source) -> None:
backup_volume(
versions_dir,
volume_name,
vol_dir,
authoritative=authoritative,
source=source,
)
if resolve_source is not None:
copy(authoritative=False, source=resolve_source(live_source))
continue
if args.everything:
stoppable = filter_stoppable(containers)
copy(authoritative=False)
change_containers_status(stoppable, "stop")
copy(authoritative=True)
if not args.shutdown:
change_containers_status(stoppable, "start")
continue
copy(authoritative=False)
if requires_stop(containers, args.images_no_stop_required):
stoppable = filter_stoppable(containers)
change_containers_status(stoppable, "stop")
copy(authoritative=True)
if not args.shutdown:
change_containers_status(stoppable, "start")
# Stamp the backup version directory using dirval (python lib)
stamp_directory(version_dir)
print("Finished volume backups.", flush=True)

17
src/baudolo/backup/cli.py
View File

@@ -39,6 +39,16 @@ def parse_args() -> argparse.Namespace:
help="Backup root directory (e.g. /var/lib/backup/)",
)
p.add_argument(
"--snapshot",
choices=["btrfs", "zfs"],
help="Capture every volume from one atomic filesystem snapshot instead of copying the live tree. Containers are not stopped, and the copy is a single pass. Requires --snapshot-subject. Omit to keep the live two-pass copy.",
)
p.add_argument(
"--snapshot-subject",
help="Btrfs subvolume or zfs dataset mountpoint holding the docker volumes, e.g. /var/lib/docker. Required with --snapshot.",
)
p.add_argument(
"--database-containers",
nargs="+",
@@ -79,4 +89,9 @@ def parse_args() -> argparse.Namespace:
"If a DB dump cannot be produced, baudolo falls back to a file backup."
),
)
return p.parse_args()
args = p.parse_args()
if bool(args.snapshot) != bool(args.snapshot_subject):
p.error("--snapshot and --snapshot-subject must be given together")
if args.snapshot and args.shutdown:
p.error("--shutdown is meaningless with --snapshot: containers are never stopped")
return args

98
src/baudolo/backup/dumps.py Normal file
View File

@@ -0,0 +1,98 @@
"""Database dumps taken before a volume's files are copied."""
from __future__ import annotations
import sys
import pandas
from pandas.errors import EmptyDataError
from .db import backup_database
from .docker import has_image
def backup_mariadb_or_postgres(
*,
container: str,
volume_dir: str,
databases_df: "pandas.DataFrame",
database_containers: list[str],
) -> tuple[bool, bool]:
"""
Returns (is_db_container, dumped_any)
"""
for img in ["mariadb", "postgres"]:
if has_image(container, img):
dumped = backup_database(
container=container,
volume_dir=volume_dir,
db_type=img,
databases_df=databases_df,
database_containers=database_containers,
)
return True, dumped
return False, False
def _empty_databases_df() -> "pandas.DataFrame":
"""
Create an empty DataFrame with the expected schema for databases.csv.
This allows the backup to continue without DB dumps when the CSV is missing
or empty (pandas EmptyDataError).
"""
return pandas.DataFrame(columns=["instance", "database", "username", "password"])
def load_databases_df(csv_path: str) -> "pandas.DataFrame":
"""
Load databases.csv robustly.
- Missing file -> warn, continue with empty df
- Empty file -> warn, continue with empty df
- Valid CSV -> return dataframe
"""
try:
return pandas.read_csv(csv_path, sep=";", keep_default_na=False, dtype=str)
except FileNotFoundError:
print(
f"WARNING: databases.csv not found: {csv_path}. Continuing without database dumps.",
file=sys.stderr,
flush=True,
)
return _empty_databases_df()
except EmptyDataError:
print(
f"WARNING: databases.csv exists but is empty: {csv_path}. Continuing without database dumps.",
file=sys.stderr,
flush=True,
)
return _empty_databases_df()
def backup_dumps_for_volume(
*,
containers: list[str],
vol_dir: str,
databases_df: "pandas.DataFrame",
database_containers: list[str],
) -> tuple[bool, bool]:
"""
Returns (found_db_container, dumped_any)
"""
found_db = False
dumped_any = False
for c in containers:
is_db, dumped = backup_mariadb_or_postgres(
container=c,
volume_dir=vol_dir,
databases_df=databases_df,
database_containers=database_containers,
)
if is_db:
found_db = True
if dumped:
dumped_any = True
return found_db, dumped_any

33
src/baudolo/backup/layout.py Normal file
View File

@@ -0,0 +1,33 @@
"""Where a backup run puts its files, and how a finished run is stamped."""
from __future__ import annotations
import os
import pathlib
from dirval import create_stamp_file
from .shell import execute_shell_command
def get_machine_id() -> str:
return execute_shell_command("sha256sum /etc/machine-id")[0][0:64]
def stamp_directory(version_dir: str) -> None:
"""
Use dirval as a Python library to stamp the directory (no CLI dependency).
"""
create_stamp_file(version_dir)
def create_version_directory(versions_dir: str, backup_time: str) -> str:
version_dir = os.path.join(versions_dir, backup_time)
pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
return version_dir
def create_volume_directory(version_dir: str, volume_name: str) -> str:
path = os.path.join(version_dir, volume_name)
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
return path

38
src/baudolo/backup/policy.py Normal file
View File

@@ -0,0 +1,38 @@
"""Which volumes are backed up, and which containers must stop for it."""
from __future__ import annotations
from .docker import get_image_info, is_swarm_task
def is_image_ignored(container: str, images_no_backup_required: list[str]) -> bool:
if not images_no_backup_required:
return False
img = get_image_info(container)
return img in images_no_backup_required
def volume_is_fully_ignored(
containers: list[str], images_no_backup_required: list[str]
) -> bool:
"""
Skip file backup only if all containers linked to the volume are ignored.
"""
if not containers:
return False
return all(is_image_ignored(c, images_no_backup_required) for c in containers)
def requires_stop(containers: list[str], images_no_stop_required: list[str]) -> bool:
"""
Stop is required if ANY stoppable container image is NOT in the exact
image whitelist. Swarm task containers never count: baudolo must
not cycle them (see docker.is_swarm_task).
"""
for c in containers:
if is_swarm_task(c):
continue
img = get_image_info(c)
if img not in images_no_stop_required:
return True
return False

86
src/baudolo/backup/snapshot.py Normal file
View File

@@ -0,0 +1,86 @@
"""Capture every volume from one atomic filesystem snapshot.
Copying a live tree file by file cannot produce a point in time: a database can
write between two files and leave a control file and its write-ahead log
disagreeing, which no recovery can repair. A snapshot freezes the whole subject
at once, so a database reads it as a crash and replays its log - a case it is
built for. That also removes the reason to stop containers at all.
The snapshot kind is stated by the caller rather than probed, because falling
back to a live copy when a probe is inconclusive would hand out backups that
look consistent and are not.
"""
from __future__ import annotations
import os
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from .shell import execute_shell_command
KINDS = ("btrfs", "zfs")
class SnapshotError(RuntimeError):
"""A snapshot could not be created, resolved or removed."""
def _resolver(subject: str, root: str) -> Callable[[str], str]:
def resolve(path: str) -> str:
relative = os.path.relpath(os.path.abspath(path), os.path.abspath(subject))
if relative.startswith(".."):
raise SnapshotError(f"{path} lies outside the snapshot subject {subject}")
return os.path.join(root, relative) if relative != "." else root
return resolve
def _btrfs(subject: str, name: str, run: Callable[[str], list[str]]) -> tuple[str, str]:
target = os.path.join(os.path.dirname(os.path.abspath(subject)), f".{name}")
run(f"btrfs subvolume snapshot -r {subject} {target}")
return target, f"btrfs subvolume delete {target}"
def _zfs(subject: str, name: str, run: Callable[[str], list[str]]) -> tuple[str, str]:
output = run(f"zfs list -H -o name {subject}")
dataset = (output[0] if output else "").strip()
if not dataset:
raise SnapshotError(f"no zfs dataset is mounted at {subject}")
run(f"zfs snapshot {dataset}@{name}")
root = os.path.join(subject, ".zfs", "snapshot", name)
return root, f"zfs destroy {dataset}@{name}"
_CREATE = {"btrfs": _btrfs, "zfs": _zfs}
@contextmanager
def volume_snapshot(
kind: str,
subject: str,
tag: str,
run: Callable[[str], list[str]] = execute_shell_command,
) -> Iterator[Callable[[str], str]]:
"""Yield a resolver mapping a path under ``subject`` into a snapshot of it.
Args:
kind: ``btrfs`` or ``zfs``; the caller states it, nothing is probed.
subject: the btrfs subvolume or zfs dataset mountpoint holding the
volumes, e.g. ``/var/lib/docker``.
tag: unique suffix for the snapshot name, e.g. the backup timestamp.
run: shell runner, injected so the mechanics are testable.
Raises:
SnapshotError: the kind is unknown, or the snapshot cannot be created.
"""
create = _CREATE.get(kind)
if create is None:
raise SnapshotError(f"unknown snapshot kind {kind!r}; expected one of {KINDS}")
root, remove = create(subject, f"baudolo-{tag}", run)
try:
yield _resolver(subject, root)
finally:
run(remove)

10
src/baudolo/backup/volume.py
View File

@@ -25,7 +25,12 @@ def get_last_backup_dir(
def backup_volume(
versions_dir: str, volume_name: str, volume_dir: str, *, authoritative: bool
versions_dir: str,
volume_name: str,
volume_dir: str,
*,
authoritative: bool,
source: str,
) -> None:
"""Perform incremental file backup of a Docker volume.
@@ -34,13 +39,14 @@ def backup_volume(
size and whole-second mtime. Required on a pass whose destination was
already written from a live source, where a file can differ while
both attributes still agree.
source: directory to read from - the volume's mountpoint, or its path
inside a snapshot.
"""
dest = os.path.join(volume_dir, "files") + "/"
pathlib.Path(dest).mkdir(parents=True, exist_ok=True)
last = get_last_backup_dir(versions_dir, volume_name, dest)
link_dest = f"--link-dest='{last}'" if last else ""
source = get_storage_path(volume_name)
verify = "--checksum " if authoritative else ""
cmd = f"rsync -abP --delete --delete-excluded {verify}{link_dest} {source} {dest}"