mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2026-08-01 12:34:50 +00:00
feat(backup): capture volumes from a filesystem snapshot
Backing up a live volume with rsync copies a moving target: a database
written to mid-copy lands on disk in a state no engine ever committed.
Stopping the container avoids that at the cost of downtime.
A snapshot removes both. `--snapshot {btrfs,zfs}` with `--snapshot-subject`
freezes the docker root once per run, and every volume copy is then read
from that frozen tree while the containers keep serving. A restore of such
a copy is an ordinary crash recovery, which every supported engine performs
on its own at startup.
An unsupported filesystem or an unknown snapshot kind fails loudly rather
than degrading to a live copy, since a silent fallback would return exactly
the torn backup the mode exists to prevent. `--shutdown` is rejected
alongside `--snapshot` instead of being ignored: under a snapshot no
container is ever stopped, so accepting the flag would promise downtime
semantics the run does not deliver.
Copies out of a snapshot skip rsync's --checksum verification. The source
is immutable for the lifetime of the copy, so size-and-mtime cannot race,
and dropping the second full read roughly halves the I/O per volume.
backup/app.py grew past what one module could carry and is split into
layout, policy and dumps along the lines it already had internally.
Tests: unit coverage for the new snapshot, layout, policy, volume and cli
units; e2e cases drive real btrfs, zfs and ext4 filesystems on loop devices
in a privileged container, including a MariaDB that is written to across
the snapshot and must recover from the restored copy without losing a
committed row. CI installs zfs and sets E2E_REQUIRE_FILESYSTEMS so a
missing kernel module fails the build instead of silently skipping a
filesystem.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
98
src/baudolo/backup/dumps.py
Normal file
98
src/baudolo/backup/dumps.py
Normal file
@@ -0,0 +1,98 @@
|
||||
"""Database dumps taken before a volume's files are copied."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
import pandas
|
||||
from pandas.errors import EmptyDataError
|
||||
|
||||
from .db import backup_database
|
||||
from .docker import has_image
|
||||
|
||||
|
||||
def backup_mariadb_or_postgres(
|
||||
*,
|
||||
container: str,
|
||||
volume_dir: str,
|
||||
databases_df: "pandas.DataFrame",
|
||||
database_containers: list[str],
|
||||
) -> tuple[bool, bool]:
|
||||
"""
|
||||
Returns (is_db_container, dumped_any)
|
||||
"""
|
||||
for img in ["mariadb", "postgres"]:
|
||||
if has_image(container, img):
|
||||
dumped = backup_database(
|
||||
container=container,
|
||||
volume_dir=volume_dir,
|
||||
db_type=img,
|
||||
databases_df=databases_df,
|
||||
database_containers=database_containers,
|
||||
)
|
||||
return True, dumped
|
||||
return False, False
|
||||
|
||||
|
||||
def _empty_databases_df() -> "pandas.DataFrame":
|
||||
"""
|
||||
Create an empty DataFrame with the expected schema for databases.csv.
|
||||
|
||||
This allows the backup to continue without DB dumps when the CSV is missing
|
||||
or empty (pandas EmptyDataError).
|
||||
"""
|
||||
return pandas.DataFrame(columns=["instance", "database", "username", "password"])
|
||||
|
||||
|
||||
def load_databases_df(csv_path: str) -> "pandas.DataFrame":
|
||||
"""
|
||||
Load databases.csv robustly.
|
||||
|
||||
- Missing file -> warn, continue with empty df
|
||||
- Empty file -> warn, continue with empty df
|
||||
- Valid CSV -> return dataframe
|
||||
"""
|
||||
try:
|
||||
return pandas.read_csv(csv_path, sep=";", keep_default_na=False, dtype=str)
|
||||
except FileNotFoundError:
|
||||
print(
|
||||
f"WARNING: databases.csv not found: {csv_path}. Continuing without database dumps.",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
return _empty_databases_df()
|
||||
except EmptyDataError:
|
||||
print(
|
||||
f"WARNING: databases.csv exists but is empty: {csv_path}. Continuing without database dumps.",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
return _empty_databases_df()
|
||||
|
||||
|
||||
def backup_dumps_for_volume(
|
||||
*,
|
||||
containers: list[str],
|
||||
vol_dir: str,
|
||||
databases_df: "pandas.DataFrame",
|
||||
database_containers: list[str],
|
||||
) -> tuple[bool, bool]:
|
||||
"""
|
||||
Returns (found_db_container, dumped_any)
|
||||
"""
|
||||
found_db = False
|
||||
dumped_any = False
|
||||
|
||||
for c in containers:
|
||||
is_db, dumped = backup_mariadb_or_postgres(
|
||||
container=c,
|
||||
volume_dir=vol_dir,
|
||||
databases_df=databases_df,
|
||||
database_containers=database_containers,
|
||||
)
|
||||
if is_db:
|
||||
found_db = True
|
||||
if dumped:
|
||||
dumped_any = True
|
||||
|
||||
return found_db, dumped_any
|
||||
Reference in New Issue
Block a user