mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2026-08-01 12:34:50 +00:00
Kevin Veen-Birkenbach
d4317827bd
Backing up a live volume with rsync copies a moving target: a database
written to mid-copy lands on disk in a state no engine ever committed.
Stopping the container avoids that at the cost of downtime.
A snapshot removes both. `--snapshot {btrfs,zfs}` with `--snapshot-subject`
freezes the docker root once per run, and every volume copy is then read
from that frozen tree while the containers keep serving. A restore of such
a copy is an ordinary crash recovery, which every supported engine performs
on its own at startup.
An unsupported filesystem or an unknown snapshot kind fails loudly rather
than degrading to a live copy, since a silent fallback would return exactly
the torn backup the mode exists to prevent. `--shutdown` is rejected
alongside `--snapshot` instead of being ignored: under a snapshot no
container is ever stopped, so accepting the flag would promise downtime
semantics the run does not deliver.
Copies out of a snapshot skip rsync's --checksum verification. The source
is immutable for the lifetime of the copy, so size-and-mtime cannot race,
and dropping the second full read roughly halves the I/O per volume.
backup/app.py grew past what one module could carry and is split into
layout, policy and dumps along the lines it already had internally.
Tests: unit coverage for the new snapshot, layout, policy, volume and cli
units; e2e cases drive real btrfs, zfs and ext4 filesystems on loop devices
in a privileged container, including a MariaDB that is written to across
the snapshot and must recover from the restored copy without losing a
committed row. CI installs zfs and sets E2E_REQUIRE_FILESYSTEMS so a
missing kernel module fails the build instead of silently skipping a
filesystem.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
99 lines
2.6 KiB
Python
99 lines
2.6 KiB
Python
"""Database dumps taken before a volume's files are copied."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
|
|
import pandas
|
|
from pandas.errors import EmptyDataError
|
|
|
|
from .db import backup_database
|
|
from .docker import has_image
|
|
|
|
|
|
def backup_mariadb_or_postgres(
|
|
*,
|
|
container: str,
|
|
volume_dir: str,
|
|
databases_df: "pandas.DataFrame",
|
|
database_containers: list[str],
|
|
) -> tuple[bool, bool]:
|
|
"""
|
|
Returns (is_db_container, dumped_any)
|
|
"""
|
|
for img in ["mariadb", "postgres"]:
|
|
if has_image(container, img):
|
|
dumped = backup_database(
|
|
container=container,
|
|
volume_dir=volume_dir,
|
|
db_type=img,
|
|
databases_df=databases_df,
|
|
database_containers=database_containers,
|
|
)
|
|
return True, dumped
|
|
return False, False
|
|
|
|
|
|
def _empty_databases_df() -> "pandas.DataFrame":
|
|
"""
|
|
Create an empty DataFrame with the expected schema for databases.csv.
|
|
|
|
This allows the backup to continue without DB dumps when the CSV is missing
|
|
or empty (pandas EmptyDataError).
|
|
"""
|
|
return pandas.DataFrame(columns=["instance", "database", "username", "password"])
|
|
|
|
|
|
def load_databases_df(csv_path: str) -> "pandas.DataFrame":
|
|
"""
|
|
Load databases.csv robustly.
|
|
|
|
- Missing file -> warn, continue with empty df
|
|
- Empty file -> warn, continue with empty df
|
|
- Valid CSV -> return dataframe
|
|
"""
|
|
try:
|
|
return pandas.read_csv(csv_path, sep=";", keep_default_na=False, dtype=str)
|
|
except FileNotFoundError:
|
|
print(
|
|
f"WARNING: databases.csv not found: {csv_path}. Continuing without database dumps.",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
return _empty_databases_df()
|
|
except EmptyDataError:
|
|
print(
|
|
f"WARNING: databases.csv exists but is empty: {csv_path}. Continuing without database dumps.",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
return _empty_databases_df()
|
|
|
|
|
|
def backup_dumps_for_volume(
|
|
*,
|
|
containers: list[str],
|
|
vol_dir: str,
|
|
databases_df: "pandas.DataFrame",
|
|
database_containers: list[str],
|
|
) -> tuple[bool, bool]:
|
|
"""
|
|
Returns (found_db_container, dumped_any)
|
|
"""
|
|
found_db = False
|
|
dumped_any = False
|
|
|
|
for c in containers:
|
|
is_db, dumped = backup_mariadb_or_postgres(
|
|
container=c,
|
|
volume_dir=vol_dir,
|
|
databases_df=databases_df,
|
|
database_containers=database_containers,
|
|
)
|
|
if is_db:
|
|
found_db = True
|
|
if dumped:
|
|
dumped_any = True
|
|
|
|
return found_db, dumped_any
|