Compare commits

...

3 Commits

Author SHA1 Message Date
934e693810 Release version 3.3.0 2026-08-02 09:11:54 +02:00
2e0e67ca87 Autolint 2026-08-02 09:10:55 +02:00
95c34d4db0 feat(backup): exclude a volume by name, not only by image
volume_is_fully_ignored can only skip a volume when every container using it is ignored, so a container holding a derived tree next to state that must be kept cannot express the exclusion at all. The matrix docker-in-docker runner is exactly that: matrix_mdad_docker, matrix_mdad_matrix and matrix_mdad_state all hang off one container, and the derived one is an inner overlay2 store that no rsync in the chain can restore faithfully (none carries -X, so trusted.overlay.* is stripped in both directions).

--volumes-no-backup-required names volumes directly. The check runs before containers_using_volume, so an excluded volume costs no docker call and the decision no longer depends on which containers happen to exist at backup time.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-02 09:10:20 +02:00
15 changed files with 316 additions and 26 deletions

View File

@@ -1,5 +1,19 @@
# Changelog # Changelog
## [3.3.0] - 2026-08-02
- Backup: *--volumes-no-backup-required* excludes a volume by name.
*--images-no-backup-required* resolves through *volume_is_fully_ignored*,
which skips a volume only when every container using it is ignored — a
container holding a derived tree beside state that must be kept cannot
express the exclusion at all. A docker-in-docker data root is exactly that
shape, and excluding by image would drop all three of its volumes.
- Backup: the name check runs before *containers_using_volume*, so an excluded
volume costs no docker inspection and the decision does not depend on which
containers exist when the run starts.
- Tests: two volumes off one container, asserting the sibling survives — the
property the image lever cannot provide — as unit and end-to-end.
## [3.2.2] - 2026-07-31 ## [3.2.2] - 2026-07-31
- Backup: the btrfs snapshot is carved inside its subject, as - Backup: the btrfs snapshot is carved inside its subject, as

View File

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project] [project]
name = "backup-docker-to-local" name = "backup-docker-to-local"
version = "3.2.2" version = "3.3.0"
description = "Backup Docker volumes to local with rsync and optional DB dumps." description = "Backup Docker volumes to local with rsync and optional DB dumps."
readme = "README.md" readme = "README.md"
requires-python = ">=3.9" requires-python = ">=3.9"

View File

@@ -54,6 +54,14 @@ def main() -> int:
for volume_name in docker_volume_names(): for volume_name in docker_volume_names():
print(f"Start backup routine for volume: {volume_name}", flush=True) print(f"Start backup routine for volume: {volume_name}", flush=True)
if volume_name in args.volumes_no_backup_required:
print(
f"Skipping volume '{volume_name}' entirely (declared no-backup).",
flush=True,
)
continue
containers = containers_using_volume(volume_name) containers = containers_using_volume(volume_name)
if volume_is_fully_ignored(containers, args.images_no_backup_required): if volume_is_fully_ignored(containers, args.images_no_backup_required):

View File

@@ -68,6 +68,13 @@ def parse_args() -> argparse.Namespace:
help="Exact image references (repo:tag, incl. any registry prefix) for which no backup should be performed", help="Exact image references (repo:tag, incl. any registry prefix) for which no backup should be performed",
) )
p.add_argument(
"--volumes-no-backup-required",
nargs="+",
default=[],
help="Exact volume names that are never backed up, whatever containers use them. For derived trees a restore cannot reproduce, above all a nested docker data root",
)
p.add_argument( p.add_argument(
"--everything", "--everything",
action="store_true", action="store_true",
@@ -93,7 +100,9 @@ def parse_args() -> argparse.Namespace:
if bool(args.snapshot) != bool(args.snapshot_subject): if bool(args.snapshot) != bool(args.snapshot_subject):
p.error("--snapshot and --snapshot-subject must be given together") p.error("--snapshot and --snapshot-subject must be given together")
if args.snapshot and args.shutdown: if args.snapshot and args.shutdown:
p.error("--shutdown is meaningless with --snapshot: containers are never stopped") p.error(
"--shutdown is meaningless with --snapshot: containers are never stopped"
)
if args.snapshot and args.hard_restart_projects: if args.snapshot and args.hard_restart_projects:
p.error( p.error(
"--hard-restart-projects is meaningless with --snapshot: the flag exists " "--hard-restart-projects is meaningless with --snapshot: the flag exists "

View File

@@ -1,4 +1,5 @@
"""Fixtures and paths the e2e suite builds its scenarios from.""" """Fixtures and paths the e2e suite builds its scenarios from."""
from __future__ import annotations from __future__ import annotations
import shutil import shutil

View File

@@ -1,4 +1,5 @@
"""Process, docker and readiness helpers for the e2e suite.""" """Process, docker and readiness helpers for the e2e suite."""
from __future__ import annotations from __future__ import annotations
import subprocess import subprocess

View File

@@ -25,7 +25,9 @@ GENERATION = f"{VERSIONS}/20260731"
def shell(command: str) -> list[str]: def shell(command: str) -> list[str]:
proc = subprocess.run(command, shell=True, capture_output=True, text=True) proc = subprocess.run(command, shell=True, capture_output=True, text=True)
if proc.returncode != 0: if proc.returncode != 0:
raise SnapshotError(f"{command} exited {proc.returncode}: {proc.stderr.strip()}") raise SnapshotError(
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
)
return proc.stdout.splitlines() return proc.stdout.splitlines()

View File

@@ -22,7 +22,9 @@ EXPECT = sys.argv[3]
def shell(command: str) -> list[str]: def shell(command: str) -> list[str]:
proc = subprocess.run(command, shell=True, capture_output=True, text=True) proc = subprocess.run(command, shell=True, capture_output=True, text=True)
if proc.returncode != 0: if proc.returncode != 0:
raise SnapshotError(f"{command} exited {proc.returncode}: {proc.stderr.strip()}") raise SnapshotError(
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
)
return proc.stdout.splitlines() return proc.stdout.splitlines()
@@ -55,5 +57,8 @@ with volume_snapshot(KIND, SUBJECT, "e2e", run=shell) as resolve:
root = Path(resolve(SUBJECT)) root = Path(resolve(SUBJECT))
check("the snapshot is removed afterwards", not root.exists() or not (root / "volumes").exists()) check(
"the snapshot is removed afterwards",
not root.exists() or not (root / "volumes").exists(),
)
print("ALL OK", flush=True) print("ALL OK", flush=True)

View File

@@ -29,7 +29,7 @@ LOOP_FS = {
"ext4": "mkdir -p /subject/docker", "ext4": "mkdir -p /subject/docker",
} }
# A container carries no /lib/modules, so modprobe fails even on a loaded module. # A container carries no /lib/modules, so modprobe fails even on a loaded module.
ZFS_READY = '{ [ -c /dev/zfs ] || modprobe zfs 2>/dev/null; }; [ -c /dev/zfs ]' ZFS_READY = "{ [ -c /dev/zfs ] || modprobe zfs 2>/dev/null; }; [ -c /dev/zfs ]"
def mount_script(fstype: str) -> str: def mount_script(fstype: str) -> str:
@@ -51,7 +51,13 @@ def zfs_usable() -> bool:
"""Whether this host's kernel can serve zfs to a privileged container.""" """Whether this host's kernel can serve zfs to a privileged container."""
proc = run( proc = run(
[ [
"docker", "run", "--rm", "--privileged", IMAGE, "sh", "-lc", "docker",
"run",
"--rm",
"--privileged",
IMAGE,
"sh",
"-lc",
f"apk add -q zfs >/dev/null 2>&1 && {ZFS_READY}", f"apk add -q zfs >/dev/null 2>&1 && {ZFS_READY}",
], ],
capture=True, capture=True,
@@ -87,11 +93,20 @@ def drive(fstype: str, kind: str, expect: str) -> str:
try: try:
proc = run( proc = run(
[ [
"docker", "run", "--rm", "--privileged", "docker",
"--name", staged.name, "run",
"-v", f"{staged / 'src'}:/src:ro", "--rm",
"-v", f"{staged / 'driver.py'}:/driver.py:ro", "--privileged",
IMAGE, "sh", "-lc", script, "--name",
staged.name,
"-v",
f"{staged / 'src'}:/src:ro",
"-v",
f"{staged / 'driver.py'}:/driver.py:ro",
IMAGE,
"sh",
"-lc",
script,
], ],
capture=True, capture=True,
check=False, check=False,
@@ -99,7 +114,9 @@ def drive(fstype: str, kind: str, expect: str) -> str:
finally: finally:
shutil.rmtree(staged, ignore_errors=True) shutil.rmtree(staged, ignore_errors=True)
if proc.returncode != 0: if proc.returncode != 0:
raise AssertionError(f"{fstype}/{kind} driver failed:\n{proc.stdout}\n{proc.stderr}") raise AssertionError(
f"{fstype}/{kind} driver failed:\n{proc.stdout}\n{proc.stderr}"
)
return proc.stdout return proc.stdout
@@ -125,7 +142,9 @@ class TestE2ESnapshot(unittest.TestCase):
"E2E_REQUIRE_FILESYSTEMS demands zfs, but this kernel provides no " "E2E_REQUIRE_FILESYSTEMS demands zfs, but this kernel provides no "
"zfs module; load it before running the suite" "zfs module; load it before running the suite"
) )
self.skipTest("this kernel provides no zfs module, so no pool can be created") self.skipTest(
"this kernel provides no zfs module, so no pool can be created"
)
self.assert_freezes("zfs") self.assert_freezes("zfs")
def test_ext4_has_no_snapshot_and_says_so(self) -> None: def test_ext4_has_no_snapshot_and_says_so(self) -> None:

View File

@@ -68,11 +68,20 @@ class TestE2ESnapshotDatabase(unittest.TestCase):
try: try:
proc = run( proc = run(
[ [
"docker", "run", "--rm", "--privileged", "docker",
"--name", staged.name, "run",
"-v", f"{staged / 'src'}:/src:ro", "--rm",
"-v", f"{staged / 'driver.py'}:/driver.py:ro", "--privileged",
IMAGE, "sh", "-lc", SCRIPT, "--name",
staged.name,
"-v",
f"{staged / 'src'}:/src:ro",
"-v",
f"{staged / 'driver.py'}:/driver.py:ro",
IMAGE,
"sh",
"-lc",
SCRIPT,
], ],
capture=True, capture=True,
check=False, check=False,

View File

@@ -0,0 +1,125 @@
import unittest
from .helpers import (
backup_path,
cleanup_docker,
create_minimal_compose_dir,
ensure_empty_dir,
latest_version_dir,
require_docker,
run,
unique,
write_databases_csv,
)
class TestE2EVolumesNoBackupRequiredEarlySkip(unittest.TestCase):
"""Both volumes hang off the same container, so an image-level exclusion
could only drop both. Only the named one may disappear."""
@classmethod
def setUpClass(cls) -> None:
require_docker()
cls.prefix = unique("baudolo-e2e-early-skip-no-backup-volume")
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
ensure_empty_dir(cls.backups_dir)
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
cls.repo_name = cls.prefix
cls.container = f"{cls.prefix}-app"
cls.excluded_volume = f"{cls.prefix}-derived-vol"
cls.kept_volume = f"{cls.prefix}-state-vol"
cls.containers = [cls.container]
cls.volumes = [cls.excluded_volume, cls.kept_volume]
run(["docker", "volume", "create", cls.excluded_volume])
run(["docker", "volume", "create", cls.kept_volume])
run(
[
"docker",
"run",
"--rm",
"-v",
f"{cls.excluded_volume}:/derived",
"-v",
f"{cls.kept_volume}:/state",
"alpine:3.20",
"sh",
"-lc",
"echo derived > /derived/derived.txt && echo state > /state/state.txt",
]
)
run(
[
"docker",
"run",
"-d",
"--name",
cls.container,
"-v",
f"{cls.excluded_volume}:/derived",
"-v",
f"{cls.kept_volume}:/state",
"alpine:3.20",
"sleep",
"600",
]
)
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
write_databases_csv(cls.databases_csv, [])
cmd = [
"baudolo",
"--compose-dir",
cls.compose_dir,
"--repo-name",
cls.repo_name,
"--databases-csv",
cls.databases_csv,
"--backups-dir",
cls.backups_dir,
"--images-no-stop-required",
"alpine:3.20",
"--volumes-no-backup-required",
cls.excluded_volume,
]
cp = run(cmd, capture=True, check=True)
cls.stdout = cp.stdout or ""
cls.stderr = cp.stderr or ""
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
@classmethod
def tearDownClass(cls) -> None:
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
def test_excluded_volume_has_no_backup_directory_at_all(self) -> None:
p = backup_path(
self.backups_dir,
self.repo_name,
self.version,
self.excluded_volume,
)
self.assertFalse(
p.exists(),
f"Expected NO backup directory for the excluded volume, but found: {p}",
)
def test_sibling_volume_of_the_same_container_is_still_backed_up(self) -> None:
p = (
backup_path(
self.backups_dir,
self.repo_name,
self.version,
self.kept_volume,
)
/ "files"
/ "state.txt"
)
self.assertTrue(p.is_file(), f"Expected backed up file at: {p}")

View File

@@ -12,6 +12,7 @@ from baudolo.backup.snapshot import volume_snapshot
def stubbed_snapshot(kind: str, subject: str, tag: str): def stubbed_snapshot(kind: str, subject: str, tag: str):
return volume_snapshot(kind, subject, tag, run=lambda command: []) return volume_snapshot(kind, subject, tag, run=lambda command: [])
ARGV = [ ARGV = [
"baudolo", "baudolo",
"--compose-dir", "--compose-dir",

View File

@@ -0,0 +1,82 @@
"""Contract of --volumes-no-backup-required: exclusion is per volume name,
independent of which containers use it."""
from __future__ import annotations
import unittest
from unittest import mock
from baudolo.backup import app
ARGV = [
"baudolo",
"--compose-dir",
"/compose",
"--backups-dir",
"/backups",
"--volumes-no-backup-required",
"derived",
]
def drive() -> tuple[list[str], list[str], list[str]]:
backed_up: list[str] = []
created: list[str] = []
inspected: list[str] = []
def record_backup(versions_dir, volume_name, volume_dir, *, authoritative, source):
backed_up.append(volume_name)
with (
mock.patch("sys.argv", ARGV),
mock.patch.object(app, "get_machine_id", return_value="machine"),
mock.patch.object(app, "create_version_directory", return_value="/gen"),
mock.patch.object(
app,
"create_volume_directory",
side_effect=lambda _version_dir, name: created.append(name) or "/gen/vol",
),
mock.patch.object(app, "load_databases_df", return_value=None),
mock.patch.object(
app, "docker_volume_names", return_value=["derived", "state"]
),
mock.patch.object(
app,
"containers_using_volume",
side_effect=lambda name: inspected.append(name) or ["app"],
),
mock.patch.object(app, "volume_is_fully_ignored", return_value=False),
mock.patch.object(app, "backup_dumps_for_volume", return_value=(False, False)),
mock.patch.object(app, "get_storage_path", return_value="/data/"),
mock.patch.object(app, "stamp_directory"),
mock.patch.object(app, "handle_docker_compose_services"),
mock.patch.object(app.os.path, "isdir", return_value=True),
mock.patch.object(app, "backup_volume", side_effect=record_backup),
mock.patch.object(app, "filter_stoppable", return_value=[]),
mock.patch.object(app, "requires_stop", return_value=False),
mock.patch.object(app, "change_containers_status"),
):
app.main()
return backed_up, created, inspected
class TestVolumesNoBackupRequired(unittest.TestCase):
def test_the_named_volume_is_never_backed_up(self) -> None:
backed_up, _created, _inspected = drive()
self.assertNotIn("derived", backed_up)
def test_a_sibling_volume_of_the_same_container_survives(self) -> None:
backed_up, _created, _inspected = drive()
self.assertEqual(backed_up, ["state"])
def test_no_generation_directory_is_created_for_it(self) -> None:
_backed_up, created, _inspected = drive()
self.assertEqual(created, ["state"])
def test_the_skip_precedes_the_container_inspection(self) -> None:
_backed_up, _created, inspected = drive()
self.assertEqual(inspected, ["state"])
if __name__ == "__main__":
unittest.main()

View File

@@ -39,7 +39,9 @@ class TestSnapshotFlags(unittest.TestCase):
parse("--snapshot", "ext4", "--snapshot-subject", "/var/lib/docker") parse("--snapshot", "ext4", "--snapshot-subject", "/var/lib/docker")
def test_zfs_is_accepted(self) -> None: def test_zfs_is_accepted(self) -> None:
self.assertEqual(parse("--snapshot", "zfs", "--snapshot-subject", "/d").snapshot, "zfs") self.assertEqual(
parse("--snapshot", "zfs", "--snapshot-subject", "/d").snapshot, "zfs"
)
def test_shutdown_is_rejected_because_nothing_is_stopped(self) -> None: def test_shutdown_is_rejected_because_nothing_is_stopped(self) -> None:
with self.assertRaises(SystemExit): with self.assertRaises(SystemExit):

View File

@@ -35,11 +35,15 @@ class TestBtrfs(unittest.TestCase):
run = Runner() run = Runner()
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run): with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
pass pass
self.assertEqual(run.calls[-1], "btrfs subvolume delete /var/lib/docker/.baudolo-20260731") self.assertEqual(
run.calls[-1], "btrfs subvolume delete /var/lib/docker/.baudolo-20260731"
)
def test_it_maps_a_volume_path_into_the_snapshot(self) -> None: def test_it_maps_a_volume_path_into_the_snapshot(self) -> None:
run = Runner() run = Runner()
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve: with volume_snapshot(
"btrfs", "/var/lib/docker", "20260731", run=run
) as resolve:
self.assertEqual( self.assertEqual(
resolve("/var/lib/docker/volumes/postgres_data/_data"), resolve("/var/lib/docker/volumes/postgres_data/_data"),
"/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data", "/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data",
@@ -47,7 +51,9 @@ class TestBtrfs(unittest.TestCase):
def test_it_keeps_the_trailing_slash_rsync_reads_as_contents(self) -> None: def test_it_keeps_the_trailing_slash_rsync_reads_as_contents(self) -> None:
run = Runner() run = Runner()
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve: with volume_snapshot(
"btrfs", "/var/lib/docker", "20260731", run=run
) as resolve:
self.assertEqual( self.assertEqual(
resolve("/var/lib/docker/volumes/postgres_data/_data/"), resolve("/var/lib/docker/volumes/postgres_data/_data/"),
"/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data/", "/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data/",
@@ -102,14 +108,20 @@ class TestRejections(unittest.TestCase):
def test_a_path_outside_the_subject_is_rejected(self) -> None: def test_a_path_outside_the_subject_is_rejected(self) -> None:
run = Runner() run = Runner()
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve: with volume_snapshot(
"btrfs", "/var/lib/docker", "20260731", run=run
) as resolve:
with self.assertRaises(SnapshotError): with self.assertRaises(SnapshotError):
resolve("/etc/passwd") resolve("/etc/passwd")
def test_the_subject_itself_resolves_to_the_snapshot_root(self) -> None: def test_the_subject_itself_resolves_to_the_snapshot_root(self) -> None:
run = Runner() run = Runner()
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve: with volume_snapshot(
self.assertEqual(resolve("/var/lib/docker"), "/var/lib/docker/.baudolo-20260731") "btrfs", "/var/lib/docker", "20260731", run=run
) as resolve:
self.assertEqual(
resolve("/var/lib/docker"), "/var/lib/docker/.baudolo-20260731"
)
class Busy(Runner): class Busy(Runner):