mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2026-08-01 12:34:50 +00:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c2f1cb8e8c | |||
| eeaa838d02 | |||
| 4e2b3641f9 | |||
| b10d50efbe |
41
CHANGELOG.md
41
CHANGELOG.md
@@ -1,5 +1,46 @@
|
||||
# Changelog
|
||||
|
||||
## [3.2.2] - 2026-07-31
|
||||
|
||||
- Backup: the btrfs snapshot is carved inside its subject, as
|
||||
*<data root>/.baudolo-<tag>*, not beside it. The kernel refuses a snapshot
|
||||
whose destination is on another filesystem, which is exactly what the parent
|
||||
directory is when the data root is a mountpoint of its own — a dedicated disk
|
||||
mounted onto */var/lib/docker* failed every run with EXDEV. Placing it inside
|
||||
makes source and destination the same filesystem by construction, and aligns
|
||||
btrfs with the zfs path, which already resolves its snapshot inside the
|
||||
subject at *<subject>/.zfs/snapshot/<tag>*. A leftover from an interrupted run
|
||||
appears in the next snapshot as an empty directory rather than recursing,
|
||||
since btrfs does not include nested subvolumes.
|
||||
|
||||
## [3.2.1] - 2026-07-31
|
||||
|
||||
- Backup: the snapshot resolver keeps the trailing separator *get_storage_path*
|
||||
puts on a volume path — *os.path.abspath* stripped it. rsync reads *dir* as
|
||||
"copy the directory" where *dir/* means "copy its contents", so every snapshot
|
||||
generation landed at *<volume>/files/_data/...* while the live path lands at
|
||||
*<volume>/files/...*. Restores read the live layout, and *--link-dest* had
|
||||
nothing to match against the previous generation.
|
||||
- Backup: snapshot teardown no longer fails a completed run. A busy
|
||||
*btrfs subvolume delete* raised out of the *finally*, skipping the generation
|
||||
stamp and the compose handling on a run whose data was already copied, and
|
||||
masking whatever the body had raised. The leftover is reported instead.
|
||||
- Backup: a volume created after the snapshot was taken is copied live with a
|
||||
warning instead of aborting the run. Nothing is stopped in snapshot mode, so
|
||||
the host keeps creating volumes for the whole copy.
|
||||
- Backup: the snapshot pass compares by content (*--checksum*) again. 3.2.0
|
||||
dropped it because a snapshot source cannot move, which is true, but the
|
||||
comparison that matters is against *--link-dest*: a file that changed while
|
||||
keeping its size and whole-second mtime was hard-linked stale out of the
|
||||
previous generation, and the single pass had no authoritative pass to repair
|
||||
it. Still one pass where the live path takes two.
|
||||
- Backup: *--hard-restart-projects* is refused alongside *--snapshot*, like
|
||||
*--shutdown* already is. It exists for stacks whose database cannot be backed
|
||||
up hot, which is what a snapshot removes.
|
||||
- Tests: the trailing separator, both teardown behaviours, the new refusal, and
|
||||
*app.main* driving the snapshot branch — the caller that runs in production,
|
||||
which no test had exercised, which is why the layout defect shipped.
|
||||
|
||||
## [3.2.0] - 2026-07-31
|
||||
|
||||
- Backup: *--snapshot {btrfs,zfs}* with *--snapshot-subject* captures every
|
||||
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "backup-docker-to-local"
|
||||
version = "3.2.0"
|
||||
version = "3.2.2"
|
||||
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
||||
@@ -95,7 +95,16 @@ def main() -> int:
|
||||
)
|
||||
|
||||
if resolve_source is not None:
|
||||
copy(authoritative=False, source=resolve_source(live_source))
|
||||
snapshot_source = resolve_source(live_source)
|
||||
if os.path.isdir(snapshot_source):
|
||||
copy(authoritative=True, source=snapshot_source)
|
||||
else:
|
||||
print(
|
||||
f"WARNING: volume '{volume_name}' is not in the snapshot "
|
||||
"(created after it was taken); copying it live instead.",
|
||||
flush=True,
|
||||
)
|
||||
copy(authoritative=False)
|
||||
continue
|
||||
|
||||
if args.everything:
|
||||
|
||||
@@ -94,4 +94,9 @@ def parse_args() -> argparse.Namespace:
|
||||
p.error("--snapshot and --snapshot-subject must be given together")
|
||||
if args.snapshot and args.shutdown:
|
||||
p.error("--shutdown is meaningless with --snapshot: containers are never stopped")
|
||||
if args.snapshot and args.hard_restart_projects:
|
||||
p.error(
|
||||
"--hard-restart-projects is meaningless with --snapshot: the flag exists "
|
||||
"for stacks whose database cannot be backed up hot, which a snapshot solves"
|
||||
)
|
||||
return args
|
||||
|
||||
@@ -17,12 +17,11 @@ import os
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
|
||||
from .shell import execute_shell_command
|
||||
from .shell import BackupException, execute_shell_command
|
||||
|
||||
KINDS = ("btrfs", "zfs")
|
||||
|
||||
|
||||
|
||||
class SnapshotError(RuntimeError):
|
||||
"""A snapshot could not be created, resolved or removed."""
|
||||
|
||||
@@ -32,13 +31,20 @@ def _resolver(subject: str, root: str) -> Callable[[str], str]:
|
||||
relative = os.path.relpath(os.path.abspath(path), os.path.abspath(subject))
|
||||
if relative.startswith(".."):
|
||||
raise SnapshotError(f"{path} lies outside the snapshot subject {subject}")
|
||||
return os.path.join(root, relative) if relative != "." else root
|
||||
resolved = root if relative == "." else os.path.join(root, relative)
|
||||
|
||||
# abspath drops a trailing separator, and rsync reads "dir/" as its
|
||||
# contents where "dir" means the directory itself.
|
||||
return resolved + os.sep if path.endswith(os.sep) else resolved
|
||||
|
||||
return resolve
|
||||
|
||||
|
||||
def _btrfs(subject: str, name: str, run: Callable[[str], list[str]]) -> tuple[str, str]:
|
||||
target = os.path.join(os.path.dirname(os.path.abspath(subject)), f".{name}")
|
||||
# The snapshot goes inside the subject, never beside it: the kernel rejects
|
||||
# a snapshot whose destination is on another filesystem, which is exactly
|
||||
# what the parent directory is when the subject is a mountpoint of its own.
|
||||
target = os.path.join(os.path.abspath(subject), f".{name}")
|
||||
run(f"btrfs subvolume snapshot -r {subject} {target}")
|
||||
return target, f"btrfs subvolume delete {target}"
|
||||
|
||||
@@ -74,6 +80,8 @@ def volume_snapshot(
|
||||
|
||||
Raises:
|
||||
SnapshotError: the kind is unknown, or the snapshot cannot be created.
|
||||
Removal failure is reported, not raised: a leftover snapshot is a
|
||||
cleanup problem and must not discard a generation that is complete.
|
||||
"""
|
||||
create = _CREATE.get(kind)
|
||||
if create is None:
|
||||
@@ -83,4 +91,8 @@ def volume_snapshot(
|
||||
try:
|
||||
yield _resolver(subject, root)
|
||||
finally:
|
||||
try:
|
||||
run(remove)
|
||||
except BackupException as error:
|
||||
# Raising here would also mask whatever the body raised.
|
||||
print(f"WARNING: {root} could not be removed: {error}", flush=True)
|
||||
|
||||
@@ -34,8 +34,8 @@ with volume_snapshot("btrfs", SUBJECT, "dbtest", run=shell) as resolve:
|
||||
VERSIONS,
|
||||
VOLUME,
|
||||
f"{GENERATION}/{VOLUME}",
|
||||
authoritative=False,
|
||||
source=resolve(DATADIR) + "/",
|
||||
authoritative=True,
|
||||
source=resolve(f"{DATADIR}/"),
|
||||
)
|
||||
|
||||
print("SNAPSHOT COPY DONE", flush=True)
|
||||
|
||||
79
tests/unit/backup/test_app_snapshot.py
Normal file
79
tests/unit/backup/test_app_snapshot.py
Normal file
@@ -0,0 +1,79 @@
|
||||
"""Contract of app.main's snapshot branch - the caller that runs in production."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from baudolo.backup import app
|
||||
from baudolo.backup.snapshot import volume_snapshot
|
||||
|
||||
|
||||
def stubbed_snapshot(kind: str, subject: str, tag: str):
|
||||
return volume_snapshot(kind, subject, tag, run=lambda command: [])
|
||||
|
||||
ARGV = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
"/compose",
|
||||
"--backups-dir",
|
||||
"/backups",
|
||||
"--snapshot",
|
||||
"btrfs",
|
||||
"--snapshot-subject",
|
||||
"/var/lib/docker",
|
||||
]
|
||||
|
||||
|
||||
def drive(*, present: bool) -> list[dict]:
|
||||
calls: list[dict] = []
|
||||
|
||||
def record(versions_dir, volume_name, volume_dir, *, authoritative, source):
|
||||
calls.append(
|
||||
{"volume": volume_name, "authoritative": authoritative, "source": source}
|
||||
)
|
||||
|
||||
with (
|
||||
mock.patch("sys.argv", ARGV),
|
||||
mock.patch.object(app, "get_machine_id", return_value="machine"),
|
||||
mock.patch.object(app, "create_version_directory", return_value="/gen"),
|
||||
mock.patch.object(app, "create_volume_directory", return_value="/gen/vol"),
|
||||
mock.patch.object(app, "load_databases_df", return_value=None),
|
||||
mock.patch.object(app, "docker_volume_names", return_value=["vol"]),
|
||||
mock.patch.object(app, "containers_using_volume", return_value=[]),
|
||||
mock.patch.object(app, "volume_is_fully_ignored", return_value=False),
|
||||
mock.patch.object(app, "backup_dumps_for_volume", return_value=(False, False)),
|
||||
mock.patch.object(
|
||||
app, "get_storage_path", return_value="/var/lib/docker/volumes/vol/_data/"
|
||||
),
|
||||
mock.patch.object(app, "stamp_directory"),
|
||||
mock.patch.object(app, "handle_docker_compose_services"),
|
||||
mock.patch.object(app.os.path, "isdir", return_value=present),
|
||||
mock.patch.object(app, "backup_volume", side_effect=record),
|
||||
mock.patch.object(app, "volume_snapshot", stubbed_snapshot),
|
||||
):
|
||||
app.main()
|
||||
return calls
|
||||
|
||||
|
||||
class TestSnapshotBranch(unittest.TestCase):
|
||||
def test_it_passes_a_path_ending_in_a_separator(self) -> None:
|
||||
source = drive(present=True)[0]["source"]
|
||||
self.assertTrue(source.endswith("/volumes/vol/_data/"), source)
|
||||
self.assertNotIn("/var/lib/docker/volumes", source)
|
||||
|
||||
def test_it_reads_from_the_snapshot_and_not_from_the_live_tree(self) -> None:
|
||||
source = drive(present=True)[0]["source"]
|
||||
self.assertTrue(source.startswith("/var/lib/docker/.baudolo-"), source)
|
||||
|
||||
def test_it_compares_by_content_against_the_previous_generation(self) -> None:
|
||||
self.assertTrue(drive(present=True)[0]["authoritative"])
|
||||
|
||||
def test_a_volume_missing_from_the_snapshot_is_copied_live(self) -> None:
|
||||
call = drive(present=False)[0]
|
||||
self.assertEqual(call["source"], "/var/lib/docker/volumes/vol/_data/")
|
||||
self.assertFalse(call["authoritative"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -48,6 +48,22 @@ class TestSnapshotFlags(unittest.TestCase):
|
||||
def test_shutdown_stays_available_without_a_snapshot(self) -> None:
|
||||
self.assertTrue(parse("--shutdown").shutdown)
|
||||
|
||||
def test_hard_restart_is_rejected_because_nothing_is_stopped(self) -> None:
|
||||
with self.assertRaises(SystemExit):
|
||||
parse(
|
||||
"--snapshot",
|
||||
"btrfs",
|
||||
"--snapshot-subject",
|
||||
"/d",
|
||||
"--hard-restart-projects",
|
||||
"mailu",
|
||||
)
|
||||
|
||||
def test_hard_restart_stays_available_without_a_snapshot(self) -> None:
|
||||
self.assertEqual(
|
||||
parse("--hard-restart-projects", "mailu").hard_restart_projects, ["mailu"]
|
||||
)
|
||||
|
||||
|
||||
class TestRequiredFlags(unittest.TestCase):
|
||||
def test_backups_dir_is_required(self) -> None:
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from baudolo.backup.shell import BackupException
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
||||
|
||||
|
||||
@@ -21,27 +22,35 @@ class Runner:
|
||||
|
||||
|
||||
class TestBtrfs(unittest.TestCase):
|
||||
def test_it_creates_a_read_only_snapshot_beside_the_subject(self) -> None:
|
||||
def test_it_creates_a_read_only_snapshot_inside_the_subject(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
||||
pass
|
||||
self.assertEqual(
|
||||
run.calls[0],
|
||||
"btrfs subvolume snapshot -r /var/lib/docker /var/lib/.baudolo-20260731",
|
||||
"btrfs subvolume snapshot -r /var/lib/docker /var/lib/docker/.baudolo-20260731",
|
||||
)
|
||||
|
||||
def test_it_removes_the_snapshot_afterwards(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
||||
pass
|
||||
self.assertEqual(run.calls[-1], "btrfs subvolume delete /var/lib/.baudolo-20260731")
|
||||
self.assertEqual(run.calls[-1], "btrfs subvolume delete /var/lib/docker/.baudolo-20260731")
|
||||
|
||||
def test_it_maps_a_volume_path_into_the_snapshot(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
||||
self.assertEqual(
|
||||
resolve("/var/lib/docker/volumes/postgres_data/_data"),
|
||||
"/var/lib/.baudolo-20260731/volumes/postgres_data/_data",
|
||||
"/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data",
|
||||
)
|
||||
|
||||
def test_it_keeps_the_trailing_slash_rsync_reads_as_contents(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
||||
self.assertEqual(
|
||||
resolve("/var/lib/docker/volumes/postgres_data/_data/"),
|
||||
"/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data/",
|
||||
)
|
||||
|
||||
def test_it_removes_the_snapshot_even_when_the_body_raises(self) -> None:
|
||||
@@ -100,7 +109,25 @@ class TestRejections(unittest.TestCase):
|
||||
def test_the_subject_itself_resolves_to_the_snapshot_root(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
||||
self.assertEqual(resolve("/var/lib/docker"), "/var/lib/.baudolo-20260731")
|
||||
self.assertEqual(resolve("/var/lib/docker"), "/var/lib/docker/.baudolo-20260731")
|
||||
|
||||
|
||||
class Busy(Runner):
|
||||
def __call__(self, command: str) -> list[str]:
|
||||
if command.startswith("btrfs subvolume delete"):
|
||||
raise BackupException("target is busy")
|
||||
return super().__call__(command)
|
||||
|
||||
|
||||
class TestRemovalFailure(unittest.TestCase):
|
||||
def test_a_failed_removal_does_not_fail_a_completed_run(self) -> None:
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=Busy()):
|
||||
pass
|
||||
|
||||
def test_a_failed_removal_does_not_mask_the_body(self) -> None:
|
||||
with self.assertRaises(ZeroDivisionError):
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=Busy()):
|
||||
raise ZeroDivisionError
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user