mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2026-08-02 13:02:41 +00:00
Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8a93a61ca9 | |||
| 988d92534c | |||
| 934e693810 | |||
| 2e0e67ca87 | |||
| 95c34d4db0 | |||
| c2f1cb8e8c | |||
| eeaa838d02 | |||
| 4e2b3641f9 | |||
| b10d50efbe |
70
CHANGELOG.md
70
CHANGELOG.md
@@ -1,5 +1,75 @@
|
||||
# Changelog
|
||||
|
||||
## [3.4.0] - 2026-08-02
|
||||
|
||||
- Backup: *-a* implies *-D*, so a generation was written with
|
||||
*--devices --specials* and rsync recreated every unix socket and fifo found in
|
||||
a volume. Where the backup root is an nfs-ganesha export, ganesha accepts the
|
||||
socket on write but cannot serve it back, and the remote pull's sender then
|
||||
fails with *readdir* / *readlink_stat* "Invalid argument (22)" and exits 23 —
|
||||
deterministically, for every retry. *--no-D* keeps them out of the generation.
|
||||
- Backup: nothing restorable is lost. Sockets and fifos are recreated by the
|
||||
daemons that own them, and the postfix queue itself — *incoming*, *active*,
|
||||
*deferred*, *hold*, *maildrop* — is unaffected, so accepted-but-undelivered
|
||||
mail stays in the backup. Device nodes go too; the only volume that could hold
|
||||
them is a nested docker data root, which does not belong in a backup anyway.
|
||||
- Tests: the flag is asserted on the rsync invocation.
|
||||
|
||||
## [3.3.0] - 2026-08-02
|
||||
|
||||
- Backup: *--volumes-no-backup-required* excludes a volume by name.
|
||||
*--images-no-backup-required* resolves through *volume_is_fully_ignored*,
|
||||
which skips a volume only when every container using it is ignored — a
|
||||
container holding a derived tree beside state that must be kept cannot
|
||||
express the exclusion at all. A docker-in-docker data root is exactly that
|
||||
shape, and excluding by image would drop all three of its volumes.
|
||||
- Backup: the name check runs before *containers_using_volume*, so an excluded
|
||||
volume costs no docker inspection and the decision does not depend on which
|
||||
containers exist when the run starts.
|
||||
- Tests: two volumes off one container, asserting the sibling survives — the
|
||||
property the image lever cannot provide — as unit and end-to-end.
|
||||
|
||||
## [3.2.2] - 2026-07-31
|
||||
|
||||
- Backup: the btrfs snapshot is carved inside its subject, as
|
||||
*<data root>/.baudolo-<tag>*, not beside it. The kernel refuses a snapshot
|
||||
whose destination is on another filesystem, which is exactly what the parent
|
||||
directory is when the data root is a mountpoint of its own — a dedicated disk
|
||||
mounted onto */var/lib/docker* failed every run with EXDEV. Placing it inside
|
||||
makes source and destination the same filesystem by construction, and aligns
|
||||
btrfs with the zfs path, which already resolves its snapshot inside the
|
||||
subject at *<subject>/.zfs/snapshot/<tag>*. A leftover from an interrupted run
|
||||
appears in the next snapshot as an empty directory rather than recursing,
|
||||
since btrfs does not include nested subvolumes.
|
||||
|
||||
## [3.2.1] - 2026-07-31
|
||||
|
||||
- Backup: the snapshot resolver keeps the trailing separator *get_storage_path*
|
||||
puts on a volume path — *os.path.abspath* stripped it. rsync reads *dir* as
|
||||
"copy the directory" where *dir/* means "copy its contents", so every snapshot
|
||||
generation landed at *<volume>/files/_data/...* while the live path lands at
|
||||
*<volume>/files/...*. Restores read the live layout, and *--link-dest* had
|
||||
nothing to match against the previous generation.
|
||||
- Backup: snapshot teardown no longer fails a completed run. A busy
|
||||
*btrfs subvolume delete* raised out of the *finally*, skipping the generation
|
||||
stamp and the compose handling on a run whose data was already copied, and
|
||||
masking whatever the body had raised. The leftover is reported instead.
|
||||
- Backup: a volume created after the snapshot was taken is copied live with a
|
||||
warning instead of aborting the run. Nothing is stopped in snapshot mode, so
|
||||
the host keeps creating volumes for the whole copy.
|
||||
- Backup: the snapshot pass compares by content (*--checksum*) again. 3.2.0
|
||||
dropped it because a snapshot source cannot move, which is true, but the
|
||||
comparison that matters is against *--link-dest*: a file that changed while
|
||||
keeping its size and whole-second mtime was hard-linked stale out of the
|
||||
previous generation, and the single pass had no authoritative pass to repair
|
||||
it. Still one pass where the live path takes two.
|
||||
- Backup: *--hard-restart-projects* is refused alongside *--snapshot*, like
|
||||
*--shutdown* already is. It exists for stacks whose database cannot be backed
|
||||
up hot, which is what a snapshot removes.
|
||||
- Tests: the trailing separator, both teardown behaviours, the new refusal, and
|
||||
*app.main* driving the snapshot branch — the caller that runs in production,
|
||||
which no test had exercised, which is why the layout defect shipped.
|
||||
|
||||
## [3.2.0] - 2026-07-31
|
||||
|
||||
- Backup: *--snapshot {btrfs,zfs}* with *--snapshot-subject* captures every
|
||||
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "backup-docker-to-local"
|
||||
version = "3.2.0"
|
||||
version = "3.4.0"
|
||||
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
||||
@@ -54,6 +54,14 @@ def main() -> int:
|
||||
|
||||
for volume_name in docker_volume_names():
|
||||
print(f"Start backup routine for volume: {volume_name}", flush=True)
|
||||
|
||||
if volume_name in args.volumes_no_backup_required:
|
||||
print(
|
||||
f"Skipping volume '{volume_name}' entirely (declared no-backup).",
|
||||
flush=True,
|
||||
)
|
||||
continue
|
||||
|
||||
containers = containers_using_volume(volume_name)
|
||||
|
||||
if volume_is_fully_ignored(containers, args.images_no_backup_required):
|
||||
@@ -95,7 +103,16 @@ def main() -> int:
|
||||
)
|
||||
|
||||
if resolve_source is not None:
|
||||
copy(authoritative=False, source=resolve_source(live_source))
|
||||
snapshot_source = resolve_source(live_source)
|
||||
if os.path.isdir(snapshot_source):
|
||||
copy(authoritative=True, source=snapshot_source)
|
||||
else:
|
||||
print(
|
||||
f"WARNING: volume '{volume_name}' is not in the snapshot "
|
||||
"(created after it was taken); copying it live instead.",
|
||||
flush=True,
|
||||
)
|
||||
copy(authoritative=False)
|
||||
continue
|
||||
|
||||
if args.everything:
|
||||
|
||||
@@ -68,6 +68,13 @@ def parse_args() -> argparse.Namespace:
|
||||
help="Exact image references (repo:tag, incl. any registry prefix) for which no backup should be performed",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--volumes-no-backup-required",
|
||||
nargs="+",
|
||||
default=[],
|
||||
help="Exact volume names that are never backed up, whatever containers use them. For derived trees a restore cannot reproduce, above all a nested docker data root",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--everything",
|
||||
action="store_true",
|
||||
@@ -93,5 +100,12 @@ def parse_args() -> argparse.Namespace:
|
||||
if bool(args.snapshot) != bool(args.snapshot_subject):
|
||||
p.error("--snapshot and --snapshot-subject must be given together")
|
||||
if args.snapshot and args.shutdown:
|
||||
p.error("--shutdown is meaningless with --snapshot: containers are never stopped")
|
||||
p.error(
|
||||
"--shutdown is meaningless with --snapshot: containers are never stopped"
|
||||
)
|
||||
if args.snapshot and args.hard_restart_projects:
|
||||
p.error(
|
||||
"--hard-restart-projects is meaningless with --snapshot: the flag exists "
|
||||
"for stacks whose database cannot be backed up hot, which a snapshot solves"
|
||||
)
|
||||
return args
|
||||
|
||||
@@ -17,12 +17,11 @@ import os
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
|
||||
from .shell import execute_shell_command
|
||||
from .shell import BackupException, execute_shell_command
|
||||
|
||||
KINDS = ("btrfs", "zfs")
|
||||
|
||||
|
||||
|
||||
class SnapshotError(RuntimeError):
|
||||
"""A snapshot could not be created, resolved or removed."""
|
||||
|
||||
@@ -32,13 +31,20 @@ def _resolver(subject: str, root: str) -> Callable[[str], str]:
|
||||
relative = os.path.relpath(os.path.abspath(path), os.path.abspath(subject))
|
||||
if relative.startswith(".."):
|
||||
raise SnapshotError(f"{path} lies outside the snapshot subject {subject}")
|
||||
return os.path.join(root, relative) if relative != "." else root
|
||||
resolved = root if relative == "." else os.path.join(root, relative)
|
||||
|
||||
# abspath drops a trailing separator, and rsync reads "dir/" as its
|
||||
# contents where "dir" means the directory itself.
|
||||
return resolved + os.sep if path.endswith(os.sep) else resolved
|
||||
|
||||
return resolve
|
||||
|
||||
|
||||
def _btrfs(subject: str, name: str, run: Callable[[str], list[str]]) -> tuple[str, str]:
|
||||
target = os.path.join(os.path.dirname(os.path.abspath(subject)), f".{name}")
|
||||
# The snapshot goes inside the subject, never beside it: the kernel rejects
|
||||
# a snapshot whose destination is on another filesystem, which is exactly
|
||||
# what the parent directory is when the subject is a mountpoint of its own.
|
||||
target = os.path.join(os.path.abspath(subject), f".{name}")
|
||||
run(f"btrfs subvolume snapshot -r {subject} {target}")
|
||||
return target, f"btrfs subvolume delete {target}"
|
||||
|
||||
@@ -74,6 +80,8 @@ def volume_snapshot(
|
||||
|
||||
Raises:
|
||||
SnapshotError: the kind is unknown, or the snapshot cannot be created.
|
||||
Removal failure is reported, not raised: a leftover snapshot is a
|
||||
cleanup problem and must not discard a generation that is complete.
|
||||
"""
|
||||
create = _CREATE.get(kind)
|
||||
if create is None:
|
||||
@@ -83,4 +91,8 @@ def volume_snapshot(
|
||||
try:
|
||||
yield _resolver(subject, root)
|
||||
finally:
|
||||
try:
|
||||
run(remove)
|
||||
except BackupException as error:
|
||||
# Raising here would also mask whatever the body raised.
|
||||
print(f"WARNING: {root} could not be removed: {error}", flush=True)
|
||||
|
||||
@@ -49,7 +49,10 @@ def backup_volume(
|
||||
link_dest = f"--link-dest='{last}'" if last else ""
|
||||
verify = "--checksum " if authoritative else ""
|
||||
|
||||
cmd = f"rsync -abP --delete --delete-excluded {verify}{link_dest} {source} {dest}"
|
||||
cmd = (
|
||||
f"rsync -abP --no-D --delete --delete-excluded "
|
||||
f"{verify}{link_dest} {source} {dest}"
|
||||
)
|
||||
|
||||
try:
|
||||
execute_shell_command(cmd)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Fixtures and paths the e2e suite builds its scenarios from."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Process, docker and readiness helpers for the e2e suite."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
|
||||
@@ -25,7 +25,9 @@ GENERATION = f"{VERSIONS}/20260731"
|
||||
def shell(command: str) -> list[str]:
|
||||
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
||||
if proc.returncode != 0:
|
||||
raise SnapshotError(f"{command} exited {proc.returncode}: {proc.stderr.strip()}")
|
||||
raise SnapshotError(
|
||||
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||
)
|
||||
return proc.stdout.splitlines()
|
||||
|
||||
|
||||
@@ -34,8 +36,8 @@ with volume_snapshot("btrfs", SUBJECT, "dbtest", run=shell) as resolve:
|
||||
VERSIONS,
|
||||
VOLUME,
|
||||
f"{GENERATION}/{VOLUME}",
|
||||
authoritative=False,
|
||||
source=resolve(DATADIR) + "/",
|
||||
authoritative=True,
|
||||
source=resolve(f"{DATADIR}/"),
|
||||
)
|
||||
|
||||
print("SNAPSHOT COPY DONE", flush=True)
|
||||
|
||||
@@ -22,7 +22,9 @@ EXPECT = sys.argv[3]
|
||||
def shell(command: str) -> list[str]:
|
||||
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
||||
if proc.returncode != 0:
|
||||
raise SnapshotError(f"{command} exited {proc.returncode}: {proc.stderr.strip()}")
|
||||
raise SnapshotError(
|
||||
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||
)
|
||||
return proc.stdout.splitlines()
|
||||
|
||||
|
||||
@@ -55,5 +57,8 @@ with volume_snapshot(KIND, SUBJECT, "e2e", run=shell) as resolve:
|
||||
|
||||
root = Path(resolve(SUBJECT))
|
||||
|
||||
check("the snapshot is removed afterwards", not root.exists() or not (root / "volumes").exists())
|
||||
check(
|
||||
"the snapshot is removed afterwards",
|
||||
not root.exists() or not (root / "volumes").exists(),
|
||||
)
|
||||
print("ALL OK", flush=True)
|
||||
|
||||
@@ -29,7 +29,7 @@ LOOP_FS = {
|
||||
"ext4": "mkdir -p /subject/docker",
|
||||
}
|
||||
# A container carries no /lib/modules, so modprobe fails even on a loaded module.
|
||||
ZFS_READY = '{ [ -c /dev/zfs ] || modprobe zfs 2>/dev/null; }; [ -c /dev/zfs ]'
|
||||
ZFS_READY = "{ [ -c /dev/zfs ] || modprobe zfs 2>/dev/null; }; [ -c /dev/zfs ]"
|
||||
|
||||
|
||||
def mount_script(fstype: str) -> str:
|
||||
@@ -51,7 +51,13 @@ def zfs_usable() -> bool:
|
||||
"""Whether this host's kernel can serve zfs to a privileged container."""
|
||||
proc = run(
|
||||
[
|
||||
"docker", "run", "--rm", "--privileged", IMAGE, "sh", "-lc",
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--privileged",
|
||||
IMAGE,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"apk add -q zfs >/dev/null 2>&1 && {ZFS_READY}",
|
||||
],
|
||||
capture=True,
|
||||
@@ -87,11 +93,20 @@ def drive(fstype: str, kind: str, expect: str) -> str:
|
||||
try:
|
||||
proc = run(
|
||||
[
|
||||
"docker", "run", "--rm", "--privileged",
|
||||
"--name", staged.name,
|
||||
"-v", f"{staged / 'src'}:/src:ro",
|
||||
"-v", f"{staged / 'driver.py'}:/driver.py:ro",
|
||||
IMAGE, "sh", "-lc", script,
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--privileged",
|
||||
"--name",
|
||||
staged.name,
|
||||
"-v",
|
||||
f"{staged / 'src'}:/src:ro",
|
||||
"-v",
|
||||
f"{staged / 'driver.py'}:/driver.py:ro",
|
||||
IMAGE,
|
||||
"sh",
|
||||
"-lc",
|
||||
script,
|
||||
],
|
||||
capture=True,
|
||||
check=False,
|
||||
@@ -99,7 +114,9 @@ def drive(fstype: str, kind: str, expect: str) -> str:
|
||||
finally:
|
||||
shutil.rmtree(staged, ignore_errors=True)
|
||||
if proc.returncode != 0:
|
||||
raise AssertionError(f"{fstype}/{kind} driver failed:\n{proc.stdout}\n{proc.stderr}")
|
||||
raise AssertionError(
|
||||
f"{fstype}/{kind} driver failed:\n{proc.stdout}\n{proc.stderr}"
|
||||
)
|
||||
return proc.stdout
|
||||
|
||||
|
||||
@@ -125,7 +142,9 @@ class TestE2ESnapshot(unittest.TestCase):
|
||||
"E2E_REQUIRE_FILESYSTEMS demands zfs, but this kernel provides no "
|
||||
"zfs module; load it before running the suite"
|
||||
)
|
||||
self.skipTest("this kernel provides no zfs module, so no pool can be created")
|
||||
self.skipTest(
|
||||
"this kernel provides no zfs module, so no pool can be created"
|
||||
)
|
||||
self.assert_freezes("zfs")
|
||||
|
||||
def test_ext4_has_no_snapshot_and_says_so(self) -> None:
|
||||
|
||||
@@ -68,11 +68,20 @@ class TestE2ESnapshotDatabase(unittest.TestCase):
|
||||
try:
|
||||
proc = run(
|
||||
[
|
||||
"docker", "run", "--rm", "--privileged",
|
||||
"--name", staged.name,
|
||||
"-v", f"{staged / 'src'}:/src:ro",
|
||||
"-v", f"{staged / 'driver.py'}:/driver.py:ro",
|
||||
IMAGE, "sh", "-lc", SCRIPT,
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--privileged",
|
||||
"--name",
|
||||
staged.name,
|
||||
"-v",
|
||||
f"{staged / 'src'}:/src:ro",
|
||||
"-v",
|
||||
f"{staged / 'driver.py'}:/driver.py:ro",
|
||||
IMAGE,
|
||||
"sh",
|
||||
"-lc",
|
||||
SCRIPT,
|
||||
],
|
||||
capture=True,
|
||||
check=False,
|
||||
|
||||
125
tests/e2e/test_e2e_volumes_no_backup_required_early_skip.py
Normal file
125
tests/e2e/test_e2e_volumes_no_backup_required_early_skip.py
Normal file
@@ -0,0 +1,125 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
class TestE2EVolumesNoBackupRequiredEarlySkip(unittest.TestCase):
|
||||
"""Both volumes hang off the same container, so an image-level exclusion
|
||||
could only drop both. Only the named one may disappear."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
|
||||
cls.prefix = unique("baudolo-e2e-early-skip-no-backup-volume")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.container = f"{cls.prefix}-app"
|
||||
cls.excluded_volume = f"{cls.prefix}-derived-vol"
|
||||
cls.kept_volume = f"{cls.prefix}-state-vol"
|
||||
|
||||
cls.containers = [cls.container]
|
||||
cls.volumes = [cls.excluded_volume, cls.kept_volume]
|
||||
|
||||
run(["docker", "volume", "create", cls.excluded_volume])
|
||||
run(["docker", "volume", "create", cls.kept_volume])
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"-v",
|
||||
f"{cls.excluded_volume}:/derived",
|
||||
"-v",
|
||||
f"{cls.kept_volume}:/state",
|
||||
"alpine:3.20",
|
||||
"sh",
|
||||
"-lc",
|
||||
"echo derived > /derived/derived.txt && echo state > /state/state.txt",
|
||||
]
|
||||
)
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.container,
|
||||
"-v",
|
||||
f"{cls.excluded_volume}:/derived",
|
||||
"-v",
|
||||
f"{cls.kept_volume}:/state",
|
||||
"alpine:3.20",
|
||||
"sleep",
|
||||
"600",
|
||||
]
|
||||
)
|
||||
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(cls.databases_csv, [])
|
||||
|
||||
cmd = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
cls.compose_dir,
|
||||
"--repo-name",
|
||||
cls.repo_name,
|
||||
"--databases-csv",
|
||||
cls.databases_csv,
|
||||
"--backups-dir",
|
||||
cls.backups_dir,
|
||||
"--images-no-stop-required",
|
||||
"alpine:3.20",
|
||||
"--volumes-no-backup-required",
|
||||
cls.excluded_volume,
|
||||
]
|
||||
cp = run(cmd, capture=True, check=True)
|
||||
cls.stdout = cp.stdout or ""
|
||||
cls.stderr = cp.stderr or ""
|
||||
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
|
||||
def test_excluded_volume_has_no_backup_directory_at_all(self) -> None:
|
||||
p = backup_path(
|
||||
self.backups_dir,
|
||||
self.repo_name,
|
||||
self.version,
|
||||
self.excluded_volume,
|
||||
)
|
||||
self.assertFalse(
|
||||
p.exists(),
|
||||
f"Expected NO backup directory for the excluded volume, but found: {p}",
|
||||
)
|
||||
|
||||
def test_sibling_volume_of_the_same_container_is_still_backed_up(self) -> None:
|
||||
p = (
|
||||
backup_path(
|
||||
self.backups_dir,
|
||||
self.repo_name,
|
||||
self.version,
|
||||
self.kept_volume,
|
||||
)
|
||||
/ "files"
|
||||
/ "state.txt"
|
||||
)
|
||||
self.assertTrue(p.is_file(), f"Expected backed up file at: {p}")
|
||||
80
tests/unit/backup/test_app_snapshot.py
Normal file
80
tests/unit/backup/test_app_snapshot.py
Normal file
@@ -0,0 +1,80 @@
|
||||
"""Contract of app.main's snapshot branch - the caller that runs in production."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from baudolo.backup import app
|
||||
from baudolo.backup.snapshot import volume_snapshot
|
||||
|
||||
|
||||
def stubbed_snapshot(kind: str, subject: str, tag: str):
|
||||
return volume_snapshot(kind, subject, tag, run=lambda command: [])
|
||||
|
||||
|
||||
ARGV = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
"/compose",
|
||||
"--backups-dir",
|
||||
"/backups",
|
||||
"--snapshot",
|
||||
"btrfs",
|
||||
"--snapshot-subject",
|
||||
"/var/lib/docker",
|
||||
]
|
||||
|
||||
|
||||
def drive(*, present: bool) -> list[dict]:
|
||||
calls: list[dict] = []
|
||||
|
||||
def record(versions_dir, volume_name, volume_dir, *, authoritative, source):
|
||||
calls.append(
|
||||
{"volume": volume_name, "authoritative": authoritative, "source": source}
|
||||
)
|
||||
|
||||
with (
|
||||
mock.patch("sys.argv", ARGV),
|
||||
mock.patch.object(app, "get_machine_id", return_value="machine"),
|
||||
mock.patch.object(app, "create_version_directory", return_value="/gen"),
|
||||
mock.patch.object(app, "create_volume_directory", return_value="/gen/vol"),
|
||||
mock.patch.object(app, "load_databases_df", return_value=None),
|
||||
mock.patch.object(app, "docker_volume_names", return_value=["vol"]),
|
||||
mock.patch.object(app, "containers_using_volume", return_value=[]),
|
||||
mock.patch.object(app, "volume_is_fully_ignored", return_value=False),
|
||||
mock.patch.object(app, "backup_dumps_for_volume", return_value=(False, False)),
|
||||
mock.patch.object(
|
||||
app, "get_storage_path", return_value="/var/lib/docker/volumes/vol/_data/"
|
||||
),
|
||||
mock.patch.object(app, "stamp_directory"),
|
||||
mock.patch.object(app, "handle_docker_compose_services"),
|
||||
mock.patch.object(app.os.path, "isdir", return_value=present),
|
||||
mock.patch.object(app, "backup_volume", side_effect=record),
|
||||
mock.patch.object(app, "volume_snapshot", stubbed_snapshot),
|
||||
):
|
||||
app.main()
|
||||
return calls
|
||||
|
||||
|
||||
class TestSnapshotBranch(unittest.TestCase):
|
||||
def test_it_passes_a_path_ending_in_a_separator(self) -> None:
|
||||
source = drive(present=True)[0]["source"]
|
||||
self.assertTrue(source.endswith("/volumes/vol/_data/"), source)
|
||||
self.assertNotIn("/var/lib/docker/volumes", source)
|
||||
|
||||
def test_it_reads_from_the_snapshot_and_not_from_the_live_tree(self) -> None:
|
||||
source = drive(present=True)[0]["source"]
|
||||
self.assertTrue(source.startswith("/var/lib/docker/.baudolo-"), source)
|
||||
|
||||
def test_it_compares_by_content_against_the_previous_generation(self) -> None:
|
||||
self.assertTrue(drive(present=True)[0]["authoritative"])
|
||||
|
||||
def test_a_volume_missing_from_the_snapshot_is_copied_live(self) -> None:
|
||||
call = drive(present=False)[0]
|
||||
self.assertEqual(call["source"], "/var/lib/docker/volumes/vol/_data/")
|
||||
self.assertFalse(call["authoritative"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
82
tests/unit/backup/test_app_volumes_no_backup_required.py
Normal file
82
tests/unit/backup/test_app_volumes_no_backup_required.py
Normal file
@@ -0,0 +1,82 @@
|
||||
"""Contract of --volumes-no-backup-required: exclusion is per volume name,
|
||||
independent of which containers use it."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from baudolo.backup import app
|
||||
|
||||
ARGV = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
"/compose",
|
||||
"--backups-dir",
|
||||
"/backups",
|
||||
"--volumes-no-backup-required",
|
||||
"derived",
|
||||
]
|
||||
|
||||
|
||||
def drive() -> tuple[list[str], list[str], list[str]]:
|
||||
backed_up: list[str] = []
|
||||
created: list[str] = []
|
||||
inspected: list[str] = []
|
||||
|
||||
def record_backup(versions_dir, volume_name, volume_dir, *, authoritative, source):
|
||||
backed_up.append(volume_name)
|
||||
|
||||
with (
|
||||
mock.patch("sys.argv", ARGV),
|
||||
mock.patch.object(app, "get_machine_id", return_value="machine"),
|
||||
mock.patch.object(app, "create_version_directory", return_value="/gen"),
|
||||
mock.patch.object(
|
||||
app,
|
||||
"create_volume_directory",
|
||||
side_effect=lambda _version_dir, name: created.append(name) or "/gen/vol",
|
||||
),
|
||||
mock.patch.object(app, "load_databases_df", return_value=None),
|
||||
mock.patch.object(
|
||||
app, "docker_volume_names", return_value=["derived", "state"]
|
||||
),
|
||||
mock.patch.object(
|
||||
app,
|
||||
"containers_using_volume",
|
||||
side_effect=lambda name: inspected.append(name) or ["app"],
|
||||
),
|
||||
mock.patch.object(app, "volume_is_fully_ignored", return_value=False),
|
||||
mock.patch.object(app, "backup_dumps_for_volume", return_value=(False, False)),
|
||||
mock.patch.object(app, "get_storage_path", return_value="/data/"),
|
||||
mock.patch.object(app, "stamp_directory"),
|
||||
mock.patch.object(app, "handle_docker_compose_services"),
|
||||
mock.patch.object(app.os.path, "isdir", return_value=True),
|
||||
mock.patch.object(app, "backup_volume", side_effect=record_backup),
|
||||
mock.patch.object(app, "filter_stoppable", return_value=[]),
|
||||
mock.patch.object(app, "requires_stop", return_value=False),
|
||||
mock.patch.object(app, "change_containers_status"),
|
||||
):
|
||||
app.main()
|
||||
return backed_up, created, inspected
|
||||
|
||||
|
||||
class TestVolumesNoBackupRequired(unittest.TestCase):
|
||||
def test_the_named_volume_is_never_backed_up(self) -> None:
|
||||
backed_up, _created, _inspected = drive()
|
||||
self.assertNotIn("derived", backed_up)
|
||||
|
||||
def test_a_sibling_volume_of_the_same_container_survives(self) -> None:
|
||||
backed_up, _created, _inspected = drive()
|
||||
self.assertEqual(backed_up, ["state"])
|
||||
|
||||
def test_no_generation_directory_is_created_for_it(self) -> None:
|
||||
_backed_up, created, _inspected = drive()
|
||||
self.assertEqual(created, ["state"])
|
||||
|
||||
def test_the_skip_precedes_the_container_inspection(self) -> None:
|
||||
_backed_up, _created, inspected = drive()
|
||||
self.assertEqual(inspected, ["state"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -39,7 +39,9 @@ class TestSnapshotFlags(unittest.TestCase):
|
||||
parse("--snapshot", "ext4", "--snapshot-subject", "/var/lib/docker")
|
||||
|
||||
def test_zfs_is_accepted(self) -> None:
|
||||
self.assertEqual(parse("--snapshot", "zfs", "--snapshot-subject", "/d").snapshot, "zfs")
|
||||
self.assertEqual(
|
||||
parse("--snapshot", "zfs", "--snapshot-subject", "/d").snapshot, "zfs"
|
||||
)
|
||||
|
||||
def test_shutdown_is_rejected_because_nothing_is_stopped(self) -> None:
|
||||
with self.assertRaises(SystemExit):
|
||||
@@ -48,6 +50,22 @@ class TestSnapshotFlags(unittest.TestCase):
|
||||
def test_shutdown_stays_available_without_a_snapshot(self) -> None:
|
||||
self.assertTrue(parse("--shutdown").shutdown)
|
||||
|
||||
def test_hard_restart_is_rejected_because_nothing_is_stopped(self) -> None:
|
||||
with self.assertRaises(SystemExit):
|
||||
parse(
|
||||
"--snapshot",
|
||||
"btrfs",
|
||||
"--snapshot-subject",
|
||||
"/d",
|
||||
"--hard-restart-projects",
|
||||
"mailu",
|
||||
)
|
||||
|
||||
def test_hard_restart_stays_available_without_a_snapshot(self) -> None:
|
||||
self.assertEqual(
|
||||
parse("--hard-restart-projects", "mailu").hard_restart_projects, ["mailu"]
|
||||
)
|
||||
|
||||
|
||||
class TestRequiredFlags(unittest.TestCase):
|
||||
def test_backups_dir_is_required(self) -> None:
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from baudolo.backup.shell import BackupException
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
||||
|
||||
|
||||
@@ -21,27 +22,41 @@ class Runner:
|
||||
|
||||
|
||||
class TestBtrfs(unittest.TestCase):
|
||||
def test_it_creates_a_read_only_snapshot_beside_the_subject(self) -> None:
|
||||
def test_it_creates_a_read_only_snapshot_inside_the_subject(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
||||
pass
|
||||
self.assertEqual(
|
||||
run.calls[0],
|
||||
"btrfs subvolume snapshot -r /var/lib/docker /var/lib/.baudolo-20260731",
|
||||
"btrfs subvolume snapshot -r /var/lib/docker /var/lib/docker/.baudolo-20260731",
|
||||
)
|
||||
|
||||
def test_it_removes_the_snapshot_afterwards(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
||||
pass
|
||||
self.assertEqual(run.calls[-1], "btrfs subvolume delete /var/lib/.baudolo-20260731")
|
||||
self.assertEqual(
|
||||
run.calls[-1], "btrfs subvolume delete /var/lib/docker/.baudolo-20260731"
|
||||
)
|
||||
|
||||
def test_it_maps_a_volume_path_into_the_snapshot(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
||||
with volume_snapshot(
|
||||
"btrfs", "/var/lib/docker", "20260731", run=run
|
||||
) as resolve:
|
||||
self.assertEqual(
|
||||
resolve("/var/lib/docker/volumes/postgres_data/_data"),
|
||||
"/var/lib/.baudolo-20260731/volumes/postgres_data/_data",
|
||||
"/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data",
|
||||
)
|
||||
|
||||
def test_it_keeps_the_trailing_slash_rsync_reads_as_contents(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot(
|
||||
"btrfs", "/var/lib/docker", "20260731", run=run
|
||||
) as resolve:
|
||||
self.assertEqual(
|
||||
resolve("/var/lib/docker/volumes/postgres_data/_data/"),
|
||||
"/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data/",
|
||||
)
|
||||
|
||||
def test_it_removes_the_snapshot_even_when_the_body_raises(self) -> None:
|
||||
@@ -93,14 +108,38 @@ class TestRejections(unittest.TestCase):
|
||||
|
||||
def test_a_path_outside_the_subject_is_rejected(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
||||
with volume_snapshot(
|
||||
"btrfs", "/var/lib/docker", "20260731", run=run
|
||||
) as resolve:
|
||||
with self.assertRaises(SnapshotError):
|
||||
resolve("/etc/passwd")
|
||||
|
||||
def test_the_subject_itself_resolves_to_the_snapshot_root(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
||||
self.assertEqual(resolve("/var/lib/docker"), "/var/lib/.baudolo-20260731")
|
||||
with volume_snapshot(
|
||||
"btrfs", "/var/lib/docker", "20260731", run=run
|
||||
) as resolve:
|
||||
self.assertEqual(
|
||||
resolve("/var/lib/docker"), "/var/lib/docker/.baudolo-20260731"
|
||||
)
|
||||
|
||||
|
||||
class Busy(Runner):
|
||||
def __call__(self, command: str) -> list[str]:
|
||||
if command.startswith("btrfs subvolume delete"):
|
||||
raise BackupException("target is busy")
|
||||
return super().__call__(command)
|
||||
|
||||
|
||||
class TestRemovalFailure(unittest.TestCase):
|
||||
def test_a_failed_removal_does_not_fail_a_completed_run(self) -> None:
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=Busy()):
|
||||
pass
|
||||
|
||||
def test_a_failed_removal_does_not_mask_the_body(self) -> None:
|
||||
with self.assertRaises(ZeroDivisionError):
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=Busy()):
|
||||
raise ZeroDivisionError
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -43,6 +43,9 @@ class TestBackupVolume(unittest.TestCase):
|
||||
def test_it_always_deletes_what_the_source_no_longer_has(self) -> None:
|
||||
self.assertIn("--delete", self.copy())
|
||||
|
||||
def test_it_carries_no_kernel_objects_into_a_generation(self) -> None:
|
||||
self.assertIn("--no-D", self.copy())
|
||||
|
||||
def test_it_creates_the_destination(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
dest = Path(tmp) / "gen" / "demo"
|
||||
|
||||
Reference in New Issue
Block a user