mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2026-08-02 13:02:41 +00:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 934e693810 | |||
| 2e0e67ca87 | |||
| 95c34d4db0 | |||
| c2f1cb8e8c | |||
| eeaa838d02 | |||
| 4e2b3641f9 | |||
| b10d50efbe |
55
CHANGELOG.md
55
CHANGELOG.md
@@ -1,5 +1,60 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## [3.3.0] - 2026-08-02
|
||||||
|
|
||||||
|
- Backup: *--volumes-no-backup-required* excludes a volume by name.
|
||||||
|
*--images-no-backup-required* resolves through *volume_is_fully_ignored*,
|
||||||
|
which skips a volume only when every container using it is ignored — a
|
||||||
|
container holding a derived tree beside state that must be kept cannot
|
||||||
|
express the exclusion at all. A docker-in-docker data root is exactly that
|
||||||
|
shape, and excluding by image would drop all three of its volumes.
|
||||||
|
- Backup: the name check runs before *containers_using_volume*, so an excluded
|
||||||
|
volume costs no docker inspection and the decision does not depend on which
|
||||||
|
containers exist when the run starts.
|
||||||
|
- Tests: two volumes off one container, asserting the sibling survives — the
|
||||||
|
property the image lever cannot provide — as unit and end-to-end.
|
||||||
|
|
||||||
|
## [3.2.2] - 2026-07-31
|
||||||
|
|
||||||
|
- Backup: the btrfs snapshot is carved inside its subject, as
|
||||||
|
*<data root>/.baudolo-<tag>*, not beside it. The kernel refuses a snapshot
|
||||||
|
whose destination is on another filesystem, which is exactly what the parent
|
||||||
|
directory is when the data root is a mountpoint of its own — a dedicated disk
|
||||||
|
mounted onto */var/lib/docker* failed every run with EXDEV. Placing it inside
|
||||||
|
makes source and destination the same filesystem by construction, and aligns
|
||||||
|
btrfs with the zfs path, which already resolves its snapshot inside the
|
||||||
|
subject at *<subject>/.zfs/snapshot/<tag>*. A leftover from an interrupted run
|
||||||
|
appears in the next snapshot as an empty directory rather than recursing,
|
||||||
|
since btrfs does not include nested subvolumes.
|
||||||
|
|
||||||
|
## [3.2.1] - 2026-07-31
|
||||||
|
|
||||||
|
- Backup: the snapshot resolver keeps the trailing separator *get_storage_path*
|
||||||
|
puts on a volume path — *os.path.abspath* stripped it. rsync reads *dir* as
|
||||||
|
"copy the directory" where *dir/* means "copy its contents", so every snapshot
|
||||||
|
generation landed at *<volume>/files/_data/...* while the live path lands at
|
||||||
|
*<volume>/files/...*. Restores read the live layout, and *--link-dest* had
|
||||||
|
nothing to match against the previous generation.
|
||||||
|
- Backup: snapshot teardown no longer fails a completed run. A busy
|
||||||
|
*btrfs subvolume delete* raised out of the *finally*, skipping the generation
|
||||||
|
stamp and the compose handling on a run whose data was already copied, and
|
||||||
|
masking whatever the body had raised. The leftover is reported instead.
|
||||||
|
- Backup: a volume created after the snapshot was taken is copied live with a
|
||||||
|
warning instead of aborting the run. Nothing is stopped in snapshot mode, so
|
||||||
|
the host keeps creating volumes for the whole copy.
|
||||||
|
- Backup: the snapshot pass compares by content (*--checksum*) again. 3.2.0
|
||||||
|
dropped it because a snapshot source cannot move, which is true, but the
|
||||||
|
comparison that matters is against *--link-dest*: a file that changed while
|
||||||
|
keeping its size and whole-second mtime was hard-linked stale out of the
|
||||||
|
previous generation, and the single pass had no authoritative pass to repair
|
||||||
|
it. Still one pass where the live path takes two.
|
||||||
|
- Backup: *--hard-restart-projects* is refused alongside *--snapshot*, like
|
||||||
|
*--shutdown* already is. It exists for stacks whose database cannot be backed
|
||||||
|
up hot, which is what a snapshot removes.
|
||||||
|
- Tests: the trailing separator, both teardown behaviours, the new refusal, and
|
||||||
|
*app.main* driving the snapshot branch — the caller that runs in production,
|
||||||
|
which no test had exercised, which is why the layout defect shipped.
|
||||||
|
|
||||||
## [3.2.0] - 2026-07-31
|
## [3.2.0] - 2026-07-31
|
||||||
|
|
||||||
- Backup: *--snapshot {btrfs,zfs}* with *--snapshot-subject* captures every
|
- Backup: *--snapshot {btrfs,zfs}* with *--snapshot-subject* captures every
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "backup-docker-to-local"
|
name = "backup-docker-to-local"
|
||||||
version = "3.2.0"
|
version = "3.3.0"
|
||||||
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.9"
|
requires-python = ">=3.9"
|
||||||
|
|||||||
@@ -54,6 +54,14 @@ def main() -> int:
|
|||||||
|
|
||||||
for volume_name in docker_volume_names():
|
for volume_name in docker_volume_names():
|
||||||
print(f"Start backup routine for volume: {volume_name}", flush=True)
|
print(f"Start backup routine for volume: {volume_name}", flush=True)
|
||||||
|
|
||||||
|
if volume_name in args.volumes_no_backup_required:
|
||||||
|
print(
|
||||||
|
f"Skipping volume '{volume_name}' entirely (declared no-backup).",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
containers = containers_using_volume(volume_name)
|
containers = containers_using_volume(volume_name)
|
||||||
|
|
||||||
if volume_is_fully_ignored(containers, args.images_no_backup_required):
|
if volume_is_fully_ignored(containers, args.images_no_backup_required):
|
||||||
@@ -95,7 +103,16 @@ def main() -> int:
|
|||||||
)
|
)
|
||||||
|
|
||||||
if resolve_source is not None:
|
if resolve_source is not None:
|
||||||
copy(authoritative=False, source=resolve_source(live_source))
|
snapshot_source = resolve_source(live_source)
|
||||||
|
if os.path.isdir(snapshot_source):
|
||||||
|
copy(authoritative=True, source=snapshot_source)
|
||||||
|
else:
|
||||||
|
print(
|
||||||
|
f"WARNING: volume '{volume_name}' is not in the snapshot "
|
||||||
|
"(created after it was taken); copying it live instead.",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
copy(authoritative=False)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if args.everything:
|
if args.everything:
|
||||||
|
|||||||
@@ -68,6 +68,13 @@ def parse_args() -> argparse.Namespace:
|
|||||||
help="Exact image references (repo:tag, incl. any registry prefix) for which no backup should be performed",
|
help="Exact image references (repo:tag, incl. any registry prefix) for which no backup should be performed",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
p.add_argument(
|
||||||
|
"--volumes-no-backup-required",
|
||||||
|
nargs="+",
|
||||||
|
default=[],
|
||||||
|
help="Exact volume names that are never backed up, whatever containers use them. For derived trees a restore cannot reproduce, above all a nested docker data root",
|
||||||
|
)
|
||||||
|
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--everything",
|
"--everything",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
@@ -93,5 +100,12 @@ def parse_args() -> argparse.Namespace:
|
|||||||
if bool(args.snapshot) != bool(args.snapshot_subject):
|
if bool(args.snapshot) != bool(args.snapshot_subject):
|
||||||
p.error("--snapshot and --snapshot-subject must be given together")
|
p.error("--snapshot and --snapshot-subject must be given together")
|
||||||
if args.snapshot and args.shutdown:
|
if args.snapshot and args.shutdown:
|
||||||
p.error("--shutdown is meaningless with --snapshot: containers are never stopped")
|
p.error(
|
||||||
|
"--shutdown is meaningless with --snapshot: containers are never stopped"
|
||||||
|
)
|
||||||
|
if args.snapshot and args.hard_restart_projects:
|
||||||
|
p.error(
|
||||||
|
"--hard-restart-projects is meaningless with --snapshot: the flag exists "
|
||||||
|
"for stacks whose database cannot be backed up hot, which a snapshot solves"
|
||||||
|
)
|
||||||
return args
|
return args
|
||||||
|
|||||||
@@ -17,12 +17,11 @@ import os
|
|||||||
from collections.abc import Callable, Iterator
|
from collections.abc import Callable, Iterator
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
|
|
||||||
from .shell import execute_shell_command
|
from .shell import BackupException, execute_shell_command
|
||||||
|
|
||||||
KINDS = ("btrfs", "zfs")
|
KINDS = ("btrfs", "zfs")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class SnapshotError(RuntimeError):
|
class SnapshotError(RuntimeError):
|
||||||
"""A snapshot could not be created, resolved or removed."""
|
"""A snapshot could not be created, resolved or removed."""
|
||||||
|
|
||||||
@@ -32,13 +31,20 @@ def _resolver(subject: str, root: str) -> Callable[[str], str]:
|
|||||||
relative = os.path.relpath(os.path.abspath(path), os.path.abspath(subject))
|
relative = os.path.relpath(os.path.abspath(path), os.path.abspath(subject))
|
||||||
if relative.startswith(".."):
|
if relative.startswith(".."):
|
||||||
raise SnapshotError(f"{path} lies outside the snapshot subject {subject}")
|
raise SnapshotError(f"{path} lies outside the snapshot subject {subject}")
|
||||||
return os.path.join(root, relative) if relative != "." else root
|
resolved = root if relative == "." else os.path.join(root, relative)
|
||||||
|
|
||||||
|
# abspath drops a trailing separator, and rsync reads "dir/" as its
|
||||||
|
# contents where "dir" means the directory itself.
|
||||||
|
return resolved + os.sep if path.endswith(os.sep) else resolved
|
||||||
|
|
||||||
return resolve
|
return resolve
|
||||||
|
|
||||||
|
|
||||||
def _btrfs(subject: str, name: str, run: Callable[[str], list[str]]) -> tuple[str, str]:
|
def _btrfs(subject: str, name: str, run: Callable[[str], list[str]]) -> tuple[str, str]:
|
||||||
target = os.path.join(os.path.dirname(os.path.abspath(subject)), f".{name}")
|
# The snapshot goes inside the subject, never beside it: the kernel rejects
|
||||||
|
# a snapshot whose destination is on another filesystem, which is exactly
|
||||||
|
# what the parent directory is when the subject is a mountpoint of its own.
|
||||||
|
target = os.path.join(os.path.abspath(subject), f".{name}")
|
||||||
run(f"btrfs subvolume snapshot -r {subject} {target}")
|
run(f"btrfs subvolume snapshot -r {subject} {target}")
|
||||||
return target, f"btrfs subvolume delete {target}"
|
return target, f"btrfs subvolume delete {target}"
|
||||||
|
|
||||||
@@ -74,6 +80,8 @@ def volume_snapshot(
|
|||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
SnapshotError: the kind is unknown, or the snapshot cannot be created.
|
SnapshotError: the kind is unknown, or the snapshot cannot be created.
|
||||||
|
Removal failure is reported, not raised: a leftover snapshot is a
|
||||||
|
cleanup problem and must not discard a generation that is complete.
|
||||||
"""
|
"""
|
||||||
create = _CREATE.get(kind)
|
create = _CREATE.get(kind)
|
||||||
if create is None:
|
if create is None:
|
||||||
@@ -83,4 +91,8 @@ def volume_snapshot(
|
|||||||
try:
|
try:
|
||||||
yield _resolver(subject, root)
|
yield _resolver(subject, root)
|
||||||
finally:
|
finally:
|
||||||
run(remove)
|
try:
|
||||||
|
run(remove)
|
||||||
|
except BackupException as error:
|
||||||
|
# Raising here would also mask whatever the body raised.
|
||||||
|
print(f"WARNING: {root} could not be removed: {error}", flush=True)
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
"""Fixtures and paths the e2e suite builds its scenarios from."""
|
"""Fixtures and paths the e2e suite builds its scenarios from."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import shutil
|
import shutil
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
"""Process, docker and readiness helpers for the e2e suite."""
|
"""Process, docker and readiness helpers for the e2e suite."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|||||||
@@ -25,7 +25,9 @@ GENERATION = f"{VERSIONS}/20260731"
|
|||||||
def shell(command: str) -> list[str]:
|
def shell(command: str) -> list[str]:
|
||||||
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
raise SnapshotError(f"{command} exited {proc.returncode}: {proc.stderr.strip()}")
|
raise SnapshotError(
|
||||||
|
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||||
|
)
|
||||||
return proc.stdout.splitlines()
|
return proc.stdout.splitlines()
|
||||||
|
|
||||||
|
|
||||||
@@ -34,8 +36,8 @@ with volume_snapshot("btrfs", SUBJECT, "dbtest", run=shell) as resolve:
|
|||||||
VERSIONS,
|
VERSIONS,
|
||||||
VOLUME,
|
VOLUME,
|
||||||
f"{GENERATION}/{VOLUME}",
|
f"{GENERATION}/{VOLUME}",
|
||||||
authoritative=False,
|
authoritative=True,
|
||||||
source=resolve(DATADIR) + "/",
|
source=resolve(f"{DATADIR}/"),
|
||||||
)
|
)
|
||||||
|
|
||||||
print("SNAPSHOT COPY DONE", flush=True)
|
print("SNAPSHOT COPY DONE", flush=True)
|
||||||
|
|||||||
@@ -22,7 +22,9 @@ EXPECT = sys.argv[3]
|
|||||||
def shell(command: str) -> list[str]:
|
def shell(command: str) -> list[str]:
|
||||||
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
raise SnapshotError(f"{command} exited {proc.returncode}: {proc.stderr.strip()}")
|
raise SnapshotError(
|
||||||
|
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||||
|
)
|
||||||
return proc.stdout.splitlines()
|
return proc.stdout.splitlines()
|
||||||
|
|
||||||
|
|
||||||
@@ -55,5 +57,8 @@ with volume_snapshot(KIND, SUBJECT, "e2e", run=shell) as resolve:
|
|||||||
|
|
||||||
root = Path(resolve(SUBJECT))
|
root = Path(resolve(SUBJECT))
|
||||||
|
|
||||||
check("the snapshot is removed afterwards", not root.exists() or not (root / "volumes").exists())
|
check(
|
||||||
|
"the snapshot is removed afterwards",
|
||||||
|
not root.exists() or not (root / "volumes").exists(),
|
||||||
|
)
|
||||||
print("ALL OK", flush=True)
|
print("ALL OK", flush=True)
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ LOOP_FS = {
|
|||||||
"ext4": "mkdir -p /subject/docker",
|
"ext4": "mkdir -p /subject/docker",
|
||||||
}
|
}
|
||||||
# A container carries no /lib/modules, so modprobe fails even on a loaded module.
|
# A container carries no /lib/modules, so modprobe fails even on a loaded module.
|
||||||
ZFS_READY = '{ [ -c /dev/zfs ] || modprobe zfs 2>/dev/null; }; [ -c /dev/zfs ]'
|
ZFS_READY = "{ [ -c /dev/zfs ] || modprobe zfs 2>/dev/null; }; [ -c /dev/zfs ]"
|
||||||
|
|
||||||
|
|
||||||
def mount_script(fstype: str) -> str:
|
def mount_script(fstype: str) -> str:
|
||||||
@@ -51,7 +51,13 @@ def zfs_usable() -> bool:
|
|||||||
"""Whether this host's kernel can serve zfs to a privileged container."""
|
"""Whether this host's kernel can serve zfs to a privileged container."""
|
||||||
proc = run(
|
proc = run(
|
||||||
[
|
[
|
||||||
"docker", "run", "--rm", "--privileged", IMAGE, "sh", "-lc",
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"--privileged",
|
||||||
|
IMAGE,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
f"apk add -q zfs >/dev/null 2>&1 && {ZFS_READY}",
|
f"apk add -q zfs >/dev/null 2>&1 && {ZFS_READY}",
|
||||||
],
|
],
|
||||||
capture=True,
|
capture=True,
|
||||||
@@ -87,11 +93,20 @@ def drive(fstype: str, kind: str, expect: str) -> str:
|
|||||||
try:
|
try:
|
||||||
proc = run(
|
proc = run(
|
||||||
[
|
[
|
||||||
"docker", "run", "--rm", "--privileged",
|
"docker",
|
||||||
"--name", staged.name,
|
"run",
|
||||||
"-v", f"{staged / 'src'}:/src:ro",
|
"--rm",
|
||||||
"-v", f"{staged / 'driver.py'}:/driver.py:ro",
|
"--privileged",
|
||||||
IMAGE, "sh", "-lc", script,
|
"--name",
|
||||||
|
staged.name,
|
||||||
|
"-v",
|
||||||
|
f"{staged / 'src'}:/src:ro",
|
||||||
|
"-v",
|
||||||
|
f"{staged / 'driver.py'}:/driver.py:ro",
|
||||||
|
IMAGE,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
script,
|
||||||
],
|
],
|
||||||
capture=True,
|
capture=True,
|
||||||
check=False,
|
check=False,
|
||||||
@@ -99,7 +114,9 @@ def drive(fstype: str, kind: str, expect: str) -> str:
|
|||||||
finally:
|
finally:
|
||||||
shutil.rmtree(staged, ignore_errors=True)
|
shutil.rmtree(staged, ignore_errors=True)
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
raise AssertionError(f"{fstype}/{kind} driver failed:\n{proc.stdout}\n{proc.stderr}")
|
raise AssertionError(
|
||||||
|
f"{fstype}/{kind} driver failed:\n{proc.stdout}\n{proc.stderr}"
|
||||||
|
)
|
||||||
return proc.stdout
|
return proc.stdout
|
||||||
|
|
||||||
|
|
||||||
@@ -125,7 +142,9 @@ class TestE2ESnapshot(unittest.TestCase):
|
|||||||
"E2E_REQUIRE_FILESYSTEMS demands zfs, but this kernel provides no "
|
"E2E_REQUIRE_FILESYSTEMS demands zfs, but this kernel provides no "
|
||||||
"zfs module; load it before running the suite"
|
"zfs module; load it before running the suite"
|
||||||
)
|
)
|
||||||
self.skipTest("this kernel provides no zfs module, so no pool can be created")
|
self.skipTest(
|
||||||
|
"this kernel provides no zfs module, so no pool can be created"
|
||||||
|
)
|
||||||
self.assert_freezes("zfs")
|
self.assert_freezes("zfs")
|
||||||
|
|
||||||
def test_ext4_has_no_snapshot_and_says_so(self) -> None:
|
def test_ext4_has_no_snapshot_and_says_so(self) -> None:
|
||||||
|
|||||||
@@ -68,11 +68,20 @@ class TestE2ESnapshotDatabase(unittest.TestCase):
|
|||||||
try:
|
try:
|
||||||
proc = run(
|
proc = run(
|
||||||
[
|
[
|
||||||
"docker", "run", "--rm", "--privileged",
|
"docker",
|
||||||
"--name", staged.name,
|
"run",
|
||||||
"-v", f"{staged / 'src'}:/src:ro",
|
"--rm",
|
||||||
"-v", f"{staged / 'driver.py'}:/driver.py:ro",
|
"--privileged",
|
||||||
IMAGE, "sh", "-lc", SCRIPT,
|
"--name",
|
||||||
|
staged.name,
|
||||||
|
"-v",
|
||||||
|
f"{staged / 'src'}:/src:ro",
|
||||||
|
"-v",
|
||||||
|
f"{staged / 'driver.py'}:/driver.py:ro",
|
||||||
|
IMAGE,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
SCRIPT,
|
||||||
],
|
],
|
||||||
capture=True,
|
capture=True,
|
||||||
check=False,
|
check=False,
|
||||||
|
|||||||
125
tests/e2e/test_e2e_volumes_no_backup_required_early_skip.py
Normal file
125
tests/e2e/test_e2e_volumes_no_backup_required_early_skip.py
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from .helpers import (
|
||||||
|
backup_path,
|
||||||
|
cleanup_docker,
|
||||||
|
create_minimal_compose_dir,
|
||||||
|
ensure_empty_dir,
|
||||||
|
latest_version_dir,
|
||||||
|
require_docker,
|
||||||
|
run,
|
||||||
|
unique,
|
||||||
|
write_databases_csv,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestE2EVolumesNoBackupRequiredEarlySkip(unittest.TestCase):
|
||||||
|
"""Both volumes hang off the same container, so an image-level exclusion
|
||||||
|
could only drop both. Only the named one may disappear."""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
require_docker()
|
||||||
|
|
||||||
|
cls.prefix = unique("baudolo-e2e-early-skip-no-backup-volume")
|
||||||
|
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||||
|
ensure_empty_dir(cls.backups_dir)
|
||||||
|
|
||||||
|
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||||
|
cls.repo_name = cls.prefix
|
||||||
|
|
||||||
|
cls.container = f"{cls.prefix}-app"
|
||||||
|
cls.excluded_volume = f"{cls.prefix}-derived-vol"
|
||||||
|
cls.kept_volume = f"{cls.prefix}-state-vol"
|
||||||
|
|
||||||
|
cls.containers = [cls.container]
|
||||||
|
cls.volumes = [cls.excluded_volume, cls.kept_volume]
|
||||||
|
|
||||||
|
run(["docker", "volume", "create", cls.excluded_volume])
|
||||||
|
run(["docker", "volume", "create", cls.kept_volume])
|
||||||
|
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"-v",
|
||||||
|
f"{cls.excluded_volume}:/derived",
|
||||||
|
"-v",
|
||||||
|
f"{cls.kept_volume}:/state",
|
||||||
|
"alpine:3.20",
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
"echo derived > /derived/derived.txt && echo state > /state/state.txt",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"-d",
|
||||||
|
"--name",
|
||||||
|
cls.container,
|
||||||
|
"-v",
|
||||||
|
f"{cls.excluded_volume}:/derived",
|
||||||
|
"-v",
|
||||||
|
f"{cls.kept_volume}:/state",
|
||||||
|
"alpine:3.20",
|
||||||
|
"sleep",
|
||||||
|
"600",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
|
write_databases_csv(cls.databases_csv, [])
|
||||||
|
|
||||||
|
cmd = [
|
||||||
|
"baudolo",
|
||||||
|
"--compose-dir",
|
||||||
|
cls.compose_dir,
|
||||||
|
"--repo-name",
|
||||||
|
cls.repo_name,
|
||||||
|
"--databases-csv",
|
||||||
|
cls.databases_csv,
|
||||||
|
"--backups-dir",
|
||||||
|
cls.backups_dir,
|
||||||
|
"--images-no-stop-required",
|
||||||
|
"alpine:3.20",
|
||||||
|
"--volumes-no-backup-required",
|
||||||
|
cls.excluded_volume,
|
||||||
|
]
|
||||||
|
cp = run(cmd, capture=True, check=True)
|
||||||
|
cls.stdout = cp.stdout or ""
|
||||||
|
cls.stderr = cp.stderr or ""
|
||||||
|
|
||||||
|
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls) -> None:
|
||||||
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
|
||||||
|
def test_excluded_volume_has_no_backup_directory_at_all(self) -> None:
|
||||||
|
p = backup_path(
|
||||||
|
self.backups_dir,
|
||||||
|
self.repo_name,
|
||||||
|
self.version,
|
||||||
|
self.excluded_volume,
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
p.exists(),
|
||||||
|
f"Expected NO backup directory for the excluded volume, but found: {p}",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_sibling_volume_of_the_same_container_is_still_backed_up(self) -> None:
|
||||||
|
p = (
|
||||||
|
backup_path(
|
||||||
|
self.backups_dir,
|
||||||
|
self.repo_name,
|
||||||
|
self.version,
|
||||||
|
self.kept_volume,
|
||||||
|
)
|
||||||
|
/ "files"
|
||||||
|
/ "state.txt"
|
||||||
|
)
|
||||||
|
self.assertTrue(p.is_file(), f"Expected backed up file at: {p}")
|
||||||
80
tests/unit/backup/test_app_snapshot.py
Normal file
80
tests/unit/backup/test_app_snapshot.py
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
"""Contract of app.main's snapshot branch - the caller that runs in production."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
from baudolo.backup import app
|
||||||
|
from baudolo.backup.snapshot import volume_snapshot
|
||||||
|
|
||||||
|
|
||||||
|
def stubbed_snapshot(kind: str, subject: str, tag: str):
|
||||||
|
return volume_snapshot(kind, subject, tag, run=lambda command: [])
|
||||||
|
|
||||||
|
|
||||||
|
ARGV = [
|
||||||
|
"baudolo",
|
||||||
|
"--compose-dir",
|
||||||
|
"/compose",
|
||||||
|
"--backups-dir",
|
||||||
|
"/backups",
|
||||||
|
"--snapshot",
|
||||||
|
"btrfs",
|
||||||
|
"--snapshot-subject",
|
||||||
|
"/var/lib/docker",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def drive(*, present: bool) -> list[dict]:
|
||||||
|
calls: list[dict] = []
|
||||||
|
|
||||||
|
def record(versions_dir, volume_name, volume_dir, *, authoritative, source):
|
||||||
|
calls.append(
|
||||||
|
{"volume": volume_name, "authoritative": authoritative, "source": source}
|
||||||
|
)
|
||||||
|
|
||||||
|
with (
|
||||||
|
mock.patch("sys.argv", ARGV),
|
||||||
|
mock.patch.object(app, "get_machine_id", return_value="machine"),
|
||||||
|
mock.patch.object(app, "create_version_directory", return_value="/gen"),
|
||||||
|
mock.patch.object(app, "create_volume_directory", return_value="/gen/vol"),
|
||||||
|
mock.patch.object(app, "load_databases_df", return_value=None),
|
||||||
|
mock.patch.object(app, "docker_volume_names", return_value=["vol"]),
|
||||||
|
mock.patch.object(app, "containers_using_volume", return_value=[]),
|
||||||
|
mock.patch.object(app, "volume_is_fully_ignored", return_value=False),
|
||||||
|
mock.patch.object(app, "backup_dumps_for_volume", return_value=(False, False)),
|
||||||
|
mock.patch.object(
|
||||||
|
app, "get_storage_path", return_value="/var/lib/docker/volumes/vol/_data/"
|
||||||
|
),
|
||||||
|
mock.patch.object(app, "stamp_directory"),
|
||||||
|
mock.patch.object(app, "handle_docker_compose_services"),
|
||||||
|
mock.patch.object(app.os.path, "isdir", return_value=present),
|
||||||
|
mock.patch.object(app, "backup_volume", side_effect=record),
|
||||||
|
mock.patch.object(app, "volume_snapshot", stubbed_snapshot),
|
||||||
|
):
|
||||||
|
app.main()
|
||||||
|
return calls
|
||||||
|
|
||||||
|
|
||||||
|
class TestSnapshotBranch(unittest.TestCase):
|
||||||
|
def test_it_passes_a_path_ending_in_a_separator(self) -> None:
|
||||||
|
source = drive(present=True)[0]["source"]
|
||||||
|
self.assertTrue(source.endswith("/volumes/vol/_data/"), source)
|
||||||
|
self.assertNotIn("/var/lib/docker/volumes", source)
|
||||||
|
|
||||||
|
def test_it_reads_from_the_snapshot_and_not_from_the_live_tree(self) -> None:
|
||||||
|
source = drive(present=True)[0]["source"]
|
||||||
|
self.assertTrue(source.startswith("/var/lib/docker/.baudolo-"), source)
|
||||||
|
|
||||||
|
def test_it_compares_by_content_against_the_previous_generation(self) -> None:
|
||||||
|
self.assertTrue(drive(present=True)[0]["authoritative"])
|
||||||
|
|
||||||
|
def test_a_volume_missing_from_the_snapshot_is_copied_live(self) -> None:
|
||||||
|
call = drive(present=False)[0]
|
||||||
|
self.assertEqual(call["source"], "/var/lib/docker/volumes/vol/_data/")
|
||||||
|
self.assertFalse(call["authoritative"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
82
tests/unit/backup/test_app_volumes_no_backup_required.py
Normal file
82
tests/unit/backup/test_app_volumes_no_backup_required.py
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
"""Contract of --volumes-no-backup-required: exclusion is per volume name,
|
||||||
|
independent of which containers use it."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
from baudolo.backup import app
|
||||||
|
|
||||||
|
ARGV = [
|
||||||
|
"baudolo",
|
||||||
|
"--compose-dir",
|
||||||
|
"/compose",
|
||||||
|
"--backups-dir",
|
||||||
|
"/backups",
|
||||||
|
"--volumes-no-backup-required",
|
||||||
|
"derived",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def drive() -> tuple[list[str], list[str], list[str]]:
|
||||||
|
backed_up: list[str] = []
|
||||||
|
created: list[str] = []
|
||||||
|
inspected: list[str] = []
|
||||||
|
|
||||||
|
def record_backup(versions_dir, volume_name, volume_dir, *, authoritative, source):
|
||||||
|
backed_up.append(volume_name)
|
||||||
|
|
||||||
|
with (
|
||||||
|
mock.patch("sys.argv", ARGV),
|
||||||
|
mock.patch.object(app, "get_machine_id", return_value="machine"),
|
||||||
|
mock.patch.object(app, "create_version_directory", return_value="/gen"),
|
||||||
|
mock.patch.object(
|
||||||
|
app,
|
||||||
|
"create_volume_directory",
|
||||||
|
side_effect=lambda _version_dir, name: created.append(name) or "/gen/vol",
|
||||||
|
),
|
||||||
|
mock.patch.object(app, "load_databases_df", return_value=None),
|
||||||
|
mock.patch.object(
|
||||||
|
app, "docker_volume_names", return_value=["derived", "state"]
|
||||||
|
),
|
||||||
|
mock.patch.object(
|
||||||
|
app,
|
||||||
|
"containers_using_volume",
|
||||||
|
side_effect=lambda name: inspected.append(name) or ["app"],
|
||||||
|
),
|
||||||
|
mock.patch.object(app, "volume_is_fully_ignored", return_value=False),
|
||||||
|
mock.patch.object(app, "backup_dumps_for_volume", return_value=(False, False)),
|
||||||
|
mock.patch.object(app, "get_storage_path", return_value="/data/"),
|
||||||
|
mock.patch.object(app, "stamp_directory"),
|
||||||
|
mock.patch.object(app, "handle_docker_compose_services"),
|
||||||
|
mock.patch.object(app.os.path, "isdir", return_value=True),
|
||||||
|
mock.patch.object(app, "backup_volume", side_effect=record_backup),
|
||||||
|
mock.patch.object(app, "filter_stoppable", return_value=[]),
|
||||||
|
mock.patch.object(app, "requires_stop", return_value=False),
|
||||||
|
mock.patch.object(app, "change_containers_status"),
|
||||||
|
):
|
||||||
|
app.main()
|
||||||
|
return backed_up, created, inspected
|
||||||
|
|
||||||
|
|
||||||
|
class TestVolumesNoBackupRequired(unittest.TestCase):
|
||||||
|
def test_the_named_volume_is_never_backed_up(self) -> None:
|
||||||
|
backed_up, _created, _inspected = drive()
|
||||||
|
self.assertNotIn("derived", backed_up)
|
||||||
|
|
||||||
|
def test_a_sibling_volume_of_the_same_container_survives(self) -> None:
|
||||||
|
backed_up, _created, _inspected = drive()
|
||||||
|
self.assertEqual(backed_up, ["state"])
|
||||||
|
|
||||||
|
def test_no_generation_directory_is_created_for_it(self) -> None:
|
||||||
|
_backed_up, created, _inspected = drive()
|
||||||
|
self.assertEqual(created, ["state"])
|
||||||
|
|
||||||
|
def test_the_skip_precedes_the_container_inspection(self) -> None:
|
||||||
|
_backed_up, _created, inspected = drive()
|
||||||
|
self.assertEqual(inspected, ["state"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -39,7 +39,9 @@ class TestSnapshotFlags(unittest.TestCase):
|
|||||||
parse("--snapshot", "ext4", "--snapshot-subject", "/var/lib/docker")
|
parse("--snapshot", "ext4", "--snapshot-subject", "/var/lib/docker")
|
||||||
|
|
||||||
def test_zfs_is_accepted(self) -> None:
|
def test_zfs_is_accepted(self) -> None:
|
||||||
self.assertEqual(parse("--snapshot", "zfs", "--snapshot-subject", "/d").snapshot, "zfs")
|
self.assertEqual(
|
||||||
|
parse("--snapshot", "zfs", "--snapshot-subject", "/d").snapshot, "zfs"
|
||||||
|
)
|
||||||
|
|
||||||
def test_shutdown_is_rejected_because_nothing_is_stopped(self) -> None:
|
def test_shutdown_is_rejected_because_nothing_is_stopped(self) -> None:
|
||||||
with self.assertRaises(SystemExit):
|
with self.assertRaises(SystemExit):
|
||||||
@@ -48,6 +50,22 @@ class TestSnapshotFlags(unittest.TestCase):
|
|||||||
def test_shutdown_stays_available_without_a_snapshot(self) -> None:
|
def test_shutdown_stays_available_without_a_snapshot(self) -> None:
|
||||||
self.assertTrue(parse("--shutdown").shutdown)
|
self.assertTrue(parse("--shutdown").shutdown)
|
||||||
|
|
||||||
|
def test_hard_restart_is_rejected_because_nothing_is_stopped(self) -> None:
|
||||||
|
with self.assertRaises(SystemExit):
|
||||||
|
parse(
|
||||||
|
"--snapshot",
|
||||||
|
"btrfs",
|
||||||
|
"--snapshot-subject",
|
||||||
|
"/d",
|
||||||
|
"--hard-restart-projects",
|
||||||
|
"mailu",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_hard_restart_stays_available_without_a_snapshot(self) -> None:
|
||||||
|
self.assertEqual(
|
||||||
|
parse("--hard-restart-projects", "mailu").hard_restart_projects, ["mailu"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestRequiredFlags(unittest.TestCase):
|
class TestRequiredFlags(unittest.TestCase):
|
||||||
def test_backups_dir_is_required(self) -> None:
|
def test_backups_dir_is_required(self) -> None:
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
|
from baudolo.backup.shell import BackupException
|
||||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
||||||
|
|
||||||
|
|
||||||
@@ -21,27 +22,41 @@ class Runner:
|
|||||||
|
|
||||||
|
|
||||||
class TestBtrfs(unittest.TestCase):
|
class TestBtrfs(unittest.TestCase):
|
||||||
def test_it_creates_a_read_only_snapshot_beside_the_subject(self) -> None:
|
def test_it_creates_a_read_only_snapshot_inside_the_subject(self) -> None:
|
||||||
run = Runner()
|
run = Runner()
|
||||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
||||||
pass
|
pass
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
run.calls[0],
|
run.calls[0],
|
||||||
"btrfs subvolume snapshot -r /var/lib/docker /var/lib/.baudolo-20260731",
|
"btrfs subvolume snapshot -r /var/lib/docker /var/lib/docker/.baudolo-20260731",
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_it_removes_the_snapshot_afterwards(self) -> None:
|
def test_it_removes_the_snapshot_afterwards(self) -> None:
|
||||||
run = Runner()
|
run = Runner()
|
||||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
||||||
pass
|
pass
|
||||||
self.assertEqual(run.calls[-1], "btrfs subvolume delete /var/lib/.baudolo-20260731")
|
self.assertEqual(
|
||||||
|
run.calls[-1], "btrfs subvolume delete /var/lib/docker/.baudolo-20260731"
|
||||||
|
)
|
||||||
|
|
||||||
def test_it_maps_a_volume_path_into_the_snapshot(self) -> None:
|
def test_it_maps_a_volume_path_into_the_snapshot(self) -> None:
|
||||||
run = Runner()
|
run = Runner()
|
||||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
with volume_snapshot(
|
||||||
|
"btrfs", "/var/lib/docker", "20260731", run=run
|
||||||
|
) as resolve:
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
resolve("/var/lib/docker/volumes/postgres_data/_data"),
|
resolve("/var/lib/docker/volumes/postgres_data/_data"),
|
||||||
"/var/lib/.baudolo-20260731/volumes/postgres_data/_data",
|
"/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_it_keeps_the_trailing_slash_rsync_reads_as_contents(self) -> None:
|
||||||
|
run = Runner()
|
||||||
|
with volume_snapshot(
|
||||||
|
"btrfs", "/var/lib/docker", "20260731", run=run
|
||||||
|
) as resolve:
|
||||||
|
self.assertEqual(
|
||||||
|
resolve("/var/lib/docker/volumes/postgres_data/_data/"),
|
||||||
|
"/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data/",
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_it_removes_the_snapshot_even_when_the_body_raises(self) -> None:
|
def test_it_removes_the_snapshot_even_when_the_body_raises(self) -> None:
|
||||||
@@ -93,14 +108,38 @@ class TestRejections(unittest.TestCase):
|
|||||||
|
|
||||||
def test_a_path_outside_the_subject_is_rejected(self) -> None:
|
def test_a_path_outside_the_subject_is_rejected(self) -> None:
|
||||||
run = Runner()
|
run = Runner()
|
||||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
with volume_snapshot(
|
||||||
|
"btrfs", "/var/lib/docker", "20260731", run=run
|
||||||
|
) as resolve:
|
||||||
with self.assertRaises(SnapshotError):
|
with self.assertRaises(SnapshotError):
|
||||||
resolve("/etc/passwd")
|
resolve("/etc/passwd")
|
||||||
|
|
||||||
def test_the_subject_itself_resolves_to_the_snapshot_root(self) -> None:
|
def test_the_subject_itself_resolves_to_the_snapshot_root(self) -> None:
|
||||||
run = Runner()
|
run = Runner()
|
||||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
with volume_snapshot(
|
||||||
self.assertEqual(resolve("/var/lib/docker"), "/var/lib/.baudolo-20260731")
|
"btrfs", "/var/lib/docker", "20260731", run=run
|
||||||
|
) as resolve:
|
||||||
|
self.assertEqual(
|
||||||
|
resolve("/var/lib/docker"), "/var/lib/docker/.baudolo-20260731"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Busy(Runner):
|
||||||
|
def __call__(self, command: str) -> list[str]:
|
||||||
|
if command.startswith("btrfs subvolume delete"):
|
||||||
|
raise BackupException("target is busy")
|
||||||
|
return super().__call__(command)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRemovalFailure(unittest.TestCase):
|
||||||
|
def test_a_failed_removal_does_not_fail_a_completed_run(self) -> None:
|
||||||
|
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=Busy()):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def test_a_failed_removal_does_not_mask_the_body(self) -> None:
|
||||||
|
with self.assertRaises(ZeroDivisionError):
|
||||||
|
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=Busy()):
|
||||||
|
raise ZeroDivisionError
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
Reference in New Issue
Block a user