mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2026-09-23 21:23:19 +00:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 022b096617 | |||
| 161db213cb | |||
|
|
9d4cc24a59 | ||
|
|
5bf6bc2afe | ||
|
|
4b47ceab30 | ||
|
|
a139f0faa1 | ||
|
|
b6ed45770b |
@@ -1,5 +1,12 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## [7.0.2] - 2026-09-23
|
||||||
|
|
||||||
|
* Backup: failed live pre-copy no longer aborts when a stopped copy follows
|
||||||
|
* Backup: stopped --checksum copy replaces pre-copy hit by live writer (rc 23)
|
||||||
|
* Backup: failing sole live copy or failing stopped copy still fails the run
|
||||||
|
* Tests: e2e live writer shrinking files mid-read reproduces the rsync failure
|
||||||
|
|
||||||
## [7.0.1] - 2026-08-18
|
## [7.0.1] - 2026-08-18
|
||||||
|
|
||||||
- Restore: *--empty* no longer aborts on a database that carries an extension.
|
- Restore: *--empty* no longer aborts on a database that carries an extension.
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "backup-docker-to-local"
|
name = "backup-docker-to-local"
|
||||||
version = "7.0.1"
|
version = "7.0.2"
|
||||||
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.9"
|
requires-python = ">=3.9"
|
||||||
@@ -19,7 +19,7 @@ dependencies = [
|
|||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
# Pinned: a ruff minor bump changes which rules fire, and `make test` gates on
|
# Pinned: a ruff minor bump changes which rules fire, and `make test` gates on
|
||||||
# a clean run, so an unpinned lint would fail the suite on an unrelated day.
|
# a clean run, so an unpinned lint would fail the suite on an unrelated day.
|
||||||
lint = ["ruff==0.16.1"]
|
lint = ["ruff==0.16.7"]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
baudolo = "baudolo.backup.__main__:main"
|
baudolo = "baudolo.backup.__main__:main"
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ from .layout import (
|
|||||||
write_manifest,
|
write_manifest,
|
||||||
)
|
)
|
||||||
from .policy import requires_stop, volume_is_fully_ignored
|
from .policy import requires_stop, volume_is_fully_ignored
|
||||||
|
from .shell import BackupError
|
||||||
from .snapshot import snapshot_source, volume_snapshot
|
from .snapshot import snapshot_source, volume_snapshot
|
||||||
from .volume import backup_volume, inspect_backing
|
from .volume import backup_volume, inspect_backing
|
||||||
|
|
||||||
@@ -125,13 +126,23 @@ def main() -> int:
|
|||||||
copy(authoritative=False)
|
copy(authoritative=False)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
copy(authoritative=False)
|
if not requires_stop(containers, args.images_no_stop_required):
|
||||||
if requires_stop(containers, args.images_no_stop_required):
|
copy(authoritative=False)
|
||||||
stoppable = filter_stoppable(containers)
|
continue
|
||||||
change_containers_status(stoppable, "stop")
|
|
||||||
copy(authoritative=True)
|
try:
|
||||||
if not args.shutdown:
|
copy(authoritative=False)
|
||||||
change_containers_status(stoppable, "start")
|
except BackupError as error:
|
||||||
|
print(
|
||||||
|
f"WARNING: live pre-copy of volume '{volume_name}' failed; "
|
||||||
|
f"the copy with its containers stopped replaces it.\n{error}",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
stoppable = filter_stoppable(containers)
|
||||||
|
change_containers_status(stoppable, "stop")
|
||||||
|
copy(authoritative=True)
|
||||||
|
if not args.shutdown:
|
||||||
|
change_containers_status(stoppable, "start")
|
||||||
|
|
||||||
write_manifest(version_dir, outcomes)
|
write_manifest(version_dir, outcomes)
|
||||||
stamp_directory(version_dir)
|
stamp_directory(version_dir)
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ def backup_run(
|
|||||||
images_no_stop_required: list[str],
|
images_no_stop_required: list[str],
|
||||||
images_no_backup_required: list[str] | None = None,
|
images_no_backup_required: list[str] | None = None,
|
||||||
only_sql: bool = False,
|
only_sql: bool = False,
|
||||||
) -> None:
|
) -> subprocess.CompletedProcess:
|
||||||
cmd = [
|
cmd = [
|
||||||
"baudolo",
|
"baudolo",
|
||||||
"--compose-dir",
|
"--compose-dir",
|
||||||
@@ -49,7 +49,7 @@ def backup_run(
|
|||||||
cmd += ["--only-sql"]
|
cmd += ["--only-sql"]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
run(cmd, capture=True, check=True)
|
return run(cmd, capture=True, check=True)
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
print(">>> baudolo failed (exit code:", e.returncode, ")")
|
print(">>> baudolo failed (exit code:", e.returncode, ")")
|
||||||
if e.stdout:
|
if e.stdout:
|
||||||
|
|||||||
110
tests/e2e/test_e2e_files_live_writer_precopy.py
Normal file
110
tests/e2e/test_e2e_files_live_writer_precopy.py
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
"""A writer that shrinks files mid-read fails the live pre-copy, not the run."""
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from .helpers import (
|
||||||
|
backup_path,
|
||||||
|
backup_run,
|
||||||
|
cleanup_docker,
|
||||||
|
create_minimal_compose_dir,
|
||||||
|
ensure_empty_dir,
|
||||||
|
latest_version_dir,
|
||||||
|
require_docker,
|
||||||
|
run,
|
||||||
|
unique,
|
||||||
|
wait_for_log,
|
||||||
|
write_databases_csv,
|
||||||
|
)
|
||||||
|
|
||||||
|
CHURN_FILES = 8
|
||||||
|
|
||||||
|
WRITER = f"""
|
||||||
|
trap 'exit 0' TERM
|
||||||
|
echo hello > /data/hello.txt
|
||||||
|
for i in $(seq 1 {CHURN_FILES}); do
|
||||||
|
(while :; do truncate -s 0 /data/churn$i; truncate -s 16M /data/churn$i; done) &
|
||||||
|
done
|
||||||
|
echo ready
|
||||||
|
wait
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class TestE2EFilesLiveWriterPreCopy(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
require_docker()
|
||||||
|
cls.prefix = unique("baudolo-e2e-live-writer")
|
||||||
|
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||||
|
ensure_empty_dir(cls.backups_dir)
|
||||||
|
|
||||||
|
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||||
|
cls.repo_name = cls.prefix
|
||||||
|
|
||||||
|
cls.volume = f"{cls.prefix}-vol"
|
||||||
|
cls.writer = f"{cls.prefix}-writer"
|
||||||
|
cls.containers = [cls.writer]
|
||||||
|
cls.volumes = [cls.volume]
|
||||||
|
|
||||||
|
run(["docker", "volume", "create", cls.volume])
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"-d",
|
||||||
|
"--name",
|
||||||
|
cls.writer,
|
||||||
|
"-v",
|
||||||
|
f"{cls.volume}:/data",
|
||||||
|
"alpine:3.20",
|
||||||
|
"sh",
|
||||||
|
"-c",
|
||||||
|
WRITER,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
wait_for_log(cls.writer, "ready")
|
||||||
|
|
||||||
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
|
write_databases_csv(cls.databases_csv, [])
|
||||||
|
|
||||||
|
cls.result = backup_run(
|
||||||
|
backups_dir=cls.backups_dir,
|
||||||
|
repo_name=cls.repo_name,
|
||||||
|
compose_dir=cls.compose_dir,
|
||||||
|
databases_csv=cls.databases_csv,
|
||||||
|
database_containers=["dummy-db"],
|
||||||
|
images_no_stop_required=["dummy-image"],
|
||||||
|
)
|
||||||
|
|
||||||
|
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls) -> None:
|
||||||
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
|
||||||
|
def test_the_live_pre_copy_hit_the_writer(self) -> None:
|
||||||
|
self.assertIn(
|
||||||
|
f"WARNING: live pre-copy of volume '{self.volume}' failed",
|
||||||
|
self.result.stdout,
|
||||||
|
"the writer never shrank a file under rsync, so this run proves nothing",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_the_stopped_copy_captured_the_volume(self) -> None:
|
||||||
|
files = (
|
||||||
|
backup_path(self.backups_dir, self.repo_name, self.version, self.volume)
|
||||||
|
/ "files"
|
||||||
|
)
|
||||||
|
self.assertEqual((files / "hello.txt").read_text().strip(), "hello")
|
||||||
|
self.assertEqual(
|
||||||
|
sorted(p.name for p in files.glob("churn*")),
|
||||||
|
sorted(f"churn{i}" for i in range(1, CHURN_FILES + 1)),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_the_writer_runs_again_after_the_backup(self) -> None:
|
||||||
|
state = run(
|
||||||
|
["docker", "inspect", "-f", "{{.State.Running}}", self.writer]
|
||||||
|
).stdout.strip()
|
||||||
|
self.assertEqual(state, "true")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
70
tests/unit/backup/test_app_live_precopy.py
Normal file
70
tests/unit/backup/test_app_live_precopy.py
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
"""Contract of app.main's live copy when no snapshot is taken."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
from baudolo.backup import app
|
||||||
|
from baudolo.backup.shell import BackupError
|
||||||
|
from baudolo.backup.volume import Backing
|
||||||
|
|
||||||
|
from . import BASE_ARGV
|
||||||
|
|
||||||
|
|
||||||
|
def drive(*, stop: bool, fail_live: bool) -> list[str]:
|
||||||
|
events: list[str] = []
|
||||||
|
|
||||||
|
def record(versions_dir, volume_name, volume_dir, *, authoritative, source):
|
||||||
|
events.append("authoritative" if authoritative else "live")
|
||||||
|
if fail_live and not authoritative:
|
||||||
|
raise BackupError("rsync exit code 23")
|
||||||
|
|
||||||
|
with (
|
||||||
|
mock.patch("sys.argv", BASE_ARGV),
|
||||||
|
mock.patch.object(app, "get_machine_id", return_value="machine"),
|
||||||
|
mock.patch.object(app, "create_version_directory", return_value="/gen"),
|
||||||
|
mock.patch.object(app, "create_volume_directory", return_value="/gen/vol"),
|
||||||
|
mock.patch.object(app, "load_databases_df", return_value=None),
|
||||||
|
mock.patch.object(app, "docker_volume_names", return_value=["vol"]),
|
||||||
|
mock.patch.object(app, "containers_using_volume", return_value=["c"]),
|
||||||
|
mock.patch.object(app, "volume_is_fully_ignored", return_value=False),
|
||||||
|
mock.patch.object(app, "backup_dumps_for_volume", return_value=(False, False)),
|
||||||
|
mock.patch.object(
|
||||||
|
app,
|
||||||
|
"inspect_backing",
|
||||||
|
return_value=Backing("/var/lib/docker/volumes/vol/_data"),
|
||||||
|
),
|
||||||
|
mock.patch.object(app, "requires_stop", return_value=stop),
|
||||||
|
mock.patch.object(app, "filter_stoppable", return_value=["c"]),
|
||||||
|
mock.patch.object(
|
||||||
|
app,
|
||||||
|
"change_containers_status",
|
||||||
|
side_effect=lambda containers, status: events.append(status),
|
||||||
|
),
|
||||||
|
mock.patch.object(app, "write_manifest"),
|
||||||
|
mock.patch.object(app, "stamp_directory"),
|
||||||
|
mock.patch.object(app, "handle_docker_compose_services"),
|
||||||
|
mock.patch.object(app, "backup_volume", side_effect=record),
|
||||||
|
):
|
||||||
|
app.main()
|
||||||
|
return events
|
||||||
|
|
||||||
|
|
||||||
|
class TestLivePreCopy(unittest.TestCase):
|
||||||
|
def test_a_failed_pre_copy_is_replaced_by_the_stopped_copy(self) -> None:
|
||||||
|
self.assertEqual(
|
||||||
|
drive(stop=True, fail_live=True),
|
||||||
|
["live", "stop", "authoritative", "start"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_a_failed_live_copy_without_a_stop_aborts_the_run(self) -> None:
|
||||||
|
with self.assertRaises(BackupError):
|
||||||
|
drive(stop=False, fail_live=True)
|
||||||
|
|
||||||
|
def test_a_volume_without_a_stop_is_copied_live_once(self) -> None:
|
||||||
|
self.assertEqual(drive(stop=False, fail_live=False), ["live"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user