mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2026-08-24 14:54:32 +00:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 90d289d92f | |||
| 57fc7c96bc | |||
| 30fd68bdcf | |||
| 36b2336742 | |||
| 756e236d10 | |||
| 1dfeb17ab4 | |||
| cd21f1fa67 |
@@ -1,6 +1,7 @@
|
|||||||
{
|
{
|
||||||
"permissions": {
|
"permissions": {
|
||||||
"ask": [
|
"ask": [
|
||||||
|
"Bash(git commit*)",
|
||||||
"Edit(CHANGELOG.md)",
|
"Edit(CHANGELOG.md)",
|
||||||
"Write(CHANGELOG.md)",
|
"Write(CHANGELOG.md)",
|
||||||
"Edit(pyproject.toml)",
|
"Edit(pyproject.toml)",
|
||||||
|
|||||||
49
CHANGELOG.md
49
CHANGELOG.md
@@ -1,5 +1,54 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## [3.4.3] - 2026-08-16
|
||||||
|
|
||||||
|
- Backup: *create_version_directory* carried *exist_ok=True*, so a run starting
|
||||||
|
in the same wall-clock second as its predecessor claimed that predecessor's
|
||||||
|
generation. Generation names carry seconds, and a host with little to copy
|
||||||
|
finishes inside one — rsync *--delete* then overwrote a finished generation,
|
||||||
|
and only afterwards did *create_stamp_file* refuse the already-stamped
|
||||||
|
directory and exit 2. The guard reported the damage instead of preventing it.
|
||||||
|
- Backup: the generation directory is claimed exclusively. Claiming it is the
|
||||||
|
first filesystem action of a run, so a collision aborts before the first
|
||||||
|
write and names the second it collided on.
|
||||||
|
- Tests: the idempotence test asserted the reuse and gave way to one that
|
||||||
|
requires the refusal.
|
||||||
|
|
||||||
|
## [3.4.2] - 2026-08-15
|
||||||
|
|
||||||
|
- Backup: *has_image* matched the raw *.Config.Image*, so the registry host and
|
||||||
|
the tag decided the dump tool. On a swarm node named after the app under test,
|
||||||
|
*svc-db-mariadb-swarm-mgr-01:5000/postgres_custom* read as MariaDB and
|
||||||
|
*mariadb-dump* ran inside a Postgres container: exit 127, and the
|
||||||
|
*BackupException* took the backup unit with it.
|
||||||
|
- Backup: *image_name* strips digest, tag and registry host, so the engine rests
|
||||||
|
on the repository path alone — the exact-matching intent of 3.0.0 applied to
|
||||||
|
the one place that change did not reach.
|
||||||
|
- Tests: both false-positive directions on *has_image*, plus an e2e that
|
||||||
|
reproduces the shape with a *docker tag* and asserts a real *pg_dump* lands.
|
||||||
|
|
||||||
|
## [3.4.1] - 2026-08-05
|
||||||
|
|
||||||
|
- Backup: each volume is copied twice into the same destination — once hot,
|
||||||
|
once cold after the container is stopped — and rsync ran with *-b*, so
|
||||||
|
*--delete* renamed rather than removed a file the source had dropped between
|
||||||
|
the passes. Stopping a container is what makes the source drop files: a
|
||||||
|
graceful shutdown flushes and the format rolls its commit point. The
|
||||||
|
superseded file survived as *name~* beside the real one and was restored into
|
||||||
|
live data.
|
||||||
|
- Backup: for an opaque payload that is stale bytes nobody reads; for a format
|
||||||
|
that enumerates its own directory it is corruption. Lucene resolves the
|
||||||
|
current commit by parsing every file starting with *segments* as a radix-36
|
||||||
|
generation, so a restored *segments_3~* leaves the shard store unreadable and
|
||||||
|
the primary at *NO_VALID_SHARD_COPY*. With *.security-7* unallocatable the
|
||||||
|
reserved *elastic* user has no password hash, every probe answers 401 and the
|
||||||
|
container never turns healthy.
|
||||||
|
- Backup: *--link-dest* already provides the incrementals and nothing reads the
|
||||||
|
twins — the restore path is an unfiltered *rsync -avv --delete* into the live
|
||||||
|
volume. Dropping *-b* leaves the predecessor generation byte-identical, keeps
|
||||||
|
the hardlinks intact and makes generations smaller, never larger.
|
||||||
|
- Tests: the absence of *--backup* is asserted on the rsync invocation.
|
||||||
|
|
||||||
## [3.4.0] - 2026-08-02
|
## [3.4.0] - 2026-08-02
|
||||||
|
|
||||||
- Backup: *-a* implies *-D*, so a generation was written with
|
- Backup: *-a* implies *-D*, so a generation was written with
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "backup-docker-to-local"
|
name = "backup-docker-to-local"
|
||||||
version = "3.4.0"
|
version = "3.4.3"
|
||||||
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.9"
|
requires-python = ">=3.9"
|
||||||
|
|||||||
@@ -9,9 +9,29 @@ def get_image_info(container: str) -> str:
|
|||||||
)[0]
|
)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def image_name(container: str) -> str:
|
||||||
|
"""The image's repository path, without registry host, tag or digest.
|
||||||
|
|
||||||
|
A swarm node that hosts the local registry puts its own hostname in front
|
||||||
|
of every pull, so the raw reference of a Postgres container can read
|
||||||
|
`svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5`. Matching the
|
||||||
|
whole reference finds "mariadb" there and dumps the database with
|
||||||
|
mariadb-dump, which the Postgres image does not ship (exit 127). Tags bite
|
||||||
|
the same way: `xwiki_custom:lts-postgres-tomcat`.
|
||||||
|
"""
|
||||||
|
reference = get_image_info(container).strip().split("@", 1)[0]
|
||||||
|
head, _, tail = reference.rpartition("/")
|
||||||
|
tail = tail.split(":", 1)[0]
|
||||||
|
if head:
|
||||||
|
registry = head.split("/", 1)[0]
|
||||||
|
if "." in registry or ":" in registry or registry == "localhost":
|
||||||
|
head = head.partition("/")[2]
|
||||||
|
return f"{head}/{tail}" if head else tail
|
||||||
|
|
||||||
|
|
||||||
def has_image(container: str, pattern: str) -> bool:
|
def has_image(container: str, pattern: str) -> bool:
|
||||||
"""Return True if container's image contains the pattern."""
|
"""Return True if the container's image name contains the pattern."""
|
||||||
return pattern in get_image_info(container)
|
return pattern in image_name(container)
|
||||||
|
|
||||||
|
|
||||||
def docker_volume_names() -> list[str]:
|
def docker_volume_names() -> list[str]:
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import pathlib
|
|||||||
|
|
||||||
from dirval import create_stamp_file
|
from dirval import create_stamp_file
|
||||||
|
|
||||||
from .shell import execute_shell_command
|
from .shell import BackupException, execute_shell_command
|
||||||
|
|
||||||
|
|
||||||
def get_machine_id() -> str:
|
def get_machine_id() -> str:
|
||||||
@@ -23,7 +23,14 @@ def stamp_directory(version_dir: str) -> None:
|
|||||||
|
|
||||||
def create_version_directory(versions_dir: str, backup_time: str) -> str:
|
def create_version_directory(versions_dir: str, backup_time: str) -> str:
|
||||||
version_dir = os.path.join(versions_dir, backup_time)
|
version_dir = os.path.join(versions_dir, backup_time)
|
||||||
pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
|
try:
|
||||||
|
pathlib.Path(version_dir).mkdir(parents=True)
|
||||||
|
except FileExistsError:
|
||||||
|
raise BackupException(
|
||||||
|
f"generation {backup_time} already exists at {version_dir}; "
|
||||||
|
"another run claimed this second - refusing to write into it, "
|
||||||
|
"since rsync --delete would overwrite that generation"
|
||||||
|
) from None
|
||||||
return version_dir
|
return version_dir
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ def backup_volume(
|
|||||||
verify = "--checksum " if authoritative else ""
|
verify = "--checksum " if authoritative else ""
|
||||||
|
|
||||||
cmd = (
|
cmd = (
|
||||||
f"rsync -abP --no-D --delete --delete-excluded "
|
f"rsync -aP --no-D --delete --delete-excluded "
|
||||||
f"{verify}{link_dest} {source} {dest}"
|
f"{verify}{link_dest} {source} {dest}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
105
tests/e2e/test_e2e_engine_detection_registry_prefix.py
Normal file
105
tests/e2e/test_e2e_engine_detection_registry_prefix.py
Normal file
@@ -0,0 +1,105 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from .helpers import (
|
||||||
|
POSTGRES_IMAGE,
|
||||||
|
POSTGRES_DATA_DIR,
|
||||||
|
backup_run,
|
||||||
|
backup_path,
|
||||||
|
cleanup_docker,
|
||||||
|
create_minimal_compose_dir,
|
||||||
|
ensure_empty_dir,
|
||||||
|
latest_version_dir,
|
||||||
|
require_docker,
|
||||||
|
unique,
|
||||||
|
write_databases_csv,
|
||||||
|
run,
|
||||||
|
wait_for_postgres,
|
||||||
|
)
|
||||||
|
|
||||||
|
REGISTRY_HOST = "svc-db-mariadb-swarm-mgr-01:5000"
|
||||||
|
|
||||||
|
|
||||||
|
class TestE2EEngineDetectionRegistryPrefix(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
require_docker()
|
||||||
|
cls.prefix = unique("baudolo-e2e-registry-prefix")
|
||||||
|
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||||
|
ensure_empty_dir(cls.backups_dir)
|
||||||
|
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||||
|
cls.repo_name = cls.prefix
|
||||||
|
|
||||||
|
cls.image = f"{REGISTRY_HOST}/postgres_custom:17-3.5"
|
||||||
|
cls.pg_container = f"{cls.prefix}-pg"
|
||||||
|
cls.pg_volume = f"{cls.prefix}-pg-vol"
|
||||||
|
cls.containers = [cls.pg_container]
|
||||||
|
cls.volumes = [cls.pg_volume]
|
||||||
|
|
||||||
|
run(["docker", "pull", POSTGRES_IMAGE])
|
||||||
|
run(["docker", "tag", POSTGRES_IMAGE, cls.image])
|
||||||
|
run(["docker", "volume", "create", cls.pg_volume])
|
||||||
|
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"-d",
|
||||||
|
"--name",
|
||||||
|
cls.pg_container,
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_PASSWORD=pgpw",
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_DB=appdb",
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_USER=postgres",
|
||||||
|
"-v",
|
||||||
|
f"{cls.pg_volume}:{POSTGRES_DATA_DIR}",
|
||||||
|
cls.image,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||||
|
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
cls.pg_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
"psql -U postgres -d appdb -c \"CREATE TABLE t (id int primary key, v text); INSERT INTO t VALUES (1,'ok');\"",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
|
write_databases_csv(
|
||||||
|
cls.databases_csv, [(cls.pg_container, "appdb", "postgres", "pgpw")]
|
||||||
|
)
|
||||||
|
|
||||||
|
backup_run(
|
||||||
|
backups_dir=cls.backups_dir,
|
||||||
|
repo_name=cls.repo_name,
|
||||||
|
compose_dir=cls.compose_dir,
|
||||||
|
databases_csv=cls.databases_csv,
|
||||||
|
database_containers=[cls.pg_container],
|
||||||
|
images_no_stop_required=[cls.image],
|
||||||
|
)
|
||||||
|
|
||||||
|
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls) -> None:
|
||||||
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
run(["docker", "rmi", cls.image], check=False)
|
||||||
|
|
||||||
|
def test_the_registry_host_does_not_pick_the_engine(self) -> None:
|
||||||
|
p = (
|
||||||
|
backup_path(self.backups_dir, self.repo_name, self.version, self.pg_volume)
|
||||||
|
/ "sql"
|
||||||
|
/ "appdb.backup.sql"
|
||||||
|
)
|
||||||
|
self.assertTrue(p.is_file(), f"Expected a pg_dump at: {p}")
|
||||||
|
self.assertIn("Dumped by pg_dump", p.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
50
tests/unit/backup/test_docker_image_name.py
Normal file
50
tests/unit/backup/test_docker_image_name.py
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from baudolo.backup import docker as docker_mod
|
||||||
|
|
||||||
|
|
||||||
|
def _with_image(reference: str):
|
||||||
|
return patch.object(docker_mod, "execute_shell_command", return_value=[reference])
|
||||||
|
|
||||||
|
|
||||||
|
class TestImageName(unittest.TestCase):
|
||||||
|
def test_plain_reference(self) -> None:
|
||||||
|
with _with_image("postgres:16"):
|
||||||
|
self.assertEqual(docker_mod.image_name("c1"), "postgres")
|
||||||
|
|
||||||
|
def test_registry_host_is_dropped(self) -> None:
|
||||||
|
with _with_image("svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5"):
|
||||||
|
self.assertEqual(docker_mod.image_name("c1"), "postgres_custom")
|
||||||
|
|
||||||
|
def test_pull_through_path_is_kept(self) -> None:
|
||||||
|
with _with_image(
|
||||||
|
"svc-db-mariadb-swarm-mgr-01:5000/ghcr.io/x/mirror/docker.io/postgres:16"
|
||||||
|
):
|
||||||
|
self.assertEqual(
|
||||||
|
docker_mod.image_name("c1"), "ghcr.io/x/mirror/docker.io/postgres"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_digest_is_dropped(self) -> None:
|
||||||
|
with _with_image("registry:5000/postgres@sha256:" + "0" * 64):
|
||||||
|
self.assertEqual(docker_mod.image_name("c1"), "postgres")
|
||||||
|
|
||||||
|
|
||||||
|
class TestHasImage(unittest.TestCase):
|
||||||
|
def test_registry_hostname_does_not_decide_the_engine(self) -> None:
|
||||||
|
with _with_image("svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5"):
|
||||||
|
self.assertFalse(docker_mod.has_image("c1", "mariadb"))
|
||||||
|
with _with_image("svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5"):
|
||||||
|
self.assertTrue(docker_mod.has_image("c1", "postgres"))
|
||||||
|
|
||||||
|
def test_tag_does_not_decide_the_engine(self) -> None:
|
||||||
|
with _with_image("registry:5000/xwiki_custom:lts-postgres-tomcat"):
|
||||||
|
self.assertFalse(docker_mod.has_image("c1", "postgres"))
|
||||||
|
|
||||||
|
def test_mirrored_mariadb_still_matches(self) -> None:
|
||||||
|
with _with_image("registry:5000/ghcr.io/x/mirror/docker.io/mariadb:11"):
|
||||||
|
self.assertTrue(docker_mod.has_image("c1", "mariadb"))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -8,6 +8,7 @@ from pathlib import Path
|
|||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
from baudolo.backup import layout as mod
|
from baudolo.backup import layout as mod
|
||||||
|
from baudolo.backup.shell import BackupException
|
||||||
|
|
||||||
|
|
||||||
class TestVersionDirectory(unittest.TestCase):
|
class TestVersionDirectory(unittest.TestCase):
|
||||||
@@ -17,11 +18,12 @@ class TestVersionDirectory(unittest.TestCase):
|
|||||||
self.assertTrue(Path(created).is_dir())
|
self.assertTrue(Path(created).is_dir())
|
||||||
self.assertEqual(Path(created).name, "20260731020304")
|
self.assertEqual(Path(created).name, "20260731020304")
|
||||||
|
|
||||||
def test_it_is_idempotent(self) -> None:
|
def test_it_refuses_a_generation_another_run_already_claimed(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
first = mod.create_version_directory(tmp, "20260731")
|
mod.create_version_directory(tmp, "20260731")
|
||||||
second = mod.create_version_directory(tmp, "20260731")
|
with self.assertRaises(BackupException) as caught:
|
||||||
self.assertEqual(first, second)
|
mod.create_version_directory(tmp, "20260731")
|
||||||
|
self.assertIn("20260731", str(caught.exception))
|
||||||
|
|
||||||
def test_it_creates_missing_parents(self) -> None:
|
def test_it_creates_missing_parents(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
|||||||
@@ -46,6 +46,11 @@ class TestBackupVolume(unittest.TestCase):
|
|||||||
def test_it_carries_no_kernel_objects_into_a_generation(self) -> None:
|
def test_it_carries_no_kernel_objects_into_a_generation(self) -> None:
|
||||||
self.assertIn("--no-D", self.copy())
|
self.assertIn("--no-D", self.copy())
|
||||||
|
|
||||||
|
def test_it_keeps_no_twin_of_what_the_second_pass_replaces(self) -> None:
|
||||||
|
command = self.copy(authoritative=True)
|
||||||
|
self.assertIn("rsync -aP ", command)
|
||||||
|
self.assertNotIn("--backup", command)
|
||||||
|
|
||||||
def test_it_creates_the_destination(self) -> None:
|
def test_it_creates_the_destination(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
dest = Path(tmp) / "gen" / "demo"
|
dest = Path(tmp) / "gen" / "demo"
|
||||||
|
|||||||
Reference in New Issue
Block a user