mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2026-08-24 14:54:32 +00:00
Compare commits
19 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f437787e64 | |||
| 2129c5e362 | |||
| a0204fd3ea | |||
| 90d289d92f | |||
| 57fc7c96bc | |||
| 30fd68bdcf | |||
| 36b2336742 | |||
| 756e236d10 | |||
| 1dfeb17ab4 | |||
| cd21f1fa67 | |||
| 8a93a61ca9 | |||
| 988d92534c | |||
| 934e693810 | |||
| 2e0e67ca87 | |||
| 95c34d4db0 | |||
| c2f1cb8e8c | |||
| eeaa838d02 | |||
| 4e2b3641f9 | |||
| b10d50efbe |
@@ -1,6 +1,7 @@
|
||||
{
|
||||
"permissions": {
|
||||
"ask": [
|
||||
"Bash(git commit*)",
|
||||
"Edit(CHANGELOG.md)",
|
||||
"Write(CHANGELOG.md)",
|
||||
"Edit(pyproject.toml)",
|
||||
|
||||
137
CHANGELOG.md
137
CHANGELOG.md
@@ -1,5 +1,142 @@
|
||||
# Changelog
|
||||
|
||||
## [3.5.0] - 2026-08-17
|
||||
|
||||
- Restore: a *database = '*'* row makes the backup write
|
||||
*<instance>.cluster.backup.sql* via *pg_dumpall*, and nothing could read it
|
||||
back — the CLI knew *files*, *postgres* and *mariadb*, so that dump was
|
||||
stored and unrestorable. *baudolo-restore cluster* replays it against the
|
||||
control database, deliberately without *--single-transaction* because
|
||||
CREATE DATABASE is forbidden inside a transaction block, and filters out the
|
||||
CREATE ROLE of the connecting role, which the pre-clean cannot drop while it
|
||||
holds the session. *--empty* drops the cluster's databases first, then
|
||||
releases what its roles still own, then the roles themselves.
|
||||
|
||||
- Lint: ruff was never wired into the repository — no target, no CI step, no
|
||||
pin — and reported 45 findings across sources and tests. *make ruff* and
|
||||
*make lint* now run it over every file, *make test* gates on a clean run as a
|
||||
fourth parallel spur, and the linter is pinned in a *lint* extra because a
|
||||
minor bump changes which rules fire.
|
||||
|
||||
## [3.4.3] - 2026-08-16
|
||||
|
||||
- Backup: *create_version_directory* carried *exist_ok=True*, so a run starting
|
||||
in the same wall-clock second as its predecessor claimed that predecessor's
|
||||
generation. Generation names carry seconds, and a host with little to copy
|
||||
finishes inside one — rsync *--delete* then overwrote a finished generation,
|
||||
and only afterwards did *create_stamp_file* refuse the already-stamped
|
||||
directory and exit 2. The guard reported the damage instead of preventing it.
|
||||
- Backup: the generation directory is claimed exclusively. Claiming it is the
|
||||
first filesystem action of a run, so a collision aborts before the first
|
||||
write and names the second it collided on.
|
||||
- Tests: the idempotence test asserted the reuse and gave way to one that
|
||||
requires the refusal.
|
||||
|
||||
## [3.4.2] - 2026-08-15
|
||||
|
||||
- Backup: *has_image* matched the raw *.Config.Image*, so the registry host and
|
||||
the tag decided the dump tool. On a swarm node named after the app under test,
|
||||
*svc-db-mariadb-swarm-mgr-01:5000/postgres_custom* read as MariaDB and
|
||||
*mariadb-dump* ran inside a Postgres container: exit 127, and the
|
||||
*BackupException* took the backup unit with it.
|
||||
- Backup: *image_name* strips digest, tag and registry host, so the engine rests
|
||||
on the repository path alone — the exact-matching intent of 3.0.0 applied to
|
||||
the one place that change did not reach.
|
||||
- Tests: both false-positive directions on *has_image*, plus an e2e that
|
||||
reproduces the shape with a *docker tag* and asserts a real *pg_dump* lands.
|
||||
|
||||
## [3.4.1] - 2026-08-05
|
||||
|
||||
- Backup: each volume is copied twice into the same destination — once hot,
|
||||
once cold after the container is stopped — and rsync ran with *-b*, so
|
||||
*--delete* renamed rather than removed a file the source had dropped between
|
||||
the passes. Stopping a container is what makes the source drop files: a
|
||||
graceful shutdown flushes and the format rolls its commit point. The
|
||||
superseded file survived as *name~* beside the real one and was restored into
|
||||
live data.
|
||||
- Backup: for an opaque payload that is stale bytes nobody reads; for a format
|
||||
that enumerates its own directory it is corruption. Lucene resolves the
|
||||
current commit by parsing every file starting with *segments* as a radix-36
|
||||
generation, so a restored *segments_3~* leaves the shard store unreadable and
|
||||
the primary at *NO_VALID_SHARD_COPY*. With *.security-7* unallocatable the
|
||||
reserved *elastic* user has no password hash, every probe answers 401 and the
|
||||
container never turns healthy.
|
||||
- Backup: *--link-dest* already provides the incrementals and nothing reads the
|
||||
twins — the restore path is an unfiltered *rsync -avv --delete* into the live
|
||||
volume. Dropping *-b* leaves the predecessor generation byte-identical, keeps
|
||||
the hardlinks intact and makes generations smaller, never larger.
|
||||
- Tests: the absence of *--backup* is asserted on the rsync invocation.
|
||||
|
||||
## [3.4.0] - 2026-08-02
|
||||
|
||||
- Backup: *-a* implies *-D*, so a generation was written with
|
||||
*--devices --specials* and rsync recreated every unix socket and fifo found in
|
||||
a volume. Where the backup root is an nfs-ganesha export, ganesha accepts the
|
||||
socket on write but cannot serve it back, and the remote pull's sender then
|
||||
fails with *readdir* / *readlink_stat* "Invalid argument (22)" and exits 23 —
|
||||
deterministically, for every retry. *--no-D* keeps them out of the generation.
|
||||
- Backup: nothing restorable is lost. Sockets and fifos are recreated by the
|
||||
daemons that own them, and the postfix queue itself — *incoming*, *active*,
|
||||
*deferred*, *hold*, *maildrop* — is unaffected, so accepted-but-undelivered
|
||||
mail stays in the backup. Device nodes go too; the only volume that could hold
|
||||
them is a nested docker data root, which does not belong in a backup anyway.
|
||||
- Tests: the flag is asserted on the rsync invocation.
|
||||
|
||||
## [3.3.0] - 2026-08-02
|
||||
|
||||
- Backup: *--volumes-no-backup-required* excludes a volume by name.
|
||||
*--images-no-backup-required* resolves through *volume_is_fully_ignored*,
|
||||
which skips a volume only when every container using it is ignored — a
|
||||
container holding a derived tree beside state that must be kept cannot
|
||||
express the exclusion at all. A docker-in-docker data root is exactly that
|
||||
shape, and excluding by image would drop all three of its volumes.
|
||||
- Backup: the name check runs before *containers_using_volume*, so an excluded
|
||||
volume costs no docker inspection and the decision does not depend on which
|
||||
containers exist when the run starts.
|
||||
- Tests: two volumes off one container, asserting the sibling survives — the
|
||||
property the image lever cannot provide — as unit and end-to-end.
|
||||
|
||||
## [3.2.2] - 2026-07-31
|
||||
|
||||
- Backup: the btrfs snapshot is carved inside its subject, as
|
||||
*<data root>/.baudolo-<tag>*, not beside it. The kernel refuses a snapshot
|
||||
whose destination is on another filesystem, which is exactly what the parent
|
||||
directory is when the data root is a mountpoint of its own — a dedicated disk
|
||||
mounted onto */var/lib/docker* failed every run with EXDEV. Placing it inside
|
||||
makes source and destination the same filesystem by construction, and aligns
|
||||
btrfs with the zfs path, which already resolves its snapshot inside the
|
||||
subject at *<subject>/.zfs/snapshot/<tag>*. A leftover from an interrupted run
|
||||
appears in the next snapshot as an empty directory rather than recursing,
|
||||
since btrfs does not include nested subvolumes.
|
||||
|
||||
## [3.2.1] - 2026-07-31
|
||||
|
||||
- Backup: the snapshot resolver keeps the trailing separator *get_storage_path*
|
||||
puts on a volume path — *os.path.abspath* stripped it. rsync reads *dir* as
|
||||
"copy the directory" where *dir/* means "copy its contents", so every snapshot
|
||||
generation landed at *<volume>/files/_data/...* while the live path lands at
|
||||
*<volume>/files/...*. Restores read the live layout, and *--link-dest* had
|
||||
nothing to match against the previous generation.
|
||||
- Backup: snapshot teardown no longer fails a completed run. A busy
|
||||
*btrfs subvolume delete* raised out of the *finally*, skipping the generation
|
||||
stamp and the compose handling on a run whose data was already copied, and
|
||||
masking whatever the body had raised. The leftover is reported instead.
|
||||
- Backup: a volume created after the snapshot was taken is copied live with a
|
||||
warning instead of aborting the run. Nothing is stopped in snapshot mode, so
|
||||
the host keeps creating volumes for the whole copy.
|
||||
- Backup: the snapshot pass compares by content (*--checksum*) again. 3.2.0
|
||||
dropped it because a snapshot source cannot move, which is true, but the
|
||||
comparison that matters is against *--link-dest*: a file that changed while
|
||||
keeping its size and whole-second mtime was hard-linked stale out of the
|
||||
previous generation, and the single pass had no authoritative pass to repair
|
||||
it. Still one pass where the live path takes two.
|
||||
- Backup: *--hard-restart-projects* is refused alongside *--snapshot*, like
|
||||
*--shutdown* already is. It exists for stacks whose database cannot be backed
|
||||
up hot, which is what a snapshot removes.
|
||||
- Tests: the trailing separator, both teardown behaviours, the new refusal, and
|
||||
*app.main* driving the snapshot branch — the caller that runs in production,
|
||||
which no test had exercised, which is why the layout defect shipped.
|
||||
|
||||
## [3.2.0] - 2026-07-31
|
||||
|
||||
- Backup: *--snapshot {btrfs,zfs}* with *--snapshot-subject* captures every
|
||||
|
||||
27
Makefile
27
Makefile
@@ -1,4 +1,4 @@
|
||||
.PHONY: install build clean \
|
||||
.PHONY: install install-lint build clean lint ruff ruff-fix \
|
||||
test test-unit test-integration test-e2e \
|
||||
test-unit-run test-integration-run test-e2e-run
|
||||
|
||||
@@ -34,13 +34,30 @@ build:
|
||||
clean:
|
||||
git clean -fdX .
|
||||
|
||||
# clean + build run once and in order, then the three suites run concurrently
|
||||
# via -j3; the *-run targets carry no clean/build prereq so the sub-make cannot
|
||||
# race a second clean against build.
|
||||
# Separate from `install` so the test image does not have to carry the linter.
|
||||
install-lint:
|
||||
@$(PY_DEFAULT) -m pip install -q -e ".[lint]"
|
||||
|
||||
# Runs on the host, not in the image, so it also covers what the Dockerfile
|
||||
# does not copy.
|
||||
ruff: install-lint
|
||||
@echo ">> Running ruff over the whole repository"
|
||||
@$(PY_DEFAULT) -m ruff check .
|
||||
@$(PY_DEFAULT) -m ruff format --check .
|
||||
|
||||
ruff-fix: install-lint
|
||||
@$(PY_DEFAULT) -m ruff check --fix .
|
||||
@$(PY_DEFAULT) -m ruff format .
|
||||
|
||||
lint: ruff
|
||||
|
||||
# clean + build run once and in order, then lint and the three suites run
|
||||
# concurrently via -j4; the *-run targets carry no clean/build prereq so the
|
||||
# sub-make cannot race a second clean against build.
|
||||
test:
|
||||
@$(MAKE) clean
|
||||
@$(MAKE) build
|
||||
@$(MAKE) -j3 test-unit-run test-integration-run test-e2e-run
|
||||
@$(MAKE) -j4 lint test-unit-run test-integration-run test-e2e-run
|
||||
|
||||
test-unit: clean build test-unit-run
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "backup-docker-to-local"
|
||||
version = "3.2.0"
|
||||
version = "3.5.0"
|
||||
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
@@ -16,6 +16,11 @@ dependencies = [
|
||||
"dirval",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
# Pinned: a ruff minor bump changes which rules fire, and `make test` gates on
|
||||
# a clean run, so an unpinned lint would fail the suite on an unrelated day.
|
||||
lint = ["ruff==0.16.1"]
|
||||
|
||||
[project.scripts]
|
||||
baudolo = "baudolo.backup.__main__:main"
|
||||
baudolo-restore = "baudolo.restore.__main__:main"
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .app import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -30,17 +30,13 @@ def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
machine_id = get_machine_id()
|
||||
backup_time = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
# Local wall clock on purpose: generations sort by this name, and UTC would
|
||||
# order new ones before the existing ones wherever the offset is positive.
|
||||
backup_time = datetime.now().strftime("%Y%m%d%H%M%S") # noqa: DTZ005
|
||||
|
||||
versions_dir = os.path.join(args.backups_dir, machine_id, args.repo_name)
|
||||
version_dir = create_version_directory(versions_dir, backup_time)
|
||||
|
||||
# IMPORTANT:
|
||||
# - keep_default_na=False prevents empty fields from turning into NaN
|
||||
# - dtype=str keeps all columns stable for comparisons/validation
|
||||
#
|
||||
# Robust behavior:
|
||||
# - if the file is missing or empty, we continue without DB dumps.
|
||||
databases_df = load_databases_df(args.databases_csv)
|
||||
|
||||
print("💾 Start volume backups...", flush=True)
|
||||
@@ -54,6 +50,14 @@ def main() -> int:
|
||||
|
||||
for volume_name in docker_volume_names():
|
||||
print(f"Start backup routine for volume: {volume_name}", flush=True)
|
||||
|
||||
if volume_name in args.volumes_no_backup_required:
|
||||
print(
|
||||
f"Skipping volume '{volume_name}' entirely (declared no-backup).",
|
||||
flush=True,
|
||||
)
|
||||
continue
|
||||
|
||||
containers = containers_using_volume(volume_name)
|
||||
|
||||
if volume_is_fully_ignored(containers, args.images_no_backup_required):
|
||||
@@ -72,30 +76,44 @@ def main() -> int:
|
||||
database_containers=args.database_containers,
|
||||
)
|
||||
|
||||
if args.dump_only_sql:
|
||||
if found_db:
|
||||
if not dumped_any:
|
||||
print(
|
||||
f"WARNING: dump-only-sql requested but no DB dump was produced for DB volume '{volume_name}'. "
|
||||
"Falling back to file backup.",
|
||||
flush=True,
|
||||
)
|
||||
else:
|
||||
continue
|
||||
if args.dump_only_sql and found_db:
|
||||
if not dumped_any:
|
||||
print(
|
||||
f"WARNING: dump-only-sql requested but no DB dump was produced for DB volume '{volume_name}'. "
|
||||
"Falling back to file backup.",
|
||||
flush=True,
|
||||
)
|
||||
else:
|
||||
continue
|
||||
|
||||
live_source = get_storage_path(volume_name)
|
||||
|
||||
def copy(*, authoritative: bool, source: str = live_source) -> None:
|
||||
def copy(
|
||||
*,
|
||||
authoritative: bool,
|
||||
source: str = live_source,
|
||||
volume: str = volume_name,
|
||||
target: str = vol_dir,
|
||||
) -> None:
|
||||
backup_volume(
|
||||
versions_dir,
|
||||
volume_name,
|
||||
vol_dir,
|
||||
volume,
|
||||
target,
|
||||
authoritative=authoritative,
|
||||
source=source,
|
||||
)
|
||||
|
||||
if resolve_source is not None:
|
||||
copy(authoritative=False, source=resolve_source(live_source))
|
||||
snapshot_source = resolve_source(live_source)
|
||||
if os.path.isdir(snapshot_source):
|
||||
copy(authoritative=True, source=snapshot_source)
|
||||
else:
|
||||
print(
|
||||
f"WARNING: volume '{volume_name}' is not in the snapshot "
|
||||
"(created after it was taken); copying it live instead.",
|
||||
flush=True,
|
||||
)
|
||||
copy(authoritative=False)
|
||||
continue
|
||||
|
||||
if args.everything:
|
||||
|
||||
@@ -68,6 +68,13 @@ def parse_args() -> argparse.Namespace:
|
||||
help="Exact image references (repo:tag, incl. any registry prefix) for which no backup should be performed",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--volumes-no-backup-required",
|
||||
nargs="+",
|
||||
default=[],
|
||||
help="Exact volume names that are never backed up, whatever containers use them. For derived trees a restore cannot reproduce, above all a nested docker data root",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--everything",
|
||||
action="store_true",
|
||||
@@ -93,5 +100,12 @@ def parse_args() -> argparse.Namespace:
|
||||
if bool(args.snapshot) != bool(args.snapshot_subject):
|
||||
p.error("--snapshot and --snapshot-subject must be given together")
|
||||
if args.snapshot and args.shutdown:
|
||||
p.error("--shutdown is meaningless with --snapshot: containers are never stopped")
|
||||
p.error(
|
||||
"--shutdown is meaningless with --snapshot: containers are never stopped"
|
||||
)
|
||||
if args.snapshot and args.hard_restart_projects:
|
||||
p.error(
|
||||
"--hard-restart-projects is meaningless with --snapshot: the flag exists "
|
||||
"for stacks whose database cannot be backed up hot, which a snapshot solves"
|
||||
)
|
||||
return args
|
||||
|
||||
@@ -4,10 +4,9 @@ import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
def _build_compose_cmd(project_dir: str, passthrough: List[str]) -> List[str]:
|
||||
def _build_compose_cmd(project_dir: str, passthrough: list[str]) -> list[str]:
|
||||
"""
|
||||
Build the compose command for this project directory.
|
||||
|
||||
@@ -30,7 +29,7 @@ def _build_compose_cmd(project_dir: str, passthrough: List[str]) -> List[str]:
|
||||
raise RuntimeError("Neither 'compose' nor 'docker' found in PATH")
|
||||
|
||||
|
||||
def _find_compose_file(project_dir: str) -> Optional[Path]:
|
||||
def _find_compose_file(project_dir: str) -> Path | None:
|
||||
"""
|
||||
Detect a compose file in `project_dir` (case-insensitive).
|
||||
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import pandas
|
||||
|
||||
@@ -22,7 +21,7 @@ def get_instance(container: str, database_containers: list[str]) -> str:
|
||||
return re.split(r"(_|-)(database|db|postgres)", container)[0]
|
||||
|
||||
|
||||
def _validate_database_value(value: Optional[str], *, instance: str) -> str:
|
||||
def _validate_database_value(value: str | None, *, instance: str) -> str:
|
||||
"""
|
||||
Enforce explicit database semantics:
|
||||
|
||||
@@ -70,7 +69,7 @@ def backup_database(
|
||||
container: str,
|
||||
volume_dir: str,
|
||||
db_type: str,
|
||||
databases_df: "pandas.DataFrame",
|
||||
databases_df: pandas.DataFrame,
|
||||
database_containers: list[str],
|
||||
) -> bool:
|
||||
"""
|
||||
@@ -97,7 +96,6 @@ def backup_database(
|
||||
|
||||
db_value = _validate_database_value(raw_db, instance=instance_name)
|
||||
|
||||
# Explicit: dump ALL databases
|
||||
if db_value == "*":
|
||||
if db_type != "postgres":
|
||||
raise ValueError(
|
||||
@@ -110,7 +108,6 @@ def backup_database(
|
||||
produced = True
|
||||
continue
|
||||
|
||||
# Concrete database dump
|
||||
db_name = db_value
|
||||
dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
|
||||
|
||||
@@ -135,7 +132,6 @@ def backup_database(
|
||||
_atomic_write_cmd(cmd, dump_file)
|
||||
produced = True
|
||||
except BackupException as e:
|
||||
# Explicit DB dump failed -> hard error
|
||||
raise BackupException(
|
||||
f"Postgres dump failed for instance '{instance_name}', "
|
||||
f"database '{db_name}'. This database was explicitly configured "
|
||||
|
||||
@@ -9,9 +9,29 @@ def get_image_info(container: str) -> str:
|
||||
)[0]
|
||||
|
||||
|
||||
def image_name(container: str) -> str:
|
||||
"""The image's repository path, without registry host, tag or digest.
|
||||
|
||||
A swarm node that hosts the local registry puts its own hostname in front
|
||||
of every pull, so the raw reference of a Postgres container can read
|
||||
`svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5`. Matching the
|
||||
whole reference finds "mariadb" there and dumps the database with
|
||||
mariadb-dump, which the Postgres image does not ship (exit 127). Tags bite
|
||||
the same way: `xwiki_custom:lts-postgres-tomcat`.
|
||||
"""
|
||||
reference = get_image_info(container).strip().split("@", 1)[0]
|
||||
head, _, tail = reference.rpartition("/")
|
||||
tail = tail.split(":", 1)[0]
|
||||
if head:
|
||||
registry = head.split("/", 1)[0]
|
||||
if "." in registry or ":" in registry or registry == "localhost":
|
||||
head = head.partition("/")[2]
|
||||
return f"{head}/{tail}" if head else tail
|
||||
|
||||
|
||||
def has_image(container: str, pattern: str) -> bool:
|
||||
"""Return True if container's image contains the pattern."""
|
||||
return pattern in get_image_info(container)
|
||||
"""Return True if the container's image name contains the pattern."""
|
||||
return pattern in image_name(container)
|
||||
|
||||
|
||||
def docker_volume_names() -> list[str]:
|
||||
@@ -78,5 +98,5 @@ def docker_volume_exists(volume: str) -> bool:
|
||||
f"docker volume inspect {volume} >/dev/null 2>&1 && echo OK"
|
||||
)
|
||||
return True
|
||||
except Exception:
|
||||
except BackupException:
|
||||
return False
|
||||
|
||||
@@ -15,7 +15,7 @@ def backup_mariadb_or_postgres(
|
||||
*,
|
||||
container: str,
|
||||
volume_dir: str,
|
||||
databases_df: "pandas.DataFrame",
|
||||
databases_df: pandas.DataFrame,
|
||||
database_containers: list[str],
|
||||
) -> tuple[bool, bool]:
|
||||
"""
|
||||
@@ -34,7 +34,7 @@ def backup_mariadb_or_postgres(
|
||||
return False, False
|
||||
|
||||
|
||||
def _empty_databases_df() -> "pandas.DataFrame":
|
||||
def _empty_databases_df() -> pandas.DataFrame:
|
||||
"""
|
||||
Create an empty DataFrame with the expected schema for databases.csv.
|
||||
|
||||
@@ -44,7 +44,7 @@ def _empty_databases_df() -> "pandas.DataFrame":
|
||||
return pandas.DataFrame(columns=["instance", "database", "username", "password"])
|
||||
|
||||
|
||||
def load_databases_df(csv_path: str) -> "pandas.DataFrame":
|
||||
def load_databases_df(csv_path: str) -> pandas.DataFrame:
|
||||
"""
|
||||
Load databases.csv robustly.
|
||||
|
||||
@@ -74,7 +74,7 @@ def backup_dumps_for_volume(
|
||||
*,
|
||||
containers: list[str],
|
||||
vol_dir: str,
|
||||
databases_df: "pandas.DataFrame",
|
||||
databases_df: pandas.DataFrame,
|
||||
database_containers: list[str],
|
||||
) -> tuple[bool, bool]:
|
||||
"""
|
||||
|
||||
@@ -7,7 +7,7 @@ import pathlib
|
||||
|
||||
from dirval import create_stamp_file
|
||||
|
||||
from .shell import execute_shell_command
|
||||
from .shell import BackupException, execute_shell_command
|
||||
|
||||
|
||||
def get_machine_id() -> str:
|
||||
@@ -23,7 +23,14 @@ def stamp_directory(version_dir: str) -> None:
|
||||
|
||||
def create_version_directory(versions_dir: str, backup_time: str) -> str:
|
||||
version_dir = os.path.join(versions_dir, backup_time)
|
||||
pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
pathlib.Path(version_dir).mkdir(parents=True)
|
||||
except FileExistsError:
|
||||
raise BackupException(
|
||||
f"generation {backup_time} already exists at {version_dir}; "
|
||||
"another run claimed this second - refusing to write into it, "
|
||||
"since rsync --delete would overwrite that generation"
|
||||
) from None
|
||||
return version_dir
|
||||
|
||||
|
||||
|
||||
@@ -17,12 +17,11 @@ import os
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
|
||||
from .shell import execute_shell_command
|
||||
from .shell import BackupException, execute_shell_command
|
||||
|
||||
KINDS = ("btrfs", "zfs")
|
||||
|
||||
|
||||
|
||||
class SnapshotError(RuntimeError):
|
||||
"""A snapshot could not be created, resolved or removed."""
|
||||
|
||||
@@ -32,13 +31,20 @@ def _resolver(subject: str, root: str) -> Callable[[str], str]:
|
||||
relative = os.path.relpath(os.path.abspath(path), os.path.abspath(subject))
|
||||
if relative.startswith(".."):
|
||||
raise SnapshotError(f"{path} lies outside the snapshot subject {subject}")
|
||||
return os.path.join(root, relative) if relative != "." else root
|
||||
resolved = root if relative == "." else os.path.join(root, relative)
|
||||
|
||||
# abspath drops a trailing separator, and rsync reads "dir/" as its
|
||||
# contents where "dir" means the directory itself.
|
||||
return resolved + os.sep if path.endswith(os.sep) else resolved
|
||||
|
||||
return resolve
|
||||
|
||||
|
||||
def _btrfs(subject: str, name: str, run: Callable[[str], list[str]]) -> tuple[str, str]:
|
||||
target = os.path.join(os.path.dirname(os.path.abspath(subject)), f".{name}")
|
||||
# The snapshot goes inside the subject, never beside it: the kernel rejects
|
||||
# a snapshot whose destination is on another filesystem, which is exactly
|
||||
# what the parent directory is when the subject is a mountpoint of its own.
|
||||
target = os.path.join(os.path.abspath(subject), f".{name}")
|
||||
run(f"btrfs subvolume snapshot -r {subject} {target}")
|
||||
return target, f"btrfs subvolume delete {target}"
|
||||
|
||||
@@ -74,6 +80,8 @@ def volume_snapshot(
|
||||
|
||||
Raises:
|
||||
SnapshotError: the kind is unknown, or the snapshot cannot be created.
|
||||
Removal failure is reported, not raised: a leftover snapshot is a
|
||||
cleanup problem and must not discard a generation that is complete.
|
||||
"""
|
||||
create = _CREATE.get(kind)
|
||||
if create is None:
|
||||
@@ -83,4 +91,8 @@ def volume_snapshot(
|
||||
try:
|
||||
yield _resolver(subject, root)
|
||||
finally:
|
||||
run(remove)
|
||||
try:
|
||||
run(remove)
|
||||
except BackupException as error:
|
||||
# Raising here would also mask whatever the body raised.
|
||||
print(f"WARNING: {root} could not be removed: {error}", flush=True)
|
||||
|
||||
@@ -49,7 +49,10 @@ def backup_volume(
|
||||
link_dest = f"--link-dest='{last}'" if last else ""
|
||||
verify = "--checksum " if authoritative else ""
|
||||
|
||||
cmd = f"rsync -abP --delete --delete-excluded {verify}{link_dest} {source} {dest}"
|
||||
cmd = (
|
||||
f"rsync -aP --no-D --delete --delete-excluded "
|
||||
f"{verify}{link_dest} {source} {dest}"
|
||||
)
|
||||
|
||||
try:
|
||||
execute_shell_command(cmd)
|
||||
|
||||
@@ -3,10 +3,11 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
from .paths import BackupPaths
|
||||
from .files import restore_volume_files
|
||||
from .db.postgres import restore_postgres_sql
|
||||
from .db.cluster import restore_cluster_sql
|
||||
from .db.mariadb import restore_mariadb_sql
|
||||
from .db.postgres import restore_postgres_sql
|
||||
from .files import restore_volume_files
|
||||
from .paths import BackupPaths
|
||||
|
||||
|
||||
def _add_common_backup_args(p: argparse.ArgumentParser) -> None:
|
||||
@@ -33,9 +34,6 @@ def main(argv: list[str] | None = None) -> int:
|
||||
)
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# files
|
||||
# ------------------------------------------------------------------
|
||||
p_files = sub.add_parser("files", help="Restore files into a docker volume")
|
||||
_add_common_backup_args(p_files)
|
||||
p_files.add_argument(
|
||||
@@ -48,9 +46,6 @@ def main(argv: list[str] | None = None) -> int:
|
||||
),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# postgres
|
||||
# ------------------------------------------------------------------
|
||||
p_pg = sub.add_parser("postgres", help="Restore a single PostgreSQL database dump")
|
||||
_add_common_backup_args(p_pg)
|
||||
p_pg.add_argument("--container", required=True)
|
||||
@@ -59,9 +54,24 @@ def main(argv: list[str] | None = None) -> int:
|
||||
p_pg.add_argument("--db-password", required=True)
|
||||
p_pg.add_argument("--empty", action="store_true")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# mariadb
|
||||
# ------------------------------------------------------------------
|
||||
p_cluster = sub.add_parser(
|
||||
"cluster", help="Restore a full PostgreSQL cluster dump (pg_dumpall)"
|
||||
)
|
||||
_add_common_backup_args(p_cluster)
|
||||
p_cluster.add_argument("--container", required=True)
|
||||
p_cluster.add_argument(
|
||||
"--instance",
|
||||
required=True,
|
||||
help="Instance the dump was taken from; names <instance>.cluster.backup.sql",
|
||||
)
|
||||
p_cluster.add_argument(
|
||||
"--db-user",
|
||||
required=True,
|
||||
help="Superuser of the instance; the dump creates roles and databases",
|
||||
)
|
||||
p_cluster.add_argument("--db-password", required=True)
|
||||
p_cluster.add_argument("--empty", action="store_true")
|
||||
|
||||
p_mdb = sub.add_parser(
|
||||
"mariadb", help="Restore a single MariaDB/MySQL-compatible dump"
|
||||
)
|
||||
@@ -76,8 +86,6 @@ def main(argv: list[str] | None = None) -> int:
|
||||
|
||||
try:
|
||||
if args.cmd == "files":
|
||||
# target volume = args.volume_name
|
||||
# source volume (backup key) defaults to target volume
|
||||
source_volume = args.source_volume or args.volume_name
|
||||
|
||||
bp_files = BackupPaths(
|
||||
@@ -111,6 +119,22 @@ def main(argv: list[str] | None = None) -> int:
|
||||
)
|
||||
return 0
|
||||
|
||||
if args.cmd == "cluster":
|
||||
restore_cluster_sql(
|
||||
container=args.container,
|
||||
user=args.db_user,
|
||||
password=args.db_password,
|
||||
sql_path=BackupPaths(
|
||||
args.volume_name,
|
||||
args.backup_hash,
|
||||
args.version,
|
||||
repo_name=args.repo_name,
|
||||
backups_dir=args.backups_dir,
|
||||
).cluster_file(args.instance),
|
||||
empty=args.empty,
|
||||
)
|
||||
return 0
|
||||
|
||||
if args.cmd == "mariadb":
|
||||
user = args.db_user or args.db_name
|
||||
restore_mariadb_sql(
|
||||
@@ -132,7 +156,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
parser.error("Unhandled command")
|
||||
return 2
|
||||
|
||||
except Exception as e:
|
||||
except Exception as e: # noqa: BLE001 - CLI boundary: any failure becomes exit 1
|
||||
print(f"ERROR: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
103
src/baudolo/restore/db/cluster.py
Normal file
103
src/baudolo/restore/db/cluster.py
Normal file
@@ -0,0 +1,103 @@
|
||||
"""Replay a full PostgreSQL cluster dump produced by ``pg_dumpall``.
|
||||
|
||||
The backup side writes one when a databases.csv row asks for every database of
|
||||
an instance (``database = '*'``, see ``backup/db.py``). Until now nothing read
|
||||
it back, so that dump was stored and unrestorable - a format whose producer has
|
||||
no consumer.
|
||||
|
||||
A cluster stream differs from a single-database one in three ways that decide
|
||||
the implementation:
|
||||
|
||||
* it recreates roles and databases, so it must be replayed against the control
|
||||
database rather than into a target database;
|
||||
* ``CREATE DATABASE`` cannot run inside a transaction block, so unlike
|
||||
:mod:`baudolo.restore.db.postgres` the replay must not be wrapped in
|
||||
``--single-transaction``;
|
||||
* it is replayed as a superuser, so the superuser-only statements that the
|
||||
single-database path filters out are exactly the ones that have to survive.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from collections.abc import Iterable, Iterator
|
||||
|
||||
from ..run import docker_exec
|
||||
|
||||
CONTROL_DB = "postgres"
|
||||
_CLUSTER_PRECLEAN_SQL = os.path.join(os.path.dirname(__file__), "cluster_preclean.sql")
|
||||
_CREATE_ROLE = re.compile(rb'^CREATE ROLE "?([^";]+)"?;\s*$')
|
||||
|
||||
|
||||
def _psql(user: str) -> list[str]:
|
||||
"""The replay client: no --single-transaction, CREATE DATABASE forbids it."""
|
||||
return ["psql", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", CONTROL_DB]
|
||||
|
||||
|
||||
def filter_own_role_creation(lines: Iterable[bytes], user: str) -> Iterator[bytes]:
|
||||
"""Drop the ``CREATE ROLE`` of the role holding this session.
|
||||
|
||||
A pg_dumpall stream recreates every role of the cluster, the bootstrap
|
||||
superuser included, and the pre-clean cannot drop the one it is connected
|
||||
as - so that single statement always collides. Its ``ALTER ROLE`` is kept:
|
||||
that is what re-applies the attributes and the password the dump captured.
|
||||
|
||||
Args:
|
||||
lines: dump lines including their trailing newlines.
|
||||
user: the connecting role.
|
||||
|
||||
Yields:
|
||||
Every line except that one CREATE.
|
||||
"""
|
||||
for line in lines:
|
||||
found = _CREATE_ROLE.match(line)
|
||||
if found and found.group(1).decode() == user:
|
||||
continue
|
||||
yield line
|
||||
|
||||
|
||||
def restore_cluster_sql(
|
||||
*,
|
||||
container: str,
|
||||
user: str,
|
||||
password: str,
|
||||
sql_path: str,
|
||||
empty: bool,
|
||||
) -> None:
|
||||
"""Replay a pg_dumpall stream into a running instance.
|
||||
|
||||
Args:
|
||||
container: the running engine to replay into.
|
||||
user: a superuser of that instance; the dump creates roles and
|
||||
databases, which an application role may not do.
|
||||
password: its password, handed to psql through the container's env.
|
||||
sql_path: the ``<instance>.cluster.backup.sql`` of a generation.
|
||||
empty: drop the cluster's databases and roles first. Without it the
|
||||
replay stops at the first object that already exists, which is the
|
||||
honest outcome: recreating a cluster over a populated one is a
|
||||
decision, not a default.
|
||||
"""
|
||||
if not os.path.isfile(sql_path):
|
||||
raise FileNotFoundError(sql_path)
|
||||
|
||||
docker_env = {"PGPASSWORD": password}
|
||||
|
||||
if empty:
|
||||
with open(_CLUSTER_PRECLEAN_SQL, encoding="utf-8") as preclean:
|
||||
drop_sql = preclean.read()
|
||||
docker_exec(
|
||||
container,
|
||||
_psql(user),
|
||||
stdin=drop_sql.encode(),
|
||||
docker_env=docker_env,
|
||||
)
|
||||
|
||||
with open(sql_path, "rb") as src, tempfile.TemporaryFile() as filtered:
|
||||
for line in filter_own_role_creation(src, user):
|
||||
filtered.write(line)
|
||||
filtered.seek(0)
|
||||
docker_exec(container, _psql(user), stdin=filtered, docker_env=docker_env)
|
||||
|
||||
print(f"PostgreSQL cluster restore complete from '{os.path.basename(sql_path)}'.")
|
||||
30
src/baudolo/restore/db/cluster_preclean.sql
Normal file
30
src/baudolo/restore/db/cluster_preclean.sql
Normal file
@@ -0,0 +1,30 @@
|
||||
-- Pre-clean for `restore cluster --empty`. A pg_dumpall stream recreates roles
|
||||
-- and databases, so replaying it into a populated cluster dies on the first
|
||||
-- CREATE ROLE. Emitted as one DROP per row and run via \gexec so each executes
|
||||
-- as its own top-level statement: DROP DATABASE cannot run inside a
|
||||
-- transaction block, which rules out a single DO block.
|
||||
-- The phase column pins the order: databases must be gone before their owners
|
||||
-- can be dropped, and DROP OWNED BY releases what a role still holds in the
|
||||
-- control database. Template databases, the control database itself, the pg_*
|
||||
-- system roles and the connecting role are kept - the dump does not recreate
|
||||
-- them and dropping them would end the session.
|
||||
SELECT statement
|
||||
FROM (
|
||||
SELECT 1 AS phase,
|
||||
format('DROP DATABASE IF EXISTS %I', datname) AS statement
|
||||
FROM pg_database
|
||||
WHERE NOT datistemplate
|
||||
AND datname <> current_database()
|
||||
UNION ALL
|
||||
SELECT 2, format('DROP OWNED BY %I', rolname)
|
||||
FROM pg_roles
|
||||
WHERE NOT starts_with(rolname, 'pg_')
|
||||
AND rolname <> current_user
|
||||
UNION ALL
|
||||
SELECT 3, format('DROP ROLE IF EXISTS %I', rolname)
|
||||
FROM pg_roles
|
||||
WHERE NOT starts_with(rolname, 'pg_')
|
||||
AND rolname <> current_user
|
||||
) drops
|
||||
ORDER BY phase
|
||||
\gexec
|
||||
@@ -22,11 +22,11 @@ exit 42
|
||||
if not out:
|
||||
raise RuntimeError("empty client detection output")
|
||||
return out
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
print(
|
||||
"ERROR: neither 'mariadb' nor 'mysql' found in container.", file=sys.stderr
|
||||
)
|
||||
raise e
|
||||
raise
|
||||
|
||||
|
||||
def restore_mariadb_sql(
|
||||
@@ -44,9 +44,7 @@ def restore_mariadb_sql(
|
||||
raise FileNotFoundError(sql_path)
|
||||
|
||||
if empty:
|
||||
# IMPORTANT:
|
||||
# Do NOT hardcode 'mysql' here. Use the detected client.
|
||||
# MariaDB 11 images may not contain the mysql binary at all.
|
||||
# Do not hardcode 'mysql': MariaDB 11 images may not ship that binary.
|
||||
result = docker_exec(
|
||||
container,
|
||||
[
|
||||
|
||||
@@ -50,7 +50,6 @@ def restore_postgres_sql(
|
||||
if not os.path.isfile(sql_path):
|
||||
raise FileNotFoundError(sql_path)
|
||||
|
||||
# Make password available INSIDE the container for psql.
|
||||
docker_env = {"PGPASSWORD": password}
|
||||
|
||||
if empty:
|
||||
|
||||
@@ -27,3 +27,7 @@ class BackupPaths:
|
||||
|
||||
def sql_file(self, db_name: str) -> str:
|
||||
return os.path.join(self.root(), "sql", f"{db_name}.backup.sql")
|
||||
|
||||
def cluster_file(self, instance: str) -> str:
|
||||
"""The pg_dumpall stream a `database = '*'` row produces."""
|
||||
return os.path.join(self.root(), "sql", f"{instance}.cluster.backup.sql")
|
||||
|
||||
@@ -2,7 +2,6 @@ from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def run(
|
||||
@@ -10,7 +9,7 @@ def run(
|
||||
*,
|
||||
stdin=None,
|
||||
capture: bool = False,
|
||||
env: Optional[dict] = None,
|
||||
env: dict | None = None,
|
||||
) -> subprocess.CompletedProcess:
|
||||
try:
|
||||
kwargs: dict = {
|
||||
@@ -26,21 +25,18 @@ def run(
|
||||
else:
|
||||
kwargs["stdin"] = stdin
|
||||
|
||||
return subprocess.run(cmd, **kwargs)
|
||||
return subprocess.run(cmd, **kwargs) # noqa: PLW1510 - check lives in kwargs
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
msg = f"ERROR: command failed ({e.returncode}): {' '.join(cmd)}"
|
||||
print(msg, file=sys.stderr)
|
||||
if e.stdout:
|
||||
for stream in (e.stdout, e.stderr):
|
||||
if not stream:
|
||||
continue
|
||||
try:
|
||||
print(e.stdout.decode(), file=sys.stderr)
|
||||
except Exception:
|
||||
print(e.stdout, file=sys.stderr)
|
||||
if e.stderr:
|
||||
try:
|
||||
print(e.stderr.decode(), file=sys.stderr)
|
||||
except Exception:
|
||||
print(e.stderr, file=sys.stderr)
|
||||
print(stream.decode(), file=sys.stderr)
|
||||
except (UnicodeDecodeError, AttributeError):
|
||||
print(stream, file=sys.stderr)
|
||||
raise
|
||||
|
||||
|
||||
@@ -50,8 +46,8 @@ def docker_exec(
|
||||
*,
|
||||
stdin=None,
|
||||
capture: bool = False,
|
||||
env: Optional[dict] = None,
|
||||
docker_env: Optional[dict[str, str]] = None,
|
||||
env: dict | None = None,
|
||||
docker_env: dict[str, str] | None = None,
|
||||
) -> subprocess.CompletedProcess:
|
||||
cmd: list[str] = ["docker", "exec", "-i"]
|
||||
if docker_env:
|
||||
@@ -67,8 +63,8 @@ def docker_exec_sh(
|
||||
*,
|
||||
stdin=None,
|
||||
capture: bool = False,
|
||||
env: Optional[dict] = None,
|
||||
docker_env: Optional[dict[str, str]] = None,
|
||||
env: dict | None = None,
|
||||
docker_env: dict[str, str] | None = None,
|
||||
) -> subprocess.CompletedProcess:
|
||||
return docker_exec(
|
||||
container,
|
||||
@@ -85,5 +81,6 @@ def docker_volume_exists(volume: str) -> bool:
|
||||
["docker", "volume", "inspect", volume],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=False,
|
||||
)
|
||||
return p.returncode == 0
|
||||
|
||||
@@ -1,18 +1,17 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
import pandas as pd
|
||||
from typing import Optional
|
||||
from pandas.errors import EmptyDataError
|
||||
|
||||
DB_NAME_RE = re.compile(r"^[a-zA-Z0-9_][a-zA-Z0-9_-]*$")
|
||||
|
||||
|
||||
def _validate_database_value(value: Optional[str], *, instance: str) -> str:
|
||||
def _validate_database_value(value: str | None, *, instance: str) -> str:
|
||||
v = (value or "").strip()
|
||||
if v == "":
|
||||
raise ValueError(
|
||||
@@ -40,7 +39,7 @@ def _empty_df() -> pd.DataFrame:
|
||||
def check_and_add_entry(
|
||||
file_path: str,
|
||||
instance: str,
|
||||
database: Optional[str],
|
||||
database: str | None,
|
||||
username: str,
|
||||
password: str,
|
||||
) -> None:
|
||||
@@ -108,7 +107,7 @@ def main() -> None:
|
||||
username=args.username,
|
||||
password=args.password,
|
||||
)
|
||||
except Exception as exc:
|
||||
except Exception as exc: # noqa: BLE001 - CLI boundary: any failure becomes exit 1
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Shared e2e helpers, re-exported so tests import one name."""
|
||||
|
||||
from .fixtures import * # noqa: F401,F403
|
||||
from .process import * # noqa: F401,F403
|
||||
from .fixtures import *
|
||||
from .process import *
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Fixtures and paths the e2e suite builds its scenarios from."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
@@ -96,8 +97,7 @@ def write_databases_csv(path: str, rows: list[tuple[str, str, str, str]]) -> Non
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write("instance;database;username;password\n")
|
||||
for inst, db, user, pw in rows:
|
||||
f.write(f"{inst};{db};{user};{pw}\n")
|
||||
f.writelines(f"{inst};{db};{user};{pw}\n" for inst, db, user, pw in rows)
|
||||
|
||||
|
||||
def cleanup_docker(*, containers: list[str], volumes: list[str]) -> None:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Process, docker and readiness helpers for the e2e suite."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
|
||||
@@ -12,8 +12,8 @@ import sys
|
||||
|
||||
sys.path.insert(0, "/src")
|
||||
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot # noqa: E402
|
||||
from baudolo.backup.volume import backup_volume # noqa: E402
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
||||
from baudolo.backup.volume import backup_volume
|
||||
|
||||
SUBJECT = "/subject/docker"
|
||||
VOLUME = "mariadb_data"
|
||||
@@ -23,9 +23,13 @@ GENERATION = f"{VERSIONS}/20260731"
|
||||
|
||||
|
||||
def shell(command: str) -> list[str]:
|
||||
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
||||
proc = subprocess.run(
|
||||
command, shell=True, capture_output=True, text=True, check=False
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise SnapshotError(f"{command} exited {proc.returncode}: {proc.stderr.strip()}")
|
||||
raise SnapshotError(
|
||||
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||
)
|
||||
return proc.stdout.splitlines()
|
||||
|
||||
|
||||
@@ -34,8 +38,8 @@ with volume_snapshot("btrfs", SUBJECT, "dbtest", run=shell) as resolve:
|
||||
VERSIONS,
|
||||
VOLUME,
|
||||
f"{GENERATION}/{VOLUME}",
|
||||
authoritative=False,
|
||||
source=resolve(DATADIR) + "/",
|
||||
authoritative=True,
|
||||
source=resolve(f"{DATADIR}/"),
|
||||
)
|
||||
|
||||
print("SNAPSHOT COPY DONE", flush=True)
|
||||
|
||||
@@ -12,7 +12,7 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, "/src")
|
||||
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot # noqa: E402
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
||||
|
||||
KIND = sys.argv[1]
|
||||
SUBJECT = sys.argv[2]
|
||||
@@ -20,9 +20,13 @@ EXPECT = sys.argv[3]
|
||||
|
||||
|
||||
def shell(command: str) -> list[str]:
|
||||
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
||||
proc = subprocess.run(
|
||||
command, shell=True, capture_output=True, text=True, check=False
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise SnapshotError(f"{command} exited {proc.returncode}: {proc.stderr.strip()}")
|
||||
raise SnapshotError(
|
||||
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||
)
|
||||
return proc.stdout.splitlines()
|
||||
|
||||
|
||||
@@ -55,5 +59,8 @@ with volume_snapshot(KIND, SUBJECT, "e2e", run=shell) as resolve:
|
||||
|
||||
root = Path(resolve(SUBJECT))
|
||||
|
||||
check("the snapshot is removed afterwards", not root.exists() or not (root / "volumes").exists())
|
||||
check(
|
||||
"the snapshot is removed afterwards",
|
||||
not root.exists() or not (root / "volumes").exists(),
|
||||
)
|
||||
print("ALL OK", flush=True)
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
# tests/e2e/test_e2e_dump_only_fallback_to_files.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
@@ -12,8 +11,8 @@ from .helpers import (
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
@@ -37,7 +36,6 @@ class TestE2EDumpOnlyFallbackToFiles(unittest.TestCase):
|
||||
|
||||
run(["docker", "volume", "create", cls.pg_volume])
|
||||
|
||||
# Start Postgres (creates a real DB volume)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
@@ -35,7 +35,6 @@ class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
|
||||
cls.containers: list[str] = []
|
||||
cls.volumes = [cls.db_volume, cls.files_volume]
|
||||
|
||||
# Create volumes
|
||||
run(["docker", "volume", "create", cls.db_volume])
|
||||
run(["docker", "volume", "create", cls.files_volume])
|
||||
|
||||
@@ -114,7 +113,6 @@ class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
|
||||
[(cls.pg_container, cls.pg_db, cls.pg_user, cls.pg_password)],
|
||||
)
|
||||
|
||||
# Run baudolo with dump-only-sql
|
||||
cmd = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
|
||||
105
tests/e2e/test_e2e_engine_detection_registry_prefix.py
Normal file
105
tests/e2e/test_e2e_engine_detection_registry_prefix.py
Normal file
@@ -0,0 +1,105 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
REGISTRY_HOST = "svc-db-mariadb-swarm-mgr-01:5000"
|
||||
|
||||
|
||||
class TestE2EEngineDetectionRegistryPrefix(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-registry-prefix")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.image = f"{REGISTRY_HOST}/postgres_custom:17-3.5"
|
||||
cls.pg_container = f"{cls.prefix}-pg"
|
||||
cls.pg_volume = f"{cls.prefix}-pg-vol"
|
||||
cls.containers = [cls.pg_container]
|
||||
cls.volumes = [cls.pg_volume]
|
||||
|
||||
run(["docker", "pull", POSTGRES_IMAGE])
|
||||
run(["docker", "tag", POSTGRES_IMAGE, cls.image])
|
||||
run(["docker", "volume", "create", cls.pg_volume])
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.pg_container,
|
||||
"-e",
|
||||
"POSTGRES_PASSWORD=pgpw",
|
||||
"-e",
|
||||
"POSTGRES_DB=appdb",
|
||||
"-e",
|
||||
"POSTGRES_USER=postgres",
|
||||
"-v",
|
||||
f"{cls.pg_volume}:{POSTGRES_DATA_DIR}",
|
||||
cls.image,
|
||||
]
|
||||
)
|
||||
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.pg_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
"psql -U postgres -d appdb -c \"CREATE TABLE t (id int primary key, v text); INSERT INTO t VALUES (1,'ok');\"",
|
||||
]
|
||||
)
|
||||
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(
|
||||
cls.databases_csv, [(cls.pg_container, "appdb", "postgres", "pgpw")]
|
||||
)
|
||||
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
compose_dir=cls.compose_dir,
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=[cls.pg_container],
|
||||
images_no_stop_required=[cls.image],
|
||||
)
|
||||
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
run(["docker", "rmi", cls.image], check=False)
|
||||
|
||||
def test_the_registry_host_does_not_pick_the_engine(self) -> None:
|
||||
p = (
|
||||
backup_path(self.backups_dir, self.repo_name, self.version, self.pg_volume)
|
||||
/ "sql"
|
||||
/ "appdb.backup.sql"
|
||||
)
|
||||
self.assertTrue(p.is_file(), f"Expected a pg_dump at: {p}")
|
||||
self.assertIn("Dumped by pg_dump", p.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,16 +1,16 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
backup_run,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
)
|
||||
|
||||
|
||||
@@ -30,7 +30,6 @@ class TestE2EFilesFull(unittest.TestCase):
|
||||
cls.containers = []
|
||||
cls.volumes = [cls.volume_src, cls.volume_dst]
|
||||
|
||||
# create source volume with a file
|
||||
run(["docker", "volume", "create", cls.volume_src])
|
||||
run(
|
||||
[
|
||||
@@ -50,7 +49,6 @@ class TestE2EFilesFull(unittest.TestCase):
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(cls.databases_csv, [])
|
||||
|
||||
# Run backup (files should be copied)
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
@@ -97,7 +95,6 @@ class TestE2EFilesFull(unittest.TestCase):
|
||||
]
|
||||
)
|
||||
|
||||
# verify restored file exists in dst volume
|
||||
p = run(
|
||||
[
|
||||
"docker",
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
backup_run,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
)
|
||||
|
||||
|
||||
@@ -29,7 +29,6 @@ class TestE2EFilesNoCopy(unittest.TestCase):
|
||||
cls.containers: list[str] = []
|
||||
cls.volumes = [cls.volume_src]
|
||||
|
||||
# Create source volume and write a marker file
|
||||
run(["docker", "volume", "create", cls.volume_src])
|
||||
run(
|
||||
[
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
# tests/e2e/test_e2e_images_no_backup_required_early_skip.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
@@ -34,11 +33,9 @@ class TestE2EImagesNoBackupRequiredEarlySkip(unittest.TestCase):
|
||||
cls.containers = [cls.redis_container]
|
||||
cls.volumes = [cls.ignored_volume, cls.normal_volume]
|
||||
|
||||
# Create volumes
|
||||
run(["docker", "volume", "create", cls.ignored_volume])
|
||||
run(["docker", "volume", "create", cls.normal_volume])
|
||||
|
||||
# Start redis container using the ignored volume
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
@@ -71,7 +68,6 @@ class TestE2EImagesNoBackupRequiredEarlySkip(unittest.TestCase):
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(cls.databases_csv, [])
|
||||
|
||||
# Run baudolo with images-no-backup-required redis
|
||||
cmd = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
|
||||
@@ -30,8 +30,8 @@ import pandas
|
||||
from baudolo.backup import db as db_mod
|
||||
|
||||
from .helpers import (
|
||||
MARIADB_IMAGE,
|
||||
MARIADB_DATA_DIR,
|
||||
MARIADB_IMAGE,
|
||||
cleanup_docker,
|
||||
require_docker,
|
||||
run,
|
||||
|
||||
@@ -1,21 +1,20 @@
|
||||
# tests/e2e/test_e2e_mariadb_full.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
MARIADB_IMAGE,
|
||||
MARIADB_DATA_DIR,
|
||||
backup_run,
|
||||
MARIADB_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
unique,
|
||||
wait_for_mariadb,
|
||||
wait_for_mariadb_sql,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
@@ -71,7 +70,6 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
||||
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
||||
)
|
||||
|
||||
# Create table + data via the dedicated user (TCP)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
@@ -79,9 +77,11 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\""
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -112,8 +112,10 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "DROP TABLE {cls.db_name}.t;"',
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "DROP TABLE {cls.db_name}.t;"'
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -161,8 +163,10 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
||||
self.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"',
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"'
|
||||
),
|
||||
]
|
||||
)
|
||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||
|
||||
@@ -1,21 +1,20 @@
|
||||
# tests/e2e/test_e2e_mariadb_no_copy.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
MARIADB_IMAGE,
|
||||
MARIADB_DATA_DIR,
|
||||
backup_run,
|
||||
MARIADB_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
unique,
|
||||
wait_for_mariadb,
|
||||
wait_for_mariadb_sql,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
@@ -69,7 +68,6 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
||||
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
||||
)
|
||||
|
||||
# Create table + data (TCP)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
@@ -77,9 +75,11 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\""
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -110,8 +110,10 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "DROP TABLE {cls.db_name}.t;"',
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "DROP TABLE {cls.db_name}.t;"'
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -158,8 +160,10 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
||||
self.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"',
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"'
|
||||
),
|
||||
]
|
||||
)
|
||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||
|
||||
177
tests/e2e/test_e2e_postgres_cluster_restore.py
Normal file
177
tests/e2e/test_e2e_postgres_cluster_restore.py
Normal file
@@ -0,0 +1,177 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
# One statement per entry: psql wraps a multi-statement -c in a transaction,
|
||||
# and CREATE DATABASE is forbidden inside one.
|
||||
SEED_SQL = (
|
||||
"CREATE ROLE app LOGIN PASSWORD 'apppw'",
|
||||
"CREATE DATABASE first OWNER app",
|
||||
"CREATE DATABASE second OWNER app",
|
||||
)
|
||||
DROP_SQL = (
|
||||
"DROP DATABASE first",
|
||||
"DROP DATABASE second",
|
||||
"DROP ROLE app",
|
||||
)
|
||||
FIRST_SQL = "CREATE TABLE t (v text); INSERT INTO t VALUES ('first-payload');"
|
||||
SECOND_SQL = "CREATE TABLE t (v text); INSERT INTO t VALUES ('second-payload');"
|
||||
|
||||
|
||||
class TestE2EPostgresClusterRestore(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-pg-cluster")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.pg_container = f"{cls.prefix}-pg"
|
||||
cls.pg_volume = f"{cls.prefix}-pg-vol"
|
||||
cls.containers = [cls.pg_container]
|
||||
cls.volumes = [cls.pg_volume]
|
||||
|
||||
run(["docker", "volume", "create", cls.pg_volume])
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.pg_container,
|
||||
"-e",
|
||||
"POSTGRES_PASSWORD=pgpw",
|
||||
"-v",
|
||||
f"{cls.pg_volume}:{POSTGRES_DATA_DIR}",
|
||||
POSTGRES_IMAGE,
|
||||
]
|
||||
)
|
||||
wait_for_postgres(cls.pg_container, user="postgres")
|
||||
|
||||
for statement in SEED_SQL:
|
||||
cls._psql("postgres", statement)
|
||||
cls._psql("first", FIRST_SQL)
|
||||
cls._psql("second", SECOND_SQL)
|
||||
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(
|
||||
cls.databases_csv, [(cls.pg_container, "*", "postgres", "pgpw")]
|
||||
)
|
||||
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
compose_dir=cls.compose_dir,
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=[cls.pg_container],
|
||||
images_no_stop_required=[POSTGRES_IMAGE],
|
||||
)
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
cls.dump = (
|
||||
backup_path(cls.backups_dir, cls.repo_name, cls.version, cls.pg_volume)
|
||||
/ "sql"
|
||||
/ f"{cls.pg_container}.cluster.backup.sql"
|
||||
)
|
||||
|
||||
for statement in DROP_SQL:
|
||||
cls._psql("postgres", statement)
|
||||
|
||||
run(
|
||||
[
|
||||
"baudolo-restore",
|
||||
"cluster",
|
||||
cls.pg_volume,
|
||||
cls.hash,
|
||||
cls.version,
|
||||
"--backups-dir",
|
||||
cls.backups_dir,
|
||||
"--repo-name",
|
||||
cls.repo_name,
|
||||
"--container",
|
||||
cls.pg_container,
|
||||
"--instance",
|
||||
cls.pg_container,
|
||||
"--db-user",
|
||||
"postgres",
|
||||
"--db-password",
|
||||
"pgpw",
|
||||
"--empty",
|
||||
]
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
|
||||
@classmethod
|
||||
def _psql(cls, database: str, sql: str) -> str:
|
||||
p = run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.pg_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f'psql -U postgres -d {database} -t -A -c "{sql}"',
|
||||
]
|
||||
)
|
||||
return (p.stdout or "").strip()
|
||||
|
||||
def test_the_backup_wrote_a_cluster_dump(self) -> None:
|
||||
self.assertTrue(self.dump.is_file(), f"no cluster dump at {self.dump}")
|
||||
|
||||
def test_both_databases_are_back(self) -> None:
|
||||
listed = self._psql(
|
||||
"postgres",
|
||||
"SELECT datname FROM pg_database WHERE datname IN ('first','second') ORDER BY 1",
|
||||
)
|
||||
self.assertEqual(listed.split(), ["first", "second"])
|
||||
|
||||
def test_each_database_carries_its_own_payload(self) -> None:
|
||||
self.assertEqual(self._psql("first", "SELECT v FROM t"), "first-payload")
|
||||
self.assertEqual(self._psql("second", "SELECT v FROM t"), "second-payload")
|
||||
|
||||
def test_the_superusers_own_create_was_filtered(self) -> None:
|
||||
self.assertEqual(
|
||||
self._psql(
|
||||
"postgres", "SELECT rolsuper FROM pg_roles WHERE rolname = 'postgres'"
|
||||
),
|
||||
"t",
|
||||
)
|
||||
|
||||
def test_the_owning_role_is_back(self) -> None:
|
||||
self.assertEqual(
|
||||
self._psql(
|
||||
"postgres", "SELECT rolname FROM pg_roles WHERE rolname = 'app'"
|
||||
),
|
||||
"app",
|
||||
)
|
||||
|
||||
def test_ownership_survived(self) -> None:
|
||||
self.assertEqual(
|
||||
self._psql(
|
||||
"postgres",
|
||||
"SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = 'first'",
|
||||
),
|
||||
"app",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,9 +1,8 @@
|
||||
# tests/e2e/test_e2e_postgres_empty_drop_hard.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
|
||||
@@ -1,20 +1,19 @@
|
||||
# tests/e2e/test_e2e_postgres_full.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
backup_run,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
@@ -55,7 +54,6 @@ class TestE2EPostgresFull(unittest.TestCase):
|
||||
)
|
||||
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||
|
||||
# Create a table + data
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
|
||||
@@ -1,20 +1,19 @@
|
||||
# tests/e2e/test_e2e_postgres_no_copy.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
backup_run,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
# tests/e2e/test_e2e_postgres_single_transaction_live_writer.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
|
||||
@@ -29,7 +29,7 @@ LOOP_FS = {
|
||||
"ext4": "mkdir -p /subject/docker",
|
||||
}
|
||||
# A container carries no /lib/modules, so modprobe fails even on a loaded module.
|
||||
ZFS_READY = '{ [ -c /dev/zfs ] || modprobe zfs 2>/dev/null; }; [ -c /dev/zfs ]'
|
||||
ZFS_READY = "{ [ -c /dev/zfs ] || modprobe zfs 2>/dev/null; }; [ -c /dev/zfs ]"
|
||||
|
||||
|
||||
def mount_script(fstype: str) -> str:
|
||||
@@ -51,7 +51,13 @@ def zfs_usable() -> bool:
|
||||
"""Whether this host's kernel can serve zfs to a privileged container."""
|
||||
proc = run(
|
||||
[
|
||||
"docker", "run", "--rm", "--privileged", IMAGE, "sh", "-lc",
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--privileged",
|
||||
IMAGE,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"apk add -q zfs >/dev/null 2>&1 && {ZFS_READY}",
|
||||
],
|
||||
capture=True,
|
||||
@@ -87,11 +93,20 @@ def drive(fstype: str, kind: str, expect: str) -> str:
|
||||
try:
|
||||
proc = run(
|
||||
[
|
||||
"docker", "run", "--rm", "--privileged",
|
||||
"--name", staged.name,
|
||||
"-v", f"{staged / 'src'}:/src:ro",
|
||||
"-v", f"{staged / 'driver.py'}:/driver.py:ro",
|
||||
IMAGE, "sh", "-lc", script,
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--privileged",
|
||||
"--name",
|
||||
staged.name,
|
||||
"-v",
|
||||
f"{staged / 'src'}:/src:ro",
|
||||
"-v",
|
||||
f"{staged / 'driver.py'}:/driver.py:ro",
|
||||
IMAGE,
|
||||
"sh",
|
||||
"-lc",
|
||||
script,
|
||||
],
|
||||
capture=True,
|
||||
check=False,
|
||||
@@ -99,7 +114,9 @@ def drive(fstype: str, kind: str, expect: str) -> str:
|
||||
finally:
|
||||
shutil.rmtree(staged, ignore_errors=True)
|
||||
if proc.returncode != 0:
|
||||
raise AssertionError(f"{fstype}/{kind} driver failed:\n{proc.stdout}\n{proc.stderr}")
|
||||
raise AssertionError(
|
||||
f"{fstype}/{kind} driver failed:\n{proc.stdout}\n{proc.stderr}"
|
||||
)
|
||||
return proc.stdout
|
||||
|
||||
|
||||
@@ -125,7 +142,9 @@ class TestE2ESnapshot(unittest.TestCase):
|
||||
"E2E_REQUIRE_FILESYSTEMS demands zfs, but this kernel provides no "
|
||||
"zfs module; load it before running the suite"
|
||||
)
|
||||
self.skipTest("this kernel provides no zfs module, so no pool can be created")
|
||||
self.skipTest(
|
||||
"this kernel provides no zfs module, so no pool can be created"
|
||||
)
|
||||
self.assert_freezes("zfs")
|
||||
|
||||
def test_ext4_has_no_snapshot_and_says_so(self) -> None:
|
||||
|
||||
@@ -68,11 +68,20 @@ class TestE2ESnapshotDatabase(unittest.TestCase):
|
||||
try:
|
||||
proc = run(
|
||||
[
|
||||
"docker", "run", "--rm", "--privileged",
|
||||
"--name", staged.name,
|
||||
"-v", f"{staged / 'src'}:/src:ro",
|
||||
"-v", f"{staged / 'driver.py'}:/driver.py:ro",
|
||||
IMAGE, "sh", "-lc", SCRIPT,
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--privileged",
|
||||
"--name",
|
||||
staged.name,
|
||||
"-v",
|
||||
f"{staged / 'src'}:/src:ro",
|
||||
"-v",
|
||||
f"{staged / 'driver.py'}:/driver.py:ro",
|
||||
IMAGE,
|
||||
"sh",
|
||||
"-lc",
|
||||
SCRIPT,
|
||||
],
|
||||
capture=True,
|
||||
check=False,
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
# tests/e2e/test_e2e_swarm_task_skip.py
|
||||
#
|
||||
# Reproduces the swarm flake fixed on this branch: baudolo used to stop a
|
||||
# swarm task container around the volume file backup because its image was
|
||||
# not whitelisted; the orchestrator immediately replaced the stopped task and
|
||||
|
||||
125
tests/e2e/test_e2e_volumes_no_backup_required_early_skip.py
Normal file
125
tests/e2e/test_e2e_volumes_no_backup_required_early_skip.py
Normal file
@@ -0,0 +1,125 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
class TestE2EVolumesNoBackupRequiredEarlySkip(unittest.TestCase):
|
||||
"""Both volumes hang off the same container, so an image-level exclusion
|
||||
could only drop both. Only the named one may disappear."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
|
||||
cls.prefix = unique("baudolo-e2e-early-skip-no-backup-volume")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.container = f"{cls.prefix}-app"
|
||||
cls.excluded_volume = f"{cls.prefix}-derived-vol"
|
||||
cls.kept_volume = f"{cls.prefix}-state-vol"
|
||||
|
||||
cls.containers = [cls.container]
|
||||
cls.volumes = [cls.excluded_volume, cls.kept_volume]
|
||||
|
||||
run(["docker", "volume", "create", cls.excluded_volume])
|
||||
run(["docker", "volume", "create", cls.kept_volume])
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"-v",
|
||||
f"{cls.excluded_volume}:/derived",
|
||||
"-v",
|
||||
f"{cls.kept_volume}:/state",
|
||||
"alpine:3.20",
|
||||
"sh",
|
||||
"-lc",
|
||||
"echo derived > /derived/derived.txt && echo state > /state/state.txt",
|
||||
]
|
||||
)
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.container,
|
||||
"-v",
|
||||
f"{cls.excluded_volume}:/derived",
|
||||
"-v",
|
||||
f"{cls.kept_volume}:/state",
|
||||
"alpine:3.20",
|
||||
"sleep",
|
||||
"600",
|
||||
]
|
||||
)
|
||||
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(cls.databases_csv, [])
|
||||
|
||||
cmd = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
cls.compose_dir,
|
||||
"--repo-name",
|
||||
cls.repo_name,
|
||||
"--databases-csv",
|
||||
cls.databases_csv,
|
||||
"--backups-dir",
|
||||
cls.backups_dir,
|
||||
"--images-no-stop-required",
|
||||
"alpine:3.20",
|
||||
"--volumes-no-backup-required",
|
||||
cls.excluded_volume,
|
||||
]
|
||||
cp = run(cmd, capture=True, check=True)
|
||||
cls.stdout = cp.stdout or ""
|
||||
cls.stderr = cp.stderr or ""
|
||||
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
|
||||
def test_excluded_volume_has_no_backup_directory_at_all(self) -> None:
|
||||
p = backup_path(
|
||||
self.backups_dir,
|
||||
self.repo_name,
|
||||
self.version,
|
||||
self.excluded_volume,
|
||||
)
|
||||
self.assertFalse(
|
||||
p.exists(),
|
||||
f"Expected NO backup directory for the excluded volume, but found: {p}",
|
||||
)
|
||||
|
||||
def test_sibling_volume_of_the_same_container_is_still_backed_up(self) -> None:
|
||||
p = (
|
||||
backup_path(
|
||||
self.backups_dir,
|
||||
self.repo_name,
|
||||
self.version,
|
||||
self.kept_volume,
|
||||
)
|
||||
/ "files"
|
||||
/ "state.txt"
|
||||
)
|
||||
self.assertTrue(p.is_file(), f"Expected backed up file at: {p}")
|
||||
@@ -9,7 +9,6 @@ import pandas as pd
|
||||
# Adjust if your package name/import path differs.
|
||||
from baudolo.backup.dumps import load_databases_df
|
||||
|
||||
|
||||
EXPECTED_COLUMNS = ["instance", "database", "username", "password"]
|
||||
|
||||
|
||||
@@ -33,7 +32,6 @@ class TestLoadDatabasesDf(unittest.TestCase):
|
||||
def test_empty_csv_is_handled_with_warning_and_empty_df(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
empty_path = os.path.join(td, "databases.csv")
|
||||
# Create an empty file (0 bytes)
|
||||
with open(empty_path, "w", encoding="utf-8") as f:
|
||||
f.write("")
|
||||
|
||||
|
||||
80
tests/unit/backup/test_app_snapshot.py
Normal file
80
tests/unit/backup/test_app_snapshot.py
Normal file
@@ -0,0 +1,80 @@
|
||||
"""Contract of app.main's snapshot branch - the caller that runs in production."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from baudolo.backup import app
|
||||
from baudolo.backup.snapshot import volume_snapshot
|
||||
|
||||
|
||||
def stubbed_snapshot(kind: str, subject: str, tag: str):
|
||||
return volume_snapshot(kind, subject, tag, run=lambda command: [])
|
||||
|
||||
|
||||
ARGV = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
"/compose",
|
||||
"--backups-dir",
|
||||
"/backups",
|
||||
"--snapshot",
|
||||
"btrfs",
|
||||
"--snapshot-subject",
|
||||
"/var/lib/docker",
|
||||
]
|
||||
|
||||
|
||||
def drive(*, present: bool) -> list[dict]:
|
||||
calls: list[dict] = []
|
||||
|
||||
def record(versions_dir, volume_name, volume_dir, *, authoritative, source):
|
||||
calls.append(
|
||||
{"volume": volume_name, "authoritative": authoritative, "source": source}
|
||||
)
|
||||
|
||||
with (
|
||||
mock.patch("sys.argv", ARGV),
|
||||
mock.patch.object(app, "get_machine_id", return_value="machine"),
|
||||
mock.patch.object(app, "create_version_directory", return_value="/gen"),
|
||||
mock.patch.object(app, "create_volume_directory", return_value="/gen/vol"),
|
||||
mock.patch.object(app, "load_databases_df", return_value=None),
|
||||
mock.patch.object(app, "docker_volume_names", return_value=["vol"]),
|
||||
mock.patch.object(app, "containers_using_volume", return_value=[]),
|
||||
mock.patch.object(app, "volume_is_fully_ignored", return_value=False),
|
||||
mock.patch.object(app, "backup_dumps_for_volume", return_value=(False, False)),
|
||||
mock.patch.object(
|
||||
app, "get_storage_path", return_value="/var/lib/docker/volumes/vol/_data/"
|
||||
),
|
||||
mock.patch.object(app, "stamp_directory"),
|
||||
mock.patch.object(app, "handle_docker_compose_services"),
|
||||
mock.patch.object(app.os.path, "isdir", return_value=present),
|
||||
mock.patch.object(app, "backup_volume", side_effect=record),
|
||||
mock.patch.object(app, "volume_snapshot", stubbed_snapshot),
|
||||
):
|
||||
app.main()
|
||||
return calls
|
||||
|
||||
|
||||
class TestSnapshotBranch(unittest.TestCase):
|
||||
def test_it_passes_a_path_ending_in_a_separator(self) -> None:
|
||||
source = drive(present=True)[0]["source"]
|
||||
self.assertTrue(source.endswith("/volumes/vol/_data/"), source)
|
||||
self.assertNotIn("/var/lib/docker/volumes", source)
|
||||
|
||||
def test_it_reads_from_the_snapshot_and_not_from_the_live_tree(self) -> None:
|
||||
source = drive(present=True)[0]["source"]
|
||||
self.assertTrue(source.startswith("/var/lib/docker/.baudolo-"), source)
|
||||
|
||||
def test_it_compares_by_content_against_the_previous_generation(self) -> None:
|
||||
self.assertTrue(drive(present=True)[0]["authoritative"])
|
||||
|
||||
def test_a_volume_missing_from_the_snapshot_is_copied_live(self) -> None:
|
||||
call = drive(present=False)[0]
|
||||
self.assertEqual(call["source"], "/var/lib/docker/volumes/vol/_data/")
|
||||
self.assertFalse(call["authoritative"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
82
tests/unit/backup/test_app_volumes_no_backup_required.py
Normal file
82
tests/unit/backup/test_app_volumes_no_backup_required.py
Normal file
@@ -0,0 +1,82 @@
|
||||
"""Contract of --volumes-no-backup-required: exclusion is per volume name,
|
||||
independent of which containers use it."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from baudolo.backup import app
|
||||
|
||||
ARGV = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
"/compose",
|
||||
"--backups-dir",
|
||||
"/backups",
|
||||
"--volumes-no-backup-required",
|
||||
"derived",
|
||||
]
|
||||
|
||||
|
||||
def drive() -> tuple[list[str], list[str], list[str]]:
|
||||
backed_up: list[str] = []
|
||||
created: list[str] = []
|
||||
inspected: list[str] = []
|
||||
|
||||
def record_backup(versions_dir, volume_name, volume_dir, *, authoritative, source):
|
||||
backed_up.append(volume_name)
|
||||
|
||||
with (
|
||||
mock.patch("sys.argv", ARGV),
|
||||
mock.patch.object(app, "get_machine_id", return_value="machine"),
|
||||
mock.patch.object(app, "create_version_directory", return_value="/gen"),
|
||||
mock.patch.object(
|
||||
app,
|
||||
"create_volume_directory",
|
||||
side_effect=lambda _version_dir, name: created.append(name) or "/gen/vol",
|
||||
),
|
||||
mock.patch.object(app, "load_databases_df", return_value=None),
|
||||
mock.patch.object(
|
||||
app, "docker_volume_names", return_value=["derived", "state"]
|
||||
),
|
||||
mock.patch.object(
|
||||
app,
|
||||
"containers_using_volume",
|
||||
side_effect=lambda name: inspected.append(name) or ["app"],
|
||||
),
|
||||
mock.patch.object(app, "volume_is_fully_ignored", return_value=False),
|
||||
mock.patch.object(app, "backup_dumps_for_volume", return_value=(False, False)),
|
||||
mock.patch.object(app, "get_storage_path", return_value="/data/"),
|
||||
mock.patch.object(app, "stamp_directory"),
|
||||
mock.patch.object(app, "handle_docker_compose_services"),
|
||||
mock.patch.object(app.os.path, "isdir", return_value=True),
|
||||
mock.patch.object(app, "backup_volume", side_effect=record_backup),
|
||||
mock.patch.object(app, "filter_stoppable", return_value=[]),
|
||||
mock.patch.object(app, "requires_stop", return_value=False),
|
||||
mock.patch.object(app, "change_containers_status"),
|
||||
):
|
||||
app.main()
|
||||
return backed_up, created, inspected
|
||||
|
||||
|
||||
class TestVolumesNoBackupRequired(unittest.TestCase):
|
||||
def test_the_named_volume_is_never_backed_up(self) -> None:
|
||||
backed_up, _created, _inspected = drive()
|
||||
self.assertNotIn("derived", backed_up)
|
||||
|
||||
def test_a_sibling_volume_of_the_same_container_survives(self) -> None:
|
||||
backed_up, _created, _inspected = drive()
|
||||
self.assertEqual(backed_up, ["state"])
|
||||
|
||||
def test_no_generation_directory_is_created_for_it(self) -> None:
|
||||
_backed_up, created, _inspected = drive()
|
||||
self.assertEqual(created, ["state"])
|
||||
|
||||
def test_the_skip_precedes_the_container_inspection(self) -> None:
|
||||
_backed_up, _created, inspected = drive()
|
||||
self.assertEqual(inspected, ["state"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -39,7 +39,9 @@ class TestSnapshotFlags(unittest.TestCase):
|
||||
parse("--snapshot", "ext4", "--snapshot-subject", "/var/lib/docker")
|
||||
|
||||
def test_zfs_is_accepted(self) -> None:
|
||||
self.assertEqual(parse("--snapshot", "zfs", "--snapshot-subject", "/d").snapshot, "zfs")
|
||||
self.assertEqual(
|
||||
parse("--snapshot", "zfs", "--snapshot-subject", "/d").snapshot, "zfs"
|
||||
)
|
||||
|
||||
def test_shutdown_is_rejected_because_nothing_is_stopped(self) -> None:
|
||||
with self.assertRaises(SystemExit):
|
||||
@@ -48,17 +50,37 @@ class TestSnapshotFlags(unittest.TestCase):
|
||||
def test_shutdown_stays_available_without_a_snapshot(self) -> None:
|
||||
self.assertTrue(parse("--shutdown").shutdown)
|
||||
|
||||
def test_hard_restart_is_rejected_because_nothing_is_stopped(self) -> None:
|
||||
with self.assertRaises(SystemExit):
|
||||
parse(
|
||||
"--snapshot",
|
||||
"btrfs",
|
||||
"--snapshot-subject",
|
||||
"/d",
|
||||
"--hard-restart-projects",
|
||||
"mailu",
|
||||
)
|
||||
|
||||
def test_hard_restart_stays_available_without_a_snapshot(self) -> None:
|
||||
self.assertEqual(
|
||||
parse("--hard-restart-projects", "mailu").hard_restart_projects, ["mailu"]
|
||||
)
|
||||
|
||||
|
||||
class TestRequiredFlags(unittest.TestCase):
|
||||
def test_backups_dir_is_required(self) -> None:
|
||||
with mock.patch("sys.argv", ["baudolo", "--compose-dir", "/compose"]):
|
||||
with self.assertRaises(SystemExit):
|
||||
parse_args()
|
||||
with (
|
||||
mock.patch("sys.argv", ["baudolo", "--compose-dir", "/compose"]),
|
||||
self.assertRaises(SystemExit),
|
||||
):
|
||||
parse_args()
|
||||
|
||||
def test_compose_dir_is_required(self) -> None:
|
||||
with mock.patch("sys.argv", ["baudolo", "--backups-dir", "/backups"]):
|
||||
with self.assertRaises(SystemExit):
|
||||
parse_args()
|
||||
with (
|
||||
mock.patch("sys.argv", ["baudolo", "--backups-dir", "/backups"]),
|
||||
self.assertRaises(SystemExit),
|
||||
):
|
||||
parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -3,7 +3,6 @@ from __future__ import annotations
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
from unittest.mock import patch
|
||||
|
||||
from .compose_fixture import setup_compose_dir as _setup_compose_dir
|
||||
@@ -99,7 +98,7 @@ class TestCompose(unittest.TestCase):
|
||||
str(d), ["up", "-d", "--force-recreate"]
|
||||
)
|
||||
|
||||
expected: List[str] = [
|
||||
expected: list[str] = [
|
||||
"/usr/bin/docker",
|
||||
"compose",
|
||||
"--chdir",
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from typing import List
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
@@ -11,7 +10,7 @@ class HardRestartArgTests(unittest.TestCase):
|
||||
the dir is a stack whose overlay network collides with compose up, pass
|
||||
nothing."""
|
||||
|
||||
def _parse(self, extra: List[str]):
|
||||
def _parse(self, extra: list[str]):
|
||||
import sys
|
||||
|
||||
from baudolo.backup import cli
|
||||
|
||||
@@ -20,15 +20,17 @@ def _capture_commands(*, db_type, rows, container):
|
||||
captured.append(cmd)
|
||||
return []
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
with patch.object(db_mod, "execute_shell_command", side_effect=_capture):
|
||||
db_mod.backup_database(
|
||||
container=container,
|
||||
volume_dir=td,
|
||||
db_type=db_type,
|
||||
databases_df=_df(rows),
|
||||
database_containers=[container],
|
||||
)
|
||||
with (
|
||||
tempfile.TemporaryDirectory() as td,
|
||||
patch.object(db_mod, "execute_shell_command", side_effect=_capture),
|
||||
):
|
||||
db_mod.backup_database(
|
||||
container=container,
|
||||
volume_dir=td,
|
||||
db_type=db_type,
|
||||
databases_df=_df(rows),
|
||||
database_containers=[container],
|
||||
)
|
||||
return captured
|
||||
|
||||
|
||||
|
||||
50
tests/unit/backup/test_docker_image_name.py
Normal file
50
tests/unit/backup/test_docker_image_name.py
Normal file
@@ -0,0 +1,50 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from baudolo.backup import docker as docker_mod
|
||||
|
||||
|
||||
def _with_image(reference: str):
|
||||
return patch.object(docker_mod, "execute_shell_command", return_value=[reference])
|
||||
|
||||
|
||||
class TestImageName(unittest.TestCase):
|
||||
def test_plain_reference(self) -> None:
|
||||
with _with_image("postgres:16"):
|
||||
self.assertEqual(docker_mod.image_name("c1"), "postgres")
|
||||
|
||||
def test_registry_host_is_dropped(self) -> None:
|
||||
with _with_image("svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5"):
|
||||
self.assertEqual(docker_mod.image_name("c1"), "postgres_custom")
|
||||
|
||||
def test_pull_through_path_is_kept(self) -> None:
|
||||
with _with_image(
|
||||
"svc-db-mariadb-swarm-mgr-01:5000/ghcr.io/x/mirror/docker.io/postgres:16"
|
||||
):
|
||||
self.assertEqual(
|
||||
docker_mod.image_name("c1"), "ghcr.io/x/mirror/docker.io/postgres"
|
||||
)
|
||||
|
||||
def test_digest_is_dropped(self) -> None:
|
||||
with _with_image("registry:5000/postgres@sha256:" + "0" * 64):
|
||||
self.assertEqual(docker_mod.image_name("c1"), "postgres")
|
||||
|
||||
|
||||
class TestHasImage(unittest.TestCase):
|
||||
def test_registry_hostname_does_not_decide_the_engine(self) -> None:
|
||||
with _with_image("svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5"):
|
||||
self.assertFalse(docker_mod.has_image("c1", "mariadb"))
|
||||
with _with_image("svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5"):
|
||||
self.assertTrue(docker_mod.has_image("c1", "postgres"))
|
||||
|
||||
def test_tag_does_not_decide_the_engine(self) -> None:
|
||||
with _with_image("registry:5000/xwiki_custom:lts-postgres-tomcat"):
|
||||
self.assertFalse(docker_mod.has_image("c1", "postgres"))
|
||||
|
||||
def test_mirrored_mariadb_still_matches(self) -> None:
|
||||
with _with_image("registry:5000/ghcr.io/x/mirror/docker.io/mariadb:11"):
|
||||
self.assertTrue(docker_mod.has_image("c1", "mariadb"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -8,6 +8,7 @@ from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from baudolo.backup import layout as mod
|
||||
from baudolo.backup.shell import BackupException
|
||||
|
||||
|
||||
class TestVersionDirectory(unittest.TestCase):
|
||||
@@ -17,11 +18,12 @@ class TestVersionDirectory(unittest.TestCase):
|
||||
self.assertTrue(Path(created).is_dir())
|
||||
self.assertEqual(Path(created).name, "20260731020304")
|
||||
|
||||
def test_it_is_idempotent(self) -> None:
|
||||
def test_it_refuses_a_generation_another_run_already_claimed(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
first = mod.create_version_directory(tmp, "20260731")
|
||||
second = mod.create_version_directory(tmp, "20260731")
|
||||
self.assertEqual(first, second)
|
||||
mod.create_version_directory(tmp, "20260731")
|
||||
with self.assertRaises(BackupException) as caught:
|
||||
mod.create_version_directory(tmp, "20260731")
|
||||
self.assertIn("20260731", str(caught.exception))
|
||||
|
||||
def test_it_creates_missing_parents(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from baudolo.backup.shell import BackupException
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
||||
|
||||
|
||||
@@ -21,34 +22,50 @@ class Runner:
|
||||
|
||||
|
||||
class TestBtrfs(unittest.TestCase):
|
||||
def test_it_creates_a_read_only_snapshot_beside_the_subject(self) -> None:
|
||||
def test_it_creates_a_read_only_snapshot_inside_the_subject(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
||||
pass
|
||||
self.assertEqual(
|
||||
run.calls[0],
|
||||
"btrfs subvolume snapshot -r /var/lib/docker /var/lib/.baudolo-20260731",
|
||||
"btrfs subvolume snapshot -r /var/lib/docker /var/lib/docker/.baudolo-20260731",
|
||||
)
|
||||
|
||||
def test_it_removes_the_snapshot_afterwards(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
||||
pass
|
||||
self.assertEqual(run.calls[-1], "btrfs subvolume delete /var/lib/.baudolo-20260731")
|
||||
self.assertEqual(
|
||||
run.calls[-1], "btrfs subvolume delete /var/lib/docker/.baudolo-20260731"
|
||||
)
|
||||
|
||||
def test_it_maps_a_volume_path_into_the_snapshot(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
||||
with volume_snapshot(
|
||||
"btrfs", "/var/lib/docker", "20260731", run=run
|
||||
) as resolve:
|
||||
self.assertEqual(
|
||||
resolve("/var/lib/docker/volumes/postgres_data/_data"),
|
||||
"/var/lib/.baudolo-20260731/volumes/postgres_data/_data",
|
||||
"/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data",
|
||||
)
|
||||
|
||||
def test_it_keeps_the_trailing_slash_rsync_reads_as_contents(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot(
|
||||
"btrfs", "/var/lib/docker", "20260731", run=run
|
||||
) as resolve:
|
||||
self.assertEqual(
|
||||
resolve("/var/lib/docker/volumes/postgres_data/_data/"),
|
||||
"/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data/",
|
||||
)
|
||||
|
||||
def test_it_removes_the_snapshot_even_when_the_body_raises(self) -> None:
|
||||
run = Runner()
|
||||
with self.assertRaises(ZeroDivisionError):
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
||||
raise ZeroDivisionError
|
||||
with (
|
||||
self.assertRaises(ZeroDivisionError),
|
||||
volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run),
|
||||
):
|
||||
raise ZeroDivisionError
|
||||
self.assertTrue(run.calls[-1].startswith("btrfs subvolume delete"))
|
||||
|
||||
|
||||
@@ -78,29 +95,59 @@ class TestZfs(unittest.TestCase):
|
||||
|
||||
def test_an_unmounted_dataset_is_an_error(self) -> None:
|
||||
run = Runner({"zfs list": [""]})
|
||||
with self.assertRaises(SnapshotError):
|
||||
with volume_snapshot("zfs", "/var/lib/docker", "20260731", run=run):
|
||||
pass
|
||||
with (
|
||||
self.assertRaises(SnapshotError),
|
||||
volume_snapshot("zfs", "/var/lib/docker", "20260731", run=run),
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
class TestRejections(unittest.TestCase):
|
||||
def test_an_unknown_kind_is_rejected(self) -> None:
|
||||
run = Runner()
|
||||
with self.assertRaises(SnapshotError):
|
||||
with volume_snapshot("ext4", "/var/lib/docker", "20260731", run=run):
|
||||
pass
|
||||
with (
|
||||
self.assertRaises(SnapshotError),
|
||||
volume_snapshot("ext4", "/var/lib/docker", "20260731", run=run),
|
||||
):
|
||||
pass
|
||||
self.assertEqual(run.calls, [])
|
||||
|
||||
def test_a_path_outside_the_subject_is_rejected(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
||||
with self.assertRaises(SnapshotError):
|
||||
resolve("/etc/passwd")
|
||||
with (
|
||||
volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve,
|
||||
self.assertRaises(SnapshotError),
|
||||
):
|
||||
resolve("/etc/passwd")
|
||||
|
||||
def test_the_subject_itself_resolves_to_the_snapshot_root(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
||||
self.assertEqual(resolve("/var/lib/docker"), "/var/lib/.baudolo-20260731")
|
||||
with volume_snapshot(
|
||||
"btrfs", "/var/lib/docker", "20260731", run=run
|
||||
) as resolve:
|
||||
self.assertEqual(
|
||||
resolve("/var/lib/docker"), "/var/lib/docker/.baudolo-20260731"
|
||||
)
|
||||
|
||||
|
||||
class Busy(Runner):
|
||||
def __call__(self, command: str) -> list[str]:
|
||||
if command.startswith("btrfs subvolume delete"):
|
||||
raise BackupException("target is busy")
|
||||
return super().__call__(command)
|
||||
|
||||
|
||||
class TestRemovalFailure(unittest.TestCase):
|
||||
def test_a_failed_removal_does_not_fail_a_completed_run(self) -> None:
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=Busy()):
|
||||
pass
|
||||
|
||||
def test_a_failed_removal_does_not_mask_the_body(self) -> None:
|
||||
with (
|
||||
self.assertRaises(ZeroDivisionError),
|
||||
volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=Busy()),
|
||||
):
|
||||
raise ZeroDivisionError
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -43,6 +43,14 @@ class TestBackupVolume(unittest.TestCase):
|
||||
def test_it_always_deletes_what_the_source_no_longer_has(self) -> None:
|
||||
self.assertIn("--delete", self.copy())
|
||||
|
||||
def test_it_carries_no_kernel_objects_into_a_generation(self) -> None:
|
||||
self.assertIn("--no-D", self.copy())
|
||||
|
||||
def test_it_keeps_no_twin_of_what_the_second_pass_replaces(self) -> None:
|
||||
command = self.copy(authoritative=True)
|
||||
self.assertIn("rsync -aP ", command)
|
||||
self.assertNotIn("--backup", command)
|
||||
|
||||
def test_it_creates_the_destination(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
dest = Path(tmp) / "gen" / "demo"
|
||||
|
||||
121
tests/unit/restore/test_cluster_replay.py
Normal file
121
tests/unit/restore/test_cluster_replay.py
Normal file
@@ -0,0 +1,121 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from baudolo.restore.db import cluster as cluster_mod
|
||||
from baudolo.restore.paths import BackupPaths
|
||||
|
||||
|
||||
class TestClusterReplay(unittest.TestCase):
|
||||
def _replay(self, *, empty: bool):
|
||||
calls = []
|
||||
|
||||
def _capture(container, argv, **kwargs):
|
||||
calls.append((argv, kwargs.get("stdin")))
|
||||
return MagicMock()
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".sql") as sql:
|
||||
sql.write(b"CREATE ROLE app;\nCREATE DATABASE app OWNER app;\n")
|
||||
sql.flush()
|
||||
with patch.object(cluster_mod, "docker_exec", side_effect=_capture):
|
||||
cluster_mod.restore_cluster_sql(
|
||||
container="db",
|
||||
user="postgres",
|
||||
password="pw",
|
||||
sql_path=sql.name,
|
||||
empty=empty,
|
||||
)
|
||||
return calls
|
||||
|
||||
def test_the_replay_is_not_wrapped_in_a_transaction(self) -> None:
|
||||
argv, _ = self._replay(empty=False)[0]
|
||||
self.assertNotIn(
|
||||
"--single-transaction",
|
||||
argv,
|
||||
"CREATE DATABASE cannot run inside a transaction block, so unlike the "
|
||||
"single-database replay this stream must not be wrapped in one",
|
||||
)
|
||||
self.assertIn("ON_ERROR_STOP=1", argv)
|
||||
|
||||
def test_the_replay_targets_the_control_database(self) -> None:
|
||||
argv, _ = self._replay(empty=False)[0]
|
||||
self.assertEqual(argv[argv.index("-d") + 1], cluster_mod.CONTROL_DB)
|
||||
self.assertEqual(argv[argv.index("-U") + 1], "postgres")
|
||||
|
||||
def test_without_empty_nothing_is_dropped_first(self) -> None:
|
||||
self.assertEqual(len(self._replay(empty=False)), 1)
|
||||
|
||||
def test_empty_drops_databases_before_their_owners(self) -> None:
|
||||
calls = self._replay(empty=True)
|
||||
self.assertEqual(len(calls), 2, f"expected pre-clean + replay: {calls}")
|
||||
preclean = calls[0][1].decode()
|
||||
self.assertLess(
|
||||
preclean.index("DROP DATABASE"),
|
||||
preclean.index("DROP ROLE"),
|
||||
"a role cannot be dropped while it still owns a database",
|
||||
)
|
||||
self.assertIn("DROP OWNED BY", preclean)
|
||||
self.assertIn("ORDER BY phase", preclean)
|
||||
|
||||
def test_the_preclean_spares_what_the_dump_does_not_recreate(self) -> None:
|
||||
preclean = self._replay(empty=True)[0][1].decode()
|
||||
self.assertIn("NOT datistemplate", preclean)
|
||||
self.assertIn("datname <> current_database()", preclean)
|
||||
self.assertIn("starts_with(rolname, 'pg_')", preclean)
|
||||
self.assertIn("rolname <> current_user", preclean)
|
||||
|
||||
def test_only_the_connecting_role_loses_its_create(self) -> None:
|
||||
# Captured from pg_dumpall 17.
|
||||
dump = [
|
||||
b"CREATE ROLE app;\n",
|
||||
b"ALTER ROLE app WITH NOSUPERUSER INHERIT LOGIN PASSWORD 'SCRAM-SHA-256$...';\n",
|
||||
b"CREATE ROLE postgres;\n",
|
||||
b"ALTER ROLE postgres WITH SUPERUSER INHERIT LOGIN PASSWORD 'SCRAM-SHA-256$...';\n",
|
||||
b'CREATE ROLE "odd-name";\n',
|
||||
]
|
||||
kept = list(cluster_mod.filter_own_role_creation(dump, "postgres"))
|
||||
self.assertNotIn(b"CREATE ROLE postgres;\n", kept)
|
||||
self.assertIn(b"CREATE ROLE app;\n", kept)
|
||||
self.assertIn(b'CREATE ROLE "odd-name";\n', kept)
|
||||
self.assertEqual(
|
||||
sum(1 for line in kept if line.startswith(b"ALTER ROLE postgres")),
|
||||
1,
|
||||
"the ALTER re-applies the superuser's attributes and password",
|
||||
)
|
||||
|
||||
def test_a_quoted_connecting_role_is_matched_too(self) -> None:
|
||||
kept = list(
|
||||
cluster_mod.filter_own_role_creation(
|
||||
[b'CREATE ROLE "odd-name";\n'], "odd-name"
|
||||
)
|
||||
)
|
||||
self.assertEqual(kept, [])
|
||||
|
||||
def test_a_role_whose_name_merely_starts_the_same_is_kept(self) -> None:
|
||||
kept = list(
|
||||
cluster_mod.filter_own_role_creation(
|
||||
[b"CREATE ROLE postgresql;\n"], "postgres"
|
||||
)
|
||||
)
|
||||
self.assertEqual(kept, [b"CREATE ROLE postgresql;\n"])
|
||||
|
||||
def test_a_missing_dump_is_reported_as_such(self) -> None:
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
cluster_mod.restore_cluster_sql(
|
||||
container="db",
|
||||
user="postgres",
|
||||
password="pw",
|
||||
sql_path="/nonexistent/x.cluster.backup.sql",
|
||||
empty=False,
|
||||
)
|
||||
|
||||
def test_the_path_helper_names_the_dumpall_file(self) -> None:
|
||||
paths = BackupPaths("vol", "hash", "v1", repo_name="repo", backups_dir="/B")
|
||||
self.assertEqual(
|
||||
paths.cluster_file("bigbluebutton"),
|
||||
"/B/hash/repo/v1/vol/sql/bigbluebutton.cluster.backup.sql",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -131,7 +131,6 @@ class TestSeedMain(unittest.TestCase):
|
||||
warning_calls,
|
||||
"Expected a WARNING print when databases.csv is empty, but none was found.",
|
||||
)
|
||||
# Ensure the warning goes to stderr
|
||||
_, warn_kwargs = warning_calls[0]
|
||||
self.assertEqual(warn_kwargs.get("file"), seed_main.sys.stderr)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user