mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2026-08-24 14:54:32 +00:00
Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f437787e64 | |||
| 2129c5e362 | |||
| a0204fd3ea | |||
| 90d289d92f | |||
| 57fc7c96bc | |||
| 30fd68bdcf | |||
| 36b2336742 | |||
| 756e236d10 | |||
| 1dfeb17ab4 | |||
| cd21f1fa67 | |||
| 8a93a61ca9 | |||
| 988d92534c | |||
| 934e693810 | |||
| 2e0e67ca87 | |||
| 95c34d4db0 |
@@ -1,6 +1,7 @@
|
|||||||
{
|
{
|
||||||
"permissions": {
|
"permissions": {
|
||||||
"ask": [
|
"ask": [
|
||||||
|
"Bash(git commit*)",
|
||||||
"Edit(CHANGELOG.md)",
|
"Edit(CHANGELOG.md)",
|
||||||
"Write(CHANGELOG.md)",
|
"Write(CHANGELOG.md)",
|
||||||
"Edit(pyproject.toml)",
|
"Edit(pyproject.toml)",
|
||||||
|
|||||||
96
CHANGELOG.md
96
CHANGELOG.md
@@ -1,5 +1,101 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## [3.5.0] - 2026-08-17
|
||||||
|
|
||||||
|
- Restore: a *database = '*'* row makes the backup write
|
||||||
|
*<instance>.cluster.backup.sql* via *pg_dumpall*, and nothing could read it
|
||||||
|
back — the CLI knew *files*, *postgres* and *mariadb*, so that dump was
|
||||||
|
stored and unrestorable. *baudolo-restore cluster* replays it against the
|
||||||
|
control database, deliberately without *--single-transaction* because
|
||||||
|
CREATE DATABASE is forbidden inside a transaction block, and filters out the
|
||||||
|
CREATE ROLE of the connecting role, which the pre-clean cannot drop while it
|
||||||
|
holds the session. *--empty* drops the cluster's databases first, then
|
||||||
|
releases what its roles still own, then the roles themselves.
|
||||||
|
|
||||||
|
- Lint: ruff was never wired into the repository — no target, no CI step, no
|
||||||
|
pin — and reported 45 findings across sources and tests. *make ruff* and
|
||||||
|
*make lint* now run it over every file, *make test* gates on a clean run as a
|
||||||
|
fourth parallel spur, and the linter is pinned in a *lint* extra because a
|
||||||
|
minor bump changes which rules fire.
|
||||||
|
|
||||||
|
## [3.4.3] - 2026-08-16
|
||||||
|
|
||||||
|
- Backup: *create_version_directory* carried *exist_ok=True*, so a run starting
|
||||||
|
in the same wall-clock second as its predecessor claimed that predecessor's
|
||||||
|
generation. Generation names carry seconds, and a host with little to copy
|
||||||
|
finishes inside one — rsync *--delete* then overwrote a finished generation,
|
||||||
|
and only afterwards did *create_stamp_file* refuse the already-stamped
|
||||||
|
directory and exit 2. The guard reported the damage instead of preventing it.
|
||||||
|
- Backup: the generation directory is claimed exclusively. Claiming it is the
|
||||||
|
first filesystem action of a run, so a collision aborts before the first
|
||||||
|
write and names the second it collided on.
|
||||||
|
- Tests: the idempotence test asserted the reuse and gave way to one that
|
||||||
|
requires the refusal.
|
||||||
|
|
||||||
|
## [3.4.2] - 2026-08-15
|
||||||
|
|
||||||
|
- Backup: *has_image* matched the raw *.Config.Image*, so the registry host and
|
||||||
|
the tag decided the dump tool. On a swarm node named after the app under test,
|
||||||
|
*svc-db-mariadb-swarm-mgr-01:5000/postgres_custom* read as MariaDB and
|
||||||
|
*mariadb-dump* ran inside a Postgres container: exit 127, and the
|
||||||
|
*BackupException* took the backup unit with it.
|
||||||
|
- Backup: *image_name* strips digest, tag and registry host, so the engine rests
|
||||||
|
on the repository path alone — the exact-matching intent of 3.0.0 applied to
|
||||||
|
the one place that change did not reach.
|
||||||
|
- Tests: both false-positive directions on *has_image*, plus an e2e that
|
||||||
|
reproduces the shape with a *docker tag* and asserts a real *pg_dump* lands.
|
||||||
|
|
||||||
|
## [3.4.1] - 2026-08-05
|
||||||
|
|
||||||
|
- Backup: each volume is copied twice into the same destination — once hot,
|
||||||
|
once cold after the container is stopped — and rsync ran with *-b*, so
|
||||||
|
*--delete* renamed rather than removed a file the source had dropped between
|
||||||
|
the passes. Stopping a container is what makes the source drop files: a
|
||||||
|
graceful shutdown flushes and the format rolls its commit point. The
|
||||||
|
superseded file survived as *name~* beside the real one and was restored into
|
||||||
|
live data.
|
||||||
|
- Backup: for an opaque payload that is stale bytes nobody reads; for a format
|
||||||
|
that enumerates its own directory it is corruption. Lucene resolves the
|
||||||
|
current commit by parsing every file starting with *segments* as a radix-36
|
||||||
|
generation, so a restored *segments_3~* leaves the shard store unreadable and
|
||||||
|
the primary at *NO_VALID_SHARD_COPY*. With *.security-7* unallocatable the
|
||||||
|
reserved *elastic* user has no password hash, every probe answers 401 and the
|
||||||
|
container never turns healthy.
|
||||||
|
- Backup: *--link-dest* already provides the incrementals and nothing reads the
|
||||||
|
twins — the restore path is an unfiltered *rsync -avv --delete* into the live
|
||||||
|
volume. Dropping *-b* leaves the predecessor generation byte-identical, keeps
|
||||||
|
the hardlinks intact and makes generations smaller, never larger.
|
||||||
|
- Tests: the absence of *--backup* is asserted on the rsync invocation.
|
||||||
|
|
||||||
|
## [3.4.0] - 2026-08-02
|
||||||
|
|
||||||
|
- Backup: *-a* implies *-D*, so a generation was written with
|
||||||
|
*--devices --specials* and rsync recreated every unix socket and fifo found in
|
||||||
|
a volume. Where the backup root is an nfs-ganesha export, ganesha accepts the
|
||||||
|
socket on write but cannot serve it back, and the remote pull's sender then
|
||||||
|
fails with *readdir* / *readlink_stat* "Invalid argument (22)" and exits 23 —
|
||||||
|
deterministically, for every retry. *--no-D* keeps them out of the generation.
|
||||||
|
- Backup: nothing restorable is lost. Sockets and fifos are recreated by the
|
||||||
|
daemons that own them, and the postfix queue itself — *incoming*, *active*,
|
||||||
|
*deferred*, *hold*, *maildrop* — is unaffected, so accepted-but-undelivered
|
||||||
|
mail stays in the backup. Device nodes go too; the only volume that could hold
|
||||||
|
them is a nested docker data root, which does not belong in a backup anyway.
|
||||||
|
- Tests: the flag is asserted on the rsync invocation.
|
||||||
|
|
||||||
|
## [3.3.0] - 2026-08-02
|
||||||
|
|
||||||
|
- Backup: *--volumes-no-backup-required* excludes a volume by name.
|
||||||
|
*--images-no-backup-required* resolves through *volume_is_fully_ignored*,
|
||||||
|
which skips a volume only when every container using it is ignored — a
|
||||||
|
container holding a derived tree beside state that must be kept cannot
|
||||||
|
express the exclusion at all. A docker-in-docker data root is exactly that
|
||||||
|
shape, and excluding by image would drop all three of its volumes.
|
||||||
|
- Backup: the name check runs before *containers_using_volume*, so an excluded
|
||||||
|
volume costs no docker inspection and the decision does not depend on which
|
||||||
|
containers exist when the run starts.
|
||||||
|
- Tests: two volumes off one container, asserting the sibling survives — the
|
||||||
|
property the image lever cannot provide — as unit and end-to-end.
|
||||||
|
|
||||||
## [3.2.2] - 2026-07-31
|
## [3.2.2] - 2026-07-31
|
||||||
|
|
||||||
- Backup: the btrfs snapshot is carved inside its subject, as
|
- Backup: the btrfs snapshot is carved inside its subject, as
|
||||||
|
|||||||
27
Makefile
27
Makefile
@@ -1,4 +1,4 @@
|
|||||||
.PHONY: install build clean \
|
.PHONY: install install-lint build clean lint ruff ruff-fix \
|
||||||
test test-unit test-integration test-e2e \
|
test test-unit test-integration test-e2e \
|
||||||
test-unit-run test-integration-run test-e2e-run
|
test-unit-run test-integration-run test-e2e-run
|
||||||
|
|
||||||
@@ -34,13 +34,30 @@ build:
|
|||||||
clean:
|
clean:
|
||||||
git clean -fdX .
|
git clean -fdX .
|
||||||
|
|
||||||
# clean + build run once and in order, then the three suites run concurrently
|
# Separate from `install` so the test image does not have to carry the linter.
|
||||||
# via -j3; the *-run targets carry no clean/build prereq so the sub-make cannot
|
install-lint:
|
||||||
# race a second clean against build.
|
@$(PY_DEFAULT) -m pip install -q -e ".[lint]"
|
||||||
|
|
||||||
|
# Runs on the host, not in the image, so it also covers what the Dockerfile
|
||||||
|
# does not copy.
|
||||||
|
ruff: install-lint
|
||||||
|
@echo ">> Running ruff over the whole repository"
|
||||||
|
@$(PY_DEFAULT) -m ruff check .
|
||||||
|
@$(PY_DEFAULT) -m ruff format --check .
|
||||||
|
|
||||||
|
ruff-fix: install-lint
|
||||||
|
@$(PY_DEFAULT) -m ruff check --fix .
|
||||||
|
@$(PY_DEFAULT) -m ruff format .
|
||||||
|
|
||||||
|
lint: ruff
|
||||||
|
|
||||||
|
# clean + build run once and in order, then lint and the three suites run
|
||||||
|
# concurrently via -j4; the *-run targets carry no clean/build prereq so the
|
||||||
|
# sub-make cannot race a second clean against build.
|
||||||
test:
|
test:
|
||||||
@$(MAKE) clean
|
@$(MAKE) clean
|
||||||
@$(MAKE) build
|
@$(MAKE) build
|
||||||
@$(MAKE) -j3 test-unit-run test-integration-run test-e2e-run
|
@$(MAKE) -j4 lint test-unit-run test-integration-run test-e2e-run
|
||||||
|
|
||||||
test-unit: clean build test-unit-run
|
test-unit: clean build test-unit-run
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "backup-docker-to-local"
|
name = "backup-docker-to-local"
|
||||||
version = "3.2.2"
|
version = "3.5.0"
|
||||||
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.9"
|
requires-python = ">=3.9"
|
||||||
@@ -16,6 +16,11 @@ dependencies = [
|
|||||||
"dirval",
|
"dirval",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
# Pinned: a ruff minor bump changes which rules fire, and `make test` gates on
|
||||||
|
# a clean run, so an unpinned lint would fail the suite on an unrelated day.
|
||||||
|
lint = ["ruff==0.16.1"]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
baudolo = "baudolo.backup.__main__:main"
|
baudolo = "baudolo.backup.__main__:main"
|
||||||
baudolo-restore = "baudolo.restore.__main__:main"
|
baudolo-restore = "baudolo.restore.__main__:main"
|
||||||
|
|||||||
@@ -1,9 +1,6 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from .app import main
|
from .app import main
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
raise SystemExit(main())
|
raise SystemExit(main())
|
||||||
|
|||||||
@@ -30,17 +30,13 @@ def main() -> int:
|
|||||||
args = parse_args()
|
args = parse_args()
|
||||||
|
|
||||||
machine_id = get_machine_id()
|
machine_id = get_machine_id()
|
||||||
backup_time = datetime.now().strftime("%Y%m%d%H%M%S")
|
# Local wall clock on purpose: generations sort by this name, and UTC would
|
||||||
|
# order new ones before the existing ones wherever the offset is positive.
|
||||||
|
backup_time = datetime.now().strftime("%Y%m%d%H%M%S") # noqa: DTZ005
|
||||||
|
|
||||||
versions_dir = os.path.join(args.backups_dir, machine_id, args.repo_name)
|
versions_dir = os.path.join(args.backups_dir, machine_id, args.repo_name)
|
||||||
version_dir = create_version_directory(versions_dir, backup_time)
|
version_dir = create_version_directory(versions_dir, backup_time)
|
||||||
|
|
||||||
# IMPORTANT:
|
|
||||||
# - keep_default_na=False prevents empty fields from turning into NaN
|
|
||||||
# - dtype=str keeps all columns stable for comparisons/validation
|
|
||||||
#
|
|
||||||
# Robust behavior:
|
|
||||||
# - if the file is missing or empty, we continue without DB dumps.
|
|
||||||
databases_df = load_databases_df(args.databases_csv)
|
databases_df = load_databases_df(args.databases_csv)
|
||||||
|
|
||||||
print("💾 Start volume backups...", flush=True)
|
print("💾 Start volume backups...", flush=True)
|
||||||
@@ -54,6 +50,14 @@ def main() -> int:
|
|||||||
|
|
||||||
for volume_name in docker_volume_names():
|
for volume_name in docker_volume_names():
|
||||||
print(f"Start backup routine for volume: {volume_name}", flush=True)
|
print(f"Start backup routine for volume: {volume_name}", flush=True)
|
||||||
|
|
||||||
|
if volume_name in args.volumes_no_backup_required:
|
||||||
|
print(
|
||||||
|
f"Skipping volume '{volume_name}' entirely (declared no-backup).",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
containers = containers_using_volume(volume_name)
|
containers = containers_using_volume(volume_name)
|
||||||
|
|
||||||
if volume_is_fully_ignored(containers, args.images_no_backup_required):
|
if volume_is_fully_ignored(containers, args.images_no_backup_required):
|
||||||
@@ -72,24 +76,29 @@ def main() -> int:
|
|||||||
database_containers=args.database_containers,
|
database_containers=args.database_containers,
|
||||||
)
|
)
|
||||||
|
|
||||||
if args.dump_only_sql:
|
if args.dump_only_sql and found_db:
|
||||||
if found_db:
|
if not dumped_any:
|
||||||
if not dumped_any:
|
print(
|
||||||
print(
|
f"WARNING: dump-only-sql requested but no DB dump was produced for DB volume '{volume_name}'. "
|
||||||
f"WARNING: dump-only-sql requested but no DB dump was produced for DB volume '{volume_name}'. "
|
"Falling back to file backup.",
|
||||||
"Falling back to file backup.",
|
flush=True,
|
||||||
flush=True,
|
)
|
||||||
)
|
else:
|
||||||
else:
|
continue
|
||||||
continue
|
|
||||||
|
|
||||||
live_source = get_storage_path(volume_name)
|
live_source = get_storage_path(volume_name)
|
||||||
|
|
||||||
def copy(*, authoritative: bool, source: str = live_source) -> None:
|
def copy(
|
||||||
|
*,
|
||||||
|
authoritative: bool,
|
||||||
|
source: str = live_source,
|
||||||
|
volume: str = volume_name,
|
||||||
|
target: str = vol_dir,
|
||||||
|
) -> None:
|
||||||
backup_volume(
|
backup_volume(
|
||||||
versions_dir,
|
versions_dir,
|
||||||
volume_name,
|
volume,
|
||||||
vol_dir,
|
target,
|
||||||
authoritative=authoritative,
|
authoritative=authoritative,
|
||||||
source=source,
|
source=source,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -68,6 +68,13 @@ def parse_args() -> argparse.Namespace:
|
|||||||
help="Exact image references (repo:tag, incl. any registry prefix) for which no backup should be performed",
|
help="Exact image references (repo:tag, incl. any registry prefix) for which no backup should be performed",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
p.add_argument(
|
||||||
|
"--volumes-no-backup-required",
|
||||||
|
nargs="+",
|
||||||
|
default=[],
|
||||||
|
help="Exact volume names that are never backed up, whatever containers use them. For derived trees a restore cannot reproduce, above all a nested docker data root",
|
||||||
|
)
|
||||||
|
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--everything",
|
"--everything",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
@@ -93,7 +100,9 @@ def parse_args() -> argparse.Namespace:
|
|||||||
if bool(args.snapshot) != bool(args.snapshot_subject):
|
if bool(args.snapshot) != bool(args.snapshot_subject):
|
||||||
p.error("--snapshot and --snapshot-subject must be given together")
|
p.error("--snapshot and --snapshot-subject must be given together")
|
||||||
if args.snapshot and args.shutdown:
|
if args.snapshot and args.shutdown:
|
||||||
p.error("--shutdown is meaningless with --snapshot: containers are never stopped")
|
p.error(
|
||||||
|
"--shutdown is meaningless with --snapshot: containers are never stopped"
|
||||||
|
)
|
||||||
if args.snapshot and args.hard_restart_projects:
|
if args.snapshot and args.hard_restart_projects:
|
||||||
p.error(
|
p.error(
|
||||||
"--hard-restart-projects is meaningless with --snapshot: the flag exists "
|
"--hard-restart-projects is meaningless with --snapshot: the flag exists "
|
||||||
|
|||||||
@@ -4,10 +4,9 @@ import os
|
|||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import List, Optional
|
|
||||||
|
|
||||||
|
|
||||||
def _build_compose_cmd(project_dir: str, passthrough: List[str]) -> List[str]:
|
def _build_compose_cmd(project_dir: str, passthrough: list[str]) -> list[str]:
|
||||||
"""
|
"""
|
||||||
Build the compose command for this project directory.
|
Build the compose command for this project directory.
|
||||||
|
|
||||||
@@ -30,7 +29,7 @@ def _build_compose_cmd(project_dir: str, passthrough: List[str]) -> List[str]:
|
|||||||
raise RuntimeError("Neither 'compose' nor 'docker' found in PATH")
|
raise RuntimeError("Neither 'compose' nor 'docker' found in PATH")
|
||||||
|
|
||||||
|
|
||||||
def _find_compose_file(project_dir: str) -> Optional[Path]:
|
def _find_compose_file(project_dir: str) -> Path | None:
|
||||||
"""
|
"""
|
||||||
Detect a compose file in `project_dir` (case-insensitive).
|
Detect a compose file in `project_dir` (case-insensitive).
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
import re
|
import re
|
||||||
import logging
|
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
import pandas
|
import pandas
|
||||||
|
|
||||||
@@ -22,7 +21,7 @@ def get_instance(container: str, database_containers: list[str]) -> str:
|
|||||||
return re.split(r"(_|-)(database|db|postgres)", container)[0]
|
return re.split(r"(_|-)(database|db|postgres)", container)[0]
|
||||||
|
|
||||||
|
|
||||||
def _validate_database_value(value: Optional[str], *, instance: str) -> str:
|
def _validate_database_value(value: str | None, *, instance: str) -> str:
|
||||||
"""
|
"""
|
||||||
Enforce explicit database semantics:
|
Enforce explicit database semantics:
|
||||||
|
|
||||||
@@ -70,7 +69,7 @@ def backup_database(
|
|||||||
container: str,
|
container: str,
|
||||||
volume_dir: str,
|
volume_dir: str,
|
||||||
db_type: str,
|
db_type: str,
|
||||||
databases_df: "pandas.DataFrame",
|
databases_df: pandas.DataFrame,
|
||||||
database_containers: list[str],
|
database_containers: list[str],
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""
|
"""
|
||||||
@@ -97,7 +96,6 @@ def backup_database(
|
|||||||
|
|
||||||
db_value = _validate_database_value(raw_db, instance=instance_name)
|
db_value = _validate_database_value(raw_db, instance=instance_name)
|
||||||
|
|
||||||
# Explicit: dump ALL databases
|
|
||||||
if db_value == "*":
|
if db_value == "*":
|
||||||
if db_type != "postgres":
|
if db_type != "postgres":
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
@@ -110,7 +108,6 @@ def backup_database(
|
|||||||
produced = True
|
produced = True
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Concrete database dump
|
|
||||||
db_name = db_value
|
db_name = db_value
|
||||||
dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
|
dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
|
||||||
|
|
||||||
@@ -135,7 +132,6 @@ def backup_database(
|
|||||||
_atomic_write_cmd(cmd, dump_file)
|
_atomic_write_cmd(cmd, dump_file)
|
||||||
produced = True
|
produced = True
|
||||||
except BackupException as e:
|
except BackupException as e:
|
||||||
# Explicit DB dump failed -> hard error
|
|
||||||
raise BackupException(
|
raise BackupException(
|
||||||
f"Postgres dump failed for instance '{instance_name}', "
|
f"Postgres dump failed for instance '{instance_name}', "
|
||||||
f"database '{db_name}'. This database was explicitly configured "
|
f"database '{db_name}'. This database was explicitly configured "
|
||||||
|
|||||||
@@ -9,9 +9,29 @@ def get_image_info(container: str) -> str:
|
|||||||
)[0]
|
)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def image_name(container: str) -> str:
|
||||||
|
"""The image's repository path, without registry host, tag or digest.
|
||||||
|
|
||||||
|
A swarm node that hosts the local registry puts its own hostname in front
|
||||||
|
of every pull, so the raw reference of a Postgres container can read
|
||||||
|
`svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5`. Matching the
|
||||||
|
whole reference finds "mariadb" there and dumps the database with
|
||||||
|
mariadb-dump, which the Postgres image does not ship (exit 127). Tags bite
|
||||||
|
the same way: `xwiki_custom:lts-postgres-tomcat`.
|
||||||
|
"""
|
||||||
|
reference = get_image_info(container).strip().split("@", 1)[0]
|
||||||
|
head, _, tail = reference.rpartition("/")
|
||||||
|
tail = tail.split(":", 1)[0]
|
||||||
|
if head:
|
||||||
|
registry = head.split("/", 1)[0]
|
||||||
|
if "." in registry or ":" in registry or registry == "localhost":
|
||||||
|
head = head.partition("/")[2]
|
||||||
|
return f"{head}/{tail}" if head else tail
|
||||||
|
|
||||||
|
|
||||||
def has_image(container: str, pattern: str) -> bool:
|
def has_image(container: str, pattern: str) -> bool:
|
||||||
"""Return True if container's image contains the pattern."""
|
"""Return True if the container's image name contains the pattern."""
|
||||||
return pattern in get_image_info(container)
|
return pattern in image_name(container)
|
||||||
|
|
||||||
|
|
||||||
def docker_volume_names() -> list[str]:
|
def docker_volume_names() -> list[str]:
|
||||||
@@ -78,5 +98,5 @@ def docker_volume_exists(volume: str) -> bool:
|
|||||||
f"docker volume inspect {volume} >/dev/null 2>&1 && echo OK"
|
f"docker volume inspect {volume} >/dev/null 2>&1 && echo OK"
|
||||||
)
|
)
|
||||||
return True
|
return True
|
||||||
except Exception:
|
except BackupException:
|
||||||
return False
|
return False
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ def backup_mariadb_or_postgres(
|
|||||||
*,
|
*,
|
||||||
container: str,
|
container: str,
|
||||||
volume_dir: str,
|
volume_dir: str,
|
||||||
databases_df: "pandas.DataFrame",
|
databases_df: pandas.DataFrame,
|
||||||
database_containers: list[str],
|
database_containers: list[str],
|
||||||
) -> tuple[bool, bool]:
|
) -> tuple[bool, bool]:
|
||||||
"""
|
"""
|
||||||
@@ -34,7 +34,7 @@ def backup_mariadb_or_postgres(
|
|||||||
return False, False
|
return False, False
|
||||||
|
|
||||||
|
|
||||||
def _empty_databases_df() -> "pandas.DataFrame":
|
def _empty_databases_df() -> pandas.DataFrame:
|
||||||
"""
|
"""
|
||||||
Create an empty DataFrame with the expected schema for databases.csv.
|
Create an empty DataFrame with the expected schema for databases.csv.
|
||||||
|
|
||||||
@@ -44,7 +44,7 @@ def _empty_databases_df() -> "pandas.DataFrame":
|
|||||||
return pandas.DataFrame(columns=["instance", "database", "username", "password"])
|
return pandas.DataFrame(columns=["instance", "database", "username", "password"])
|
||||||
|
|
||||||
|
|
||||||
def load_databases_df(csv_path: str) -> "pandas.DataFrame":
|
def load_databases_df(csv_path: str) -> pandas.DataFrame:
|
||||||
"""
|
"""
|
||||||
Load databases.csv robustly.
|
Load databases.csv robustly.
|
||||||
|
|
||||||
@@ -74,7 +74,7 @@ def backup_dumps_for_volume(
|
|||||||
*,
|
*,
|
||||||
containers: list[str],
|
containers: list[str],
|
||||||
vol_dir: str,
|
vol_dir: str,
|
||||||
databases_df: "pandas.DataFrame",
|
databases_df: pandas.DataFrame,
|
||||||
database_containers: list[str],
|
database_containers: list[str],
|
||||||
) -> tuple[bool, bool]:
|
) -> tuple[bool, bool]:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import pathlib
|
|||||||
|
|
||||||
from dirval import create_stamp_file
|
from dirval import create_stamp_file
|
||||||
|
|
||||||
from .shell import execute_shell_command
|
from .shell import BackupException, execute_shell_command
|
||||||
|
|
||||||
|
|
||||||
def get_machine_id() -> str:
|
def get_machine_id() -> str:
|
||||||
@@ -23,7 +23,14 @@ def stamp_directory(version_dir: str) -> None:
|
|||||||
|
|
||||||
def create_version_directory(versions_dir: str, backup_time: str) -> str:
|
def create_version_directory(versions_dir: str, backup_time: str) -> str:
|
||||||
version_dir = os.path.join(versions_dir, backup_time)
|
version_dir = os.path.join(versions_dir, backup_time)
|
||||||
pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
|
try:
|
||||||
|
pathlib.Path(version_dir).mkdir(parents=True)
|
||||||
|
except FileExistsError:
|
||||||
|
raise BackupException(
|
||||||
|
f"generation {backup_time} already exists at {version_dir}; "
|
||||||
|
"another run claimed this second - refusing to write into it, "
|
||||||
|
"since rsync --delete would overwrite that generation"
|
||||||
|
) from None
|
||||||
return version_dir
|
return version_dir
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -49,7 +49,10 @@ def backup_volume(
|
|||||||
link_dest = f"--link-dest='{last}'" if last else ""
|
link_dest = f"--link-dest='{last}'" if last else ""
|
||||||
verify = "--checksum " if authoritative else ""
|
verify = "--checksum " if authoritative else ""
|
||||||
|
|
||||||
cmd = f"rsync -abP --delete --delete-excluded {verify}{link_dest} {source} {dest}"
|
cmd = (
|
||||||
|
f"rsync -aP --no-D --delete --delete-excluded "
|
||||||
|
f"{verify}{link_dest} {source} {dest}"
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
execute_shell_command(cmd)
|
execute_shell_command(cmd)
|
||||||
|
|||||||
@@ -3,10 +3,11 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from .paths import BackupPaths
|
from .db.cluster import restore_cluster_sql
|
||||||
from .files import restore_volume_files
|
|
||||||
from .db.postgres import restore_postgres_sql
|
|
||||||
from .db.mariadb import restore_mariadb_sql
|
from .db.mariadb import restore_mariadb_sql
|
||||||
|
from .db.postgres import restore_postgres_sql
|
||||||
|
from .files import restore_volume_files
|
||||||
|
from .paths import BackupPaths
|
||||||
|
|
||||||
|
|
||||||
def _add_common_backup_args(p: argparse.ArgumentParser) -> None:
|
def _add_common_backup_args(p: argparse.ArgumentParser) -> None:
|
||||||
@@ -33,9 +34,6 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
)
|
)
|
||||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
# files
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
p_files = sub.add_parser("files", help="Restore files into a docker volume")
|
p_files = sub.add_parser("files", help="Restore files into a docker volume")
|
||||||
_add_common_backup_args(p_files)
|
_add_common_backup_args(p_files)
|
||||||
p_files.add_argument(
|
p_files.add_argument(
|
||||||
@@ -48,9 +46,6 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
# postgres
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
p_pg = sub.add_parser("postgres", help="Restore a single PostgreSQL database dump")
|
p_pg = sub.add_parser("postgres", help="Restore a single PostgreSQL database dump")
|
||||||
_add_common_backup_args(p_pg)
|
_add_common_backup_args(p_pg)
|
||||||
p_pg.add_argument("--container", required=True)
|
p_pg.add_argument("--container", required=True)
|
||||||
@@ -59,9 +54,24 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
p_pg.add_argument("--db-password", required=True)
|
p_pg.add_argument("--db-password", required=True)
|
||||||
p_pg.add_argument("--empty", action="store_true")
|
p_pg.add_argument("--empty", action="store_true")
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
p_cluster = sub.add_parser(
|
||||||
# mariadb
|
"cluster", help="Restore a full PostgreSQL cluster dump (pg_dumpall)"
|
||||||
# ------------------------------------------------------------------
|
)
|
||||||
|
_add_common_backup_args(p_cluster)
|
||||||
|
p_cluster.add_argument("--container", required=True)
|
||||||
|
p_cluster.add_argument(
|
||||||
|
"--instance",
|
||||||
|
required=True,
|
||||||
|
help="Instance the dump was taken from; names <instance>.cluster.backup.sql",
|
||||||
|
)
|
||||||
|
p_cluster.add_argument(
|
||||||
|
"--db-user",
|
||||||
|
required=True,
|
||||||
|
help="Superuser of the instance; the dump creates roles and databases",
|
||||||
|
)
|
||||||
|
p_cluster.add_argument("--db-password", required=True)
|
||||||
|
p_cluster.add_argument("--empty", action="store_true")
|
||||||
|
|
||||||
p_mdb = sub.add_parser(
|
p_mdb = sub.add_parser(
|
||||||
"mariadb", help="Restore a single MariaDB/MySQL-compatible dump"
|
"mariadb", help="Restore a single MariaDB/MySQL-compatible dump"
|
||||||
)
|
)
|
||||||
@@ -76,8 +86,6 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
if args.cmd == "files":
|
if args.cmd == "files":
|
||||||
# target volume = args.volume_name
|
|
||||||
# source volume (backup key) defaults to target volume
|
|
||||||
source_volume = args.source_volume or args.volume_name
|
source_volume = args.source_volume or args.volume_name
|
||||||
|
|
||||||
bp_files = BackupPaths(
|
bp_files = BackupPaths(
|
||||||
@@ -111,6 +119,22 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
)
|
)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
if args.cmd == "cluster":
|
||||||
|
restore_cluster_sql(
|
||||||
|
container=args.container,
|
||||||
|
user=args.db_user,
|
||||||
|
password=args.db_password,
|
||||||
|
sql_path=BackupPaths(
|
||||||
|
args.volume_name,
|
||||||
|
args.backup_hash,
|
||||||
|
args.version,
|
||||||
|
repo_name=args.repo_name,
|
||||||
|
backups_dir=args.backups_dir,
|
||||||
|
).cluster_file(args.instance),
|
||||||
|
empty=args.empty,
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
if args.cmd == "mariadb":
|
if args.cmd == "mariadb":
|
||||||
user = args.db_user or args.db_name
|
user = args.db_user or args.db_name
|
||||||
restore_mariadb_sql(
|
restore_mariadb_sql(
|
||||||
@@ -132,7 +156,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
parser.error("Unhandled command")
|
parser.error("Unhandled command")
|
||||||
return 2
|
return 2
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e: # noqa: BLE001 - CLI boundary: any failure becomes exit 1
|
||||||
print(f"ERROR: {e}", file=sys.stderr)
|
print(f"ERROR: {e}", file=sys.stderr)
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
|||||||
103
src/baudolo/restore/db/cluster.py
Normal file
103
src/baudolo/restore/db/cluster.py
Normal file
@@ -0,0 +1,103 @@
|
|||||||
|
"""Replay a full PostgreSQL cluster dump produced by ``pg_dumpall``.
|
||||||
|
|
||||||
|
The backup side writes one when a databases.csv row asks for every database of
|
||||||
|
an instance (``database = '*'``, see ``backup/db.py``). Until now nothing read
|
||||||
|
it back, so that dump was stored and unrestorable - a format whose producer has
|
||||||
|
no consumer.
|
||||||
|
|
||||||
|
A cluster stream differs from a single-database one in three ways that decide
|
||||||
|
the implementation:
|
||||||
|
|
||||||
|
* it recreates roles and databases, so it must be replayed against the control
|
||||||
|
database rather than into a target database;
|
||||||
|
* ``CREATE DATABASE`` cannot run inside a transaction block, so unlike
|
||||||
|
:mod:`baudolo.restore.db.postgres` the replay must not be wrapped in
|
||||||
|
``--single-transaction``;
|
||||||
|
* it is replayed as a superuser, so the superuser-only statements that the
|
||||||
|
single-database path filters out are exactly the ones that have to survive.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
from collections.abc import Iterable, Iterator
|
||||||
|
|
||||||
|
from ..run import docker_exec
|
||||||
|
|
||||||
|
CONTROL_DB = "postgres"
|
||||||
|
_CLUSTER_PRECLEAN_SQL = os.path.join(os.path.dirname(__file__), "cluster_preclean.sql")
|
||||||
|
_CREATE_ROLE = re.compile(rb'^CREATE ROLE "?([^";]+)"?;\s*$')
|
||||||
|
|
||||||
|
|
||||||
|
def _psql(user: str) -> list[str]:
|
||||||
|
"""The replay client: no --single-transaction, CREATE DATABASE forbids it."""
|
||||||
|
return ["psql", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", CONTROL_DB]
|
||||||
|
|
||||||
|
|
||||||
|
def filter_own_role_creation(lines: Iterable[bytes], user: str) -> Iterator[bytes]:
|
||||||
|
"""Drop the ``CREATE ROLE`` of the role holding this session.
|
||||||
|
|
||||||
|
A pg_dumpall stream recreates every role of the cluster, the bootstrap
|
||||||
|
superuser included, and the pre-clean cannot drop the one it is connected
|
||||||
|
as - so that single statement always collides. Its ``ALTER ROLE`` is kept:
|
||||||
|
that is what re-applies the attributes and the password the dump captured.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
lines: dump lines including their trailing newlines.
|
||||||
|
user: the connecting role.
|
||||||
|
|
||||||
|
Yields:
|
||||||
|
Every line except that one CREATE.
|
||||||
|
"""
|
||||||
|
for line in lines:
|
||||||
|
found = _CREATE_ROLE.match(line)
|
||||||
|
if found and found.group(1).decode() == user:
|
||||||
|
continue
|
||||||
|
yield line
|
||||||
|
|
||||||
|
|
||||||
|
def restore_cluster_sql(
|
||||||
|
*,
|
||||||
|
container: str,
|
||||||
|
user: str,
|
||||||
|
password: str,
|
||||||
|
sql_path: str,
|
||||||
|
empty: bool,
|
||||||
|
) -> None:
|
||||||
|
"""Replay a pg_dumpall stream into a running instance.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
container: the running engine to replay into.
|
||||||
|
user: a superuser of that instance; the dump creates roles and
|
||||||
|
databases, which an application role may not do.
|
||||||
|
password: its password, handed to psql through the container's env.
|
||||||
|
sql_path: the ``<instance>.cluster.backup.sql`` of a generation.
|
||||||
|
empty: drop the cluster's databases and roles first. Without it the
|
||||||
|
replay stops at the first object that already exists, which is the
|
||||||
|
honest outcome: recreating a cluster over a populated one is a
|
||||||
|
decision, not a default.
|
||||||
|
"""
|
||||||
|
if not os.path.isfile(sql_path):
|
||||||
|
raise FileNotFoundError(sql_path)
|
||||||
|
|
||||||
|
docker_env = {"PGPASSWORD": password}
|
||||||
|
|
||||||
|
if empty:
|
||||||
|
with open(_CLUSTER_PRECLEAN_SQL, encoding="utf-8") as preclean:
|
||||||
|
drop_sql = preclean.read()
|
||||||
|
docker_exec(
|
||||||
|
container,
|
||||||
|
_psql(user),
|
||||||
|
stdin=drop_sql.encode(),
|
||||||
|
docker_env=docker_env,
|
||||||
|
)
|
||||||
|
|
||||||
|
with open(sql_path, "rb") as src, tempfile.TemporaryFile() as filtered:
|
||||||
|
for line in filter_own_role_creation(src, user):
|
||||||
|
filtered.write(line)
|
||||||
|
filtered.seek(0)
|
||||||
|
docker_exec(container, _psql(user), stdin=filtered, docker_env=docker_env)
|
||||||
|
|
||||||
|
print(f"PostgreSQL cluster restore complete from '{os.path.basename(sql_path)}'.")
|
||||||
30
src/baudolo/restore/db/cluster_preclean.sql
Normal file
30
src/baudolo/restore/db/cluster_preclean.sql
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
-- Pre-clean for `restore cluster --empty`. A pg_dumpall stream recreates roles
|
||||||
|
-- and databases, so replaying it into a populated cluster dies on the first
|
||||||
|
-- CREATE ROLE. Emitted as one DROP per row and run via \gexec so each executes
|
||||||
|
-- as its own top-level statement: DROP DATABASE cannot run inside a
|
||||||
|
-- transaction block, which rules out a single DO block.
|
||||||
|
-- The phase column pins the order: databases must be gone before their owners
|
||||||
|
-- can be dropped, and DROP OWNED BY releases what a role still holds in the
|
||||||
|
-- control database. Template databases, the control database itself, the pg_*
|
||||||
|
-- system roles and the connecting role are kept - the dump does not recreate
|
||||||
|
-- them and dropping them would end the session.
|
||||||
|
SELECT statement
|
||||||
|
FROM (
|
||||||
|
SELECT 1 AS phase,
|
||||||
|
format('DROP DATABASE IF EXISTS %I', datname) AS statement
|
||||||
|
FROM pg_database
|
||||||
|
WHERE NOT datistemplate
|
||||||
|
AND datname <> current_database()
|
||||||
|
UNION ALL
|
||||||
|
SELECT 2, format('DROP OWNED BY %I', rolname)
|
||||||
|
FROM pg_roles
|
||||||
|
WHERE NOT starts_with(rolname, 'pg_')
|
||||||
|
AND rolname <> current_user
|
||||||
|
UNION ALL
|
||||||
|
SELECT 3, format('DROP ROLE IF EXISTS %I', rolname)
|
||||||
|
FROM pg_roles
|
||||||
|
WHERE NOT starts_with(rolname, 'pg_')
|
||||||
|
AND rolname <> current_user
|
||||||
|
) drops
|
||||||
|
ORDER BY phase
|
||||||
|
\gexec
|
||||||
@@ -22,11 +22,11 @@ exit 42
|
|||||||
if not out:
|
if not out:
|
||||||
raise RuntimeError("empty client detection output")
|
raise RuntimeError("empty client detection output")
|
||||||
return out
|
return out
|
||||||
except Exception as e:
|
except Exception:
|
||||||
print(
|
print(
|
||||||
"ERROR: neither 'mariadb' nor 'mysql' found in container.", file=sys.stderr
|
"ERROR: neither 'mariadb' nor 'mysql' found in container.", file=sys.stderr
|
||||||
)
|
)
|
||||||
raise e
|
raise
|
||||||
|
|
||||||
|
|
||||||
def restore_mariadb_sql(
|
def restore_mariadb_sql(
|
||||||
@@ -44,9 +44,7 @@ def restore_mariadb_sql(
|
|||||||
raise FileNotFoundError(sql_path)
|
raise FileNotFoundError(sql_path)
|
||||||
|
|
||||||
if empty:
|
if empty:
|
||||||
# IMPORTANT:
|
# Do not hardcode 'mysql': MariaDB 11 images may not ship that binary.
|
||||||
# Do NOT hardcode 'mysql' here. Use the detected client.
|
|
||||||
# MariaDB 11 images may not contain the mysql binary at all.
|
|
||||||
result = docker_exec(
|
result = docker_exec(
|
||||||
container,
|
container,
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -50,7 +50,6 @@ def restore_postgres_sql(
|
|||||||
if not os.path.isfile(sql_path):
|
if not os.path.isfile(sql_path):
|
||||||
raise FileNotFoundError(sql_path)
|
raise FileNotFoundError(sql_path)
|
||||||
|
|
||||||
# Make password available INSIDE the container for psql.
|
|
||||||
docker_env = {"PGPASSWORD": password}
|
docker_env = {"PGPASSWORD": password}
|
||||||
|
|
||||||
if empty:
|
if empty:
|
||||||
|
|||||||
@@ -27,3 +27,7 @@ class BackupPaths:
|
|||||||
|
|
||||||
def sql_file(self, db_name: str) -> str:
|
def sql_file(self, db_name: str) -> str:
|
||||||
return os.path.join(self.root(), "sql", f"{db_name}.backup.sql")
|
return os.path.join(self.root(), "sql", f"{db_name}.backup.sql")
|
||||||
|
|
||||||
|
def cluster_file(self, instance: str) -> str:
|
||||||
|
"""The pg_dumpall stream a `database = '*'` row produces."""
|
||||||
|
return os.path.join(self.root(), "sql", f"{instance}.cluster.backup.sql")
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
|
|
||||||
def run(
|
def run(
|
||||||
@@ -10,7 +9,7 @@ def run(
|
|||||||
*,
|
*,
|
||||||
stdin=None,
|
stdin=None,
|
||||||
capture: bool = False,
|
capture: bool = False,
|
||||||
env: Optional[dict] = None,
|
env: dict | None = None,
|
||||||
) -> subprocess.CompletedProcess:
|
) -> subprocess.CompletedProcess:
|
||||||
try:
|
try:
|
||||||
kwargs: dict = {
|
kwargs: dict = {
|
||||||
@@ -26,21 +25,18 @@ def run(
|
|||||||
else:
|
else:
|
||||||
kwargs["stdin"] = stdin
|
kwargs["stdin"] = stdin
|
||||||
|
|
||||||
return subprocess.run(cmd, **kwargs)
|
return subprocess.run(cmd, **kwargs) # noqa: PLW1510 - check lives in kwargs
|
||||||
|
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
msg = f"ERROR: command failed ({e.returncode}): {' '.join(cmd)}"
|
msg = f"ERROR: command failed ({e.returncode}): {' '.join(cmd)}"
|
||||||
print(msg, file=sys.stderr)
|
print(msg, file=sys.stderr)
|
||||||
if e.stdout:
|
for stream in (e.stdout, e.stderr):
|
||||||
|
if not stream:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
print(e.stdout.decode(), file=sys.stderr)
|
print(stream.decode(), file=sys.stderr)
|
||||||
except Exception:
|
except (UnicodeDecodeError, AttributeError):
|
||||||
print(e.stdout, file=sys.stderr)
|
print(stream, file=sys.stderr)
|
||||||
if e.stderr:
|
|
||||||
try:
|
|
||||||
print(e.stderr.decode(), file=sys.stderr)
|
|
||||||
except Exception:
|
|
||||||
print(e.stderr, file=sys.stderr)
|
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
@@ -50,8 +46,8 @@ def docker_exec(
|
|||||||
*,
|
*,
|
||||||
stdin=None,
|
stdin=None,
|
||||||
capture: bool = False,
|
capture: bool = False,
|
||||||
env: Optional[dict] = None,
|
env: dict | None = None,
|
||||||
docker_env: Optional[dict[str, str]] = None,
|
docker_env: dict[str, str] | None = None,
|
||||||
) -> subprocess.CompletedProcess:
|
) -> subprocess.CompletedProcess:
|
||||||
cmd: list[str] = ["docker", "exec", "-i"]
|
cmd: list[str] = ["docker", "exec", "-i"]
|
||||||
if docker_env:
|
if docker_env:
|
||||||
@@ -67,8 +63,8 @@ def docker_exec_sh(
|
|||||||
*,
|
*,
|
||||||
stdin=None,
|
stdin=None,
|
||||||
capture: bool = False,
|
capture: bool = False,
|
||||||
env: Optional[dict] = None,
|
env: dict | None = None,
|
||||||
docker_env: Optional[dict[str, str]] = None,
|
docker_env: dict[str, str] | None = None,
|
||||||
) -> subprocess.CompletedProcess:
|
) -> subprocess.CompletedProcess:
|
||||||
return docker_exec(
|
return docker_exec(
|
||||||
container,
|
container,
|
||||||
@@ -85,5 +81,6 @@ def docker_volume_exists(volume: str) -> bool:
|
|||||||
["docker", "volume", "inspect", volume],
|
["docker", "volume", "inspect", volume],
|
||||||
stdout=subprocess.DEVNULL,
|
stdout=subprocess.DEVNULL,
|
||||||
stderr=subprocess.DEVNULL,
|
stderr=subprocess.DEVNULL,
|
||||||
|
check=False,
|
||||||
)
|
)
|
||||||
return p.returncode == 0
|
return p.returncode == 0
|
||||||
|
|||||||
@@ -1,18 +1,17 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from typing import Optional
|
|
||||||
from pandas.errors import EmptyDataError
|
from pandas.errors import EmptyDataError
|
||||||
|
|
||||||
DB_NAME_RE = re.compile(r"^[a-zA-Z0-9_][a-zA-Z0-9_-]*$")
|
DB_NAME_RE = re.compile(r"^[a-zA-Z0-9_][a-zA-Z0-9_-]*$")
|
||||||
|
|
||||||
|
|
||||||
def _validate_database_value(value: Optional[str], *, instance: str) -> str:
|
def _validate_database_value(value: str | None, *, instance: str) -> str:
|
||||||
v = (value or "").strip()
|
v = (value or "").strip()
|
||||||
if v == "":
|
if v == "":
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
@@ -40,7 +39,7 @@ def _empty_df() -> pd.DataFrame:
|
|||||||
def check_and_add_entry(
|
def check_and_add_entry(
|
||||||
file_path: str,
|
file_path: str,
|
||||||
instance: str,
|
instance: str,
|
||||||
database: Optional[str],
|
database: str | None,
|
||||||
username: str,
|
username: str,
|
||||||
password: str,
|
password: str,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -108,7 +107,7 @@ def main() -> None:
|
|||||||
username=args.username,
|
username=args.username,
|
||||||
password=args.password,
|
password=args.password,
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc: # noqa: BLE001 - CLI boundary: any failure becomes exit 1
|
||||||
print(f"ERROR: {exc}", file=sys.stderr)
|
print(f"ERROR: {exc}", file=sys.stderr)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""Shared e2e helpers, re-exported so tests import one name."""
|
"""Shared e2e helpers, re-exported so tests import one name."""
|
||||||
|
|
||||||
from .fixtures import * # noqa: F401,F403
|
from .fixtures import *
|
||||||
from .process import * # noqa: F401,F403
|
from .process import *
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
"""Fixtures and paths the e2e suite builds its scenarios from."""
|
"""Fixtures and paths the e2e suite builds its scenarios from."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import shutil
|
import shutil
|
||||||
@@ -96,8 +97,7 @@ def write_databases_csv(path: str, rows: list[tuple[str, str, str, str]]) -> Non
|
|||||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
with open(path, "w", encoding="utf-8") as f:
|
with open(path, "w", encoding="utf-8") as f:
|
||||||
f.write("instance;database;username;password\n")
|
f.write("instance;database;username;password\n")
|
||||||
for inst, db, user, pw in rows:
|
f.writelines(f"{inst};{db};{user};{pw}\n" for inst, db, user, pw in rows)
|
||||||
f.write(f"{inst};{db};{user};{pw}\n")
|
|
||||||
|
|
||||||
|
|
||||||
def cleanup_docker(*, containers: list[str], volumes: list[str]) -> None:
|
def cleanup_docker(*, containers: list[str], volumes: list[str]) -> None:
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
"""Process, docker and readiness helpers for the e2e suite."""
|
"""Process, docker and readiness helpers for the e2e suite."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|||||||
@@ -12,8 +12,8 @@ import sys
|
|||||||
|
|
||||||
sys.path.insert(0, "/src")
|
sys.path.insert(0, "/src")
|
||||||
|
|
||||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot # noqa: E402
|
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
||||||
from baudolo.backup.volume import backup_volume # noqa: E402
|
from baudolo.backup.volume import backup_volume
|
||||||
|
|
||||||
SUBJECT = "/subject/docker"
|
SUBJECT = "/subject/docker"
|
||||||
VOLUME = "mariadb_data"
|
VOLUME = "mariadb_data"
|
||||||
@@ -23,9 +23,13 @@ GENERATION = f"{VERSIONS}/20260731"
|
|||||||
|
|
||||||
|
|
||||||
def shell(command: str) -> list[str]:
|
def shell(command: str) -> list[str]:
|
||||||
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
proc = subprocess.run(
|
||||||
|
command, shell=True, capture_output=True, text=True, check=False
|
||||||
|
)
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
raise SnapshotError(f"{command} exited {proc.returncode}: {proc.stderr.strip()}")
|
raise SnapshotError(
|
||||||
|
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||||
|
)
|
||||||
return proc.stdout.splitlines()
|
return proc.stdout.splitlines()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from pathlib import Path
|
|||||||
|
|
||||||
sys.path.insert(0, "/src")
|
sys.path.insert(0, "/src")
|
||||||
|
|
||||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot # noqa: E402
|
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
||||||
|
|
||||||
KIND = sys.argv[1]
|
KIND = sys.argv[1]
|
||||||
SUBJECT = sys.argv[2]
|
SUBJECT = sys.argv[2]
|
||||||
@@ -20,9 +20,13 @@ EXPECT = sys.argv[3]
|
|||||||
|
|
||||||
|
|
||||||
def shell(command: str) -> list[str]:
|
def shell(command: str) -> list[str]:
|
||||||
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
proc = subprocess.run(
|
||||||
|
command, shell=True, capture_output=True, text=True, check=False
|
||||||
|
)
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
raise SnapshotError(f"{command} exited {proc.returncode}: {proc.stderr.strip()}")
|
raise SnapshotError(
|
||||||
|
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||||
|
)
|
||||||
return proc.stdout.splitlines()
|
return proc.stdout.splitlines()
|
||||||
|
|
||||||
|
|
||||||
@@ -55,5 +59,8 @@ with volume_snapshot(KIND, SUBJECT, "e2e", run=shell) as resolve:
|
|||||||
|
|
||||||
root = Path(resolve(SUBJECT))
|
root = Path(resolve(SUBJECT))
|
||||||
|
|
||||||
check("the snapshot is removed afterwards", not root.exists() or not (root / "volumes").exists())
|
check(
|
||||||
|
"the snapshot is removed afterwards",
|
||||||
|
not root.exists() or not (root / "volumes").exists(),
|
||||||
|
)
|
||||||
print("ALL OK", flush=True)
|
print("ALL OK", flush=True)
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
# tests/e2e/test_e2e_dump_only_fallback_to_files.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
POSTGRES_IMAGE,
|
|
||||||
POSTGRES_DATA_DIR,
|
POSTGRES_DATA_DIR,
|
||||||
|
POSTGRES_IMAGE,
|
||||||
backup_path,
|
backup_path,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
@@ -12,8 +11,8 @@ from .helpers import (
|
|||||||
require_docker,
|
require_docker,
|
||||||
run,
|
run,
|
||||||
unique,
|
unique,
|
||||||
write_databases_csv,
|
|
||||||
wait_for_postgres,
|
wait_for_postgres,
|
||||||
|
write_databases_csv,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -37,7 +36,6 @@ class TestE2EDumpOnlyFallbackToFiles(unittest.TestCase):
|
|||||||
|
|
||||||
run(["docker", "volume", "create", cls.pg_volume])
|
run(["docker", "volume", "create", cls.pg_volume])
|
||||||
|
|
||||||
# Start Postgres (creates a real DB volume)
|
|
||||||
run(
|
run(
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
POSTGRES_IMAGE,
|
|
||||||
POSTGRES_DATA_DIR,
|
POSTGRES_DATA_DIR,
|
||||||
|
POSTGRES_IMAGE,
|
||||||
backup_path,
|
backup_path,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
@@ -35,7 +35,6 @@ class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
|
|||||||
cls.containers: list[str] = []
|
cls.containers: list[str] = []
|
||||||
cls.volumes = [cls.db_volume, cls.files_volume]
|
cls.volumes = [cls.db_volume, cls.files_volume]
|
||||||
|
|
||||||
# Create volumes
|
|
||||||
run(["docker", "volume", "create", cls.db_volume])
|
run(["docker", "volume", "create", cls.db_volume])
|
||||||
run(["docker", "volume", "create", cls.files_volume])
|
run(["docker", "volume", "create", cls.files_volume])
|
||||||
|
|
||||||
@@ -114,7 +113,6 @@ class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
|
|||||||
[(cls.pg_container, cls.pg_db, cls.pg_user, cls.pg_password)],
|
[(cls.pg_container, cls.pg_db, cls.pg_user, cls.pg_password)],
|
||||||
)
|
)
|
||||||
|
|
||||||
# Run baudolo with dump-only-sql
|
|
||||||
cmd = [
|
cmd = [
|
||||||
"baudolo",
|
"baudolo",
|
||||||
"--compose-dir",
|
"--compose-dir",
|
||||||
|
|||||||
105
tests/e2e/test_e2e_engine_detection_registry_prefix.py
Normal file
105
tests/e2e/test_e2e_engine_detection_registry_prefix.py
Normal file
@@ -0,0 +1,105 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from .helpers import (
|
||||||
|
POSTGRES_DATA_DIR,
|
||||||
|
POSTGRES_IMAGE,
|
||||||
|
backup_path,
|
||||||
|
backup_run,
|
||||||
|
cleanup_docker,
|
||||||
|
create_minimal_compose_dir,
|
||||||
|
ensure_empty_dir,
|
||||||
|
latest_version_dir,
|
||||||
|
require_docker,
|
||||||
|
run,
|
||||||
|
unique,
|
||||||
|
wait_for_postgres,
|
||||||
|
write_databases_csv,
|
||||||
|
)
|
||||||
|
|
||||||
|
REGISTRY_HOST = "svc-db-mariadb-swarm-mgr-01:5000"
|
||||||
|
|
||||||
|
|
||||||
|
class TestE2EEngineDetectionRegistryPrefix(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
require_docker()
|
||||||
|
cls.prefix = unique("baudolo-e2e-registry-prefix")
|
||||||
|
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||||
|
ensure_empty_dir(cls.backups_dir)
|
||||||
|
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||||
|
cls.repo_name = cls.prefix
|
||||||
|
|
||||||
|
cls.image = f"{REGISTRY_HOST}/postgres_custom:17-3.5"
|
||||||
|
cls.pg_container = f"{cls.prefix}-pg"
|
||||||
|
cls.pg_volume = f"{cls.prefix}-pg-vol"
|
||||||
|
cls.containers = [cls.pg_container]
|
||||||
|
cls.volumes = [cls.pg_volume]
|
||||||
|
|
||||||
|
run(["docker", "pull", POSTGRES_IMAGE])
|
||||||
|
run(["docker", "tag", POSTGRES_IMAGE, cls.image])
|
||||||
|
run(["docker", "volume", "create", cls.pg_volume])
|
||||||
|
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"-d",
|
||||||
|
"--name",
|
||||||
|
cls.pg_container,
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_PASSWORD=pgpw",
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_DB=appdb",
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_USER=postgres",
|
||||||
|
"-v",
|
||||||
|
f"{cls.pg_volume}:{POSTGRES_DATA_DIR}",
|
||||||
|
cls.image,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||||
|
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
cls.pg_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
"psql -U postgres -d appdb -c \"CREATE TABLE t (id int primary key, v text); INSERT INTO t VALUES (1,'ok');\"",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
|
write_databases_csv(
|
||||||
|
cls.databases_csv, [(cls.pg_container, "appdb", "postgres", "pgpw")]
|
||||||
|
)
|
||||||
|
|
||||||
|
backup_run(
|
||||||
|
backups_dir=cls.backups_dir,
|
||||||
|
repo_name=cls.repo_name,
|
||||||
|
compose_dir=cls.compose_dir,
|
||||||
|
databases_csv=cls.databases_csv,
|
||||||
|
database_containers=[cls.pg_container],
|
||||||
|
images_no_stop_required=[cls.image],
|
||||||
|
)
|
||||||
|
|
||||||
|
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls) -> None:
|
||||||
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
run(["docker", "rmi", cls.image], check=False)
|
||||||
|
|
||||||
|
def test_the_registry_host_does_not_pick_the_engine(self) -> None:
|
||||||
|
p = (
|
||||||
|
backup_path(self.backups_dir, self.repo_name, self.version, self.pg_volume)
|
||||||
|
/ "sql"
|
||||||
|
/ "appdb.backup.sql"
|
||||||
|
)
|
||||||
|
self.assertTrue(p.is_file(), f"Expected a pg_dump at: {p}")
|
||||||
|
self.assertIn("Dumped by pg_dump", p.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -1,16 +1,16 @@
|
|||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
backup_run,
|
|
||||||
backup_path,
|
backup_path,
|
||||||
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
ensure_empty_dir,
|
ensure_empty_dir,
|
||||||
latest_version_dir,
|
latest_version_dir,
|
||||||
require_docker,
|
require_docker,
|
||||||
|
run,
|
||||||
unique,
|
unique,
|
||||||
write_databases_csv,
|
write_databases_csv,
|
||||||
run,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -30,7 +30,6 @@ class TestE2EFilesFull(unittest.TestCase):
|
|||||||
cls.containers = []
|
cls.containers = []
|
||||||
cls.volumes = [cls.volume_src, cls.volume_dst]
|
cls.volumes = [cls.volume_src, cls.volume_dst]
|
||||||
|
|
||||||
# create source volume with a file
|
|
||||||
run(["docker", "volume", "create", cls.volume_src])
|
run(["docker", "volume", "create", cls.volume_src])
|
||||||
run(
|
run(
|
||||||
[
|
[
|
||||||
@@ -50,7 +49,6 @@ class TestE2EFilesFull(unittest.TestCase):
|
|||||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
write_databases_csv(cls.databases_csv, [])
|
write_databases_csv(cls.databases_csv, [])
|
||||||
|
|
||||||
# Run backup (files should be copied)
|
|
||||||
backup_run(
|
backup_run(
|
||||||
backups_dir=cls.backups_dir,
|
backups_dir=cls.backups_dir,
|
||||||
repo_name=cls.repo_name,
|
repo_name=cls.repo_name,
|
||||||
@@ -97,7 +95,6 @@ class TestE2EFilesFull(unittest.TestCase):
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
# verify restored file exists in dst volume
|
|
||||||
p = run(
|
p = run(
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
|
|||||||
@@ -1,16 +1,16 @@
|
|||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
backup_run,
|
|
||||||
backup_path,
|
backup_path,
|
||||||
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
ensure_empty_dir,
|
ensure_empty_dir,
|
||||||
latest_version_dir,
|
latest_version_dir,
|
||||||
require_docker,
|
require_docker,
|
||||||
|
run,
|
||||||
unique,
|
unique,
|
||||||
write_databases_csv,
|
write_databases_csv,
|
||||||
run,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -29,7 +29,6 @@ class TestE2EFilesNoCopy(unittest.TestCase):
|
|||||||
cls.containers: list[str] = []
|
cls.containers: list[str] = []
|
||||||
cls.volumes = [cls.volume_src]
|
cls.volumes = [cls.volume_src]
|
||||||
|
|
||||||
# Create source volume and write a marker file
|
|
||||||
run(["docker", "volume", "create", cls.volume_src])
|
run(["docker", "volume", "create", cls.volume_src])
|
||||||
run(
|
run(
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
# tests/e2e/test_e2e_images_no_backup_required_early_skip.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
@@ -34,11 +33,9 @@ class TestE2EImagesNoBackupRequiredEarlySkip(unittest.TestCase):
|
|||||||
cls.containers = [cls.redis_container]
|
cls.containers = [cls.redis_container]
|
||||||
cls.volumes = [cls.ignored_volume, cls.normal_volume]
|
cls.volumes = [cls.ignored_volume, cls.normal_volume]
|
||||||
|
|
||||||
# Create volumes
|
|
||||||
run(["docker", "volume", "create", cls.ignored_volume])
|
run(["docker", "volume", "create", cls.ignored_volume])
|
||||||
run(["docker", "volume", "create", cls.normal_volume])
|
run(["docker", "volume", "create", cls.normal_volume])
|
||||||
|
|
||||||
# Start redis container using the ignored volume
|
|
||||||
run(
|
run(
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
@@ -71,7 +68,6 @@ class TestE2EImagesNoBackupRequiredEarlySkip(unittest.TestCase):
|
|||||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
write_databases_csv(cls.databases_csv, [])
|
write_databases_csv(cls.databases_csv, [])
|
||||||
|
|
||||||
# Run baudolo with images-no-backup-required redis
|
|
||||||
cmd = [
|
cmd = [
|
||||||
"baudolo",
|
"baudolo",
|
||||||
"--compose-dir",
|
"--compose-dir",
|
||||||
|
|||||||
@@ -30,8 +30,8 @@ import pandas
|
|||||||
from baudolo.backup import db as db_mod
|
from baudolo.backup import db as db_mod
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
MARIADB_IMAGE,
|
|
||||||
MARIADB_DATA_DIR,
|
MARIADB_DATA_DIR,
|
||||||
|
MARIADB_IMAGE,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
require_docker,
|
require_docker,
|
||||||
run,
|
run,
|
||||||
|
|||||||
@@ -1,21 +1,20 @@
|
|||||||
# tests/e2e/test_e2e_mariadb_full.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
MARIADB_IMAGE,
|
|
||||||
MARIADB_DATA_DIR,
|
MARIADB_DATA_DIR,
|
||||||
backup_run,
|
MARIADB_IMAGE,
|
||||||
backup_path,
|
backup_path,
|
||||||
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
ensure_empty_dir,
|
ensure_empty_dir,
|
||||||
latest_version_dir,
|
latest_version_dir,
|
||||||
require_docker,
|
require_docker,
|
||||||
unique,
|
|
||||||
write_databases_csv,
|
|
||||||
run,
|
run,
|
||||||
|
unique,
|
||||||
wait_for_mariadb,
|
wait_for_mariadb,
|
||||||
wait_for_mariadb_sql,
|
wait_for_mariadb_sql,
|
||||||
|
write_databases_csv,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -71,7 +70,6 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
|||||||
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
||||||
)
|
)
|
||||||
|
|
||||||
# Create table + data via the dedicated user (TCP)
|
|
||||||
run(
|
run(
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
@@ -79,9 +77,11 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
|||||||
cls.db_container,
|
cls.db_container,
|
||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
(
|
||||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||||
|
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\""
|
||||||
|
),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -112,8 +112,10 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
|||||||
cls.db_container,
|
cls.db_container,
|
||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
(
|
||||||
f'-e "DROP TABLE {cls.db_name}.t;"',
|
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||||
|
f'-e "DROP TABLE {cls.db_name}.t;"'
|
||||||
|
),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -161,8 +163,10 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
|||||||
self.db_container,
|
self.db_container,
|
||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
(
|
||||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"',
|
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||||
|
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"'
|
||||||
|
),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||||
|
|||||||
@@ -1,21 +1,20 @@
|
|||||||
# tests/e2e/test_e2e_mariadb_no_copy.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
MARIADB_IMAGE,
|
|
||||||
MARIADB_DATA_DIR,
|
MARIADB_DATA_DIR,
|
||||||
backup_run,
|
MARIADB_IMAGE,
|
||||||
backup_path,
|
backup_path,
|
||||||
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
ensure_empty_dir,
|
ensure_empty_dir,
|
||||||
latest_version_dir,
|
latest_version_dir,
|
||||||
require_docker,
|
require_docker,
|
||||||
unique,
|
|
||||||
write_databases_csv,
|
|
||||||
run,
|
run,
|
||||||
|
unique,
|
||||||
wait_for_mariadb,
|
wait_for_mariadb,
|
||||||
wait_for_mariadb_sql,
|
wait_for_mariadb_sql,
|
||||||
|
write_databases_csv,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -69,7 +68,6 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
|||||||
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
||||||
)
|
)
|
||||||
|
|
||||||
# Create table + data (TCP)
|
|
||||||
run(
|
run(
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
@@ -77,9 +75,11 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
|||||||
cls.db_container,
|
cls.db_container,
|
||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
(
|
||||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||||
|
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\""
|
||||||
|
),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -110,8 +110,10 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
|||||||
cls.db_container,
|
cls.db_container,
|
||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
(
|
||||||
f'-e "DROP TABLE {cls.db_name}.t;"',
|
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||||
|
f'-e "DROP TABLE {cls.db_name}.t;"'
|
||||||
|
),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -158,8 +160,10 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
|||||||
self.db_container,
|
self.db_container,
|
||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
(
|
||||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"',
|
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||||
|
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"'
|
||||||
|
),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||||
|
|||||||
177
tests/e2e/test_e2e_postgres_cluster_restore.py
Normal file
177
tests/e2e/test_e2e_postgres_cluster_restore.py
Normal file
@@ -0,0 +1,177 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from .helpers import (
|
||||||
|
POSTGRES_DATA_DIR,
|
||||||
|
POSTGRES_IMAGE,
|
||||||
|
backup_path,
|
||||||
|
backup_run,
|
||||||
|
cleanup_docker,
|
||||||
|
create_minimal_compose_dir,
|
||||||
|
ensure_empty_dir,
|
||||||
|
latest_version_dir,
|
||||||
|
require_docker,
|
||||||
|
run,
|
||||||
|
unique,
|
||||||
|
wait_for_postgres,
|
||||||
|
write_databases_csv,
|
||||||
|
)
|
||||||
|
|
||||||
|
# One statement per entry: psql wraps a multi-statement -c in a transaction,
|
||||||
|
# and CREATE DATABASE is forbidden inside one.
|
||||||
|
SEED_SQL = (
|
||||||
|
"CREATE ROLE app LOGIN PASSWORD 'apppw'",
|
||||||
|
"CREATE DATABASE first OWNER app",
|
||||||
|
"CREATE DATABASE second OWNER app",
|
||||||
|
)
|
||||||
|
DROP_SQL = (
|
||||||
|
"DROP DATABASE first",
|
||||||
|
"DROP DATABASE second",
|
||||||
|
"DROP ROLE app",
|
||||||
|
)
|
||||||
|
FIRST_SQL = "CREATE TABLE t (v text); INSERT INTO t VALUES ('first-payload');"
|
||||||
|
SECOND_SQL = "CREATE TABLE t (v text); INSERT INTO t VALUES ('second-payload');"
|
||||||
|
|
||||||
|
|
||||||
|
class TestE2EPostgresClusterRestore(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
require_docker()
|
||||||
|
cls.prefix = unique("baudolo-e2e-pg-cluster")
|
||||||
|
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||||
|
ensure_empty_dir(cls.backups_dir)
|
||||||
|
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||||
|
cls.repo_name = cls.prefix
|
||||||
|
|
||||||
|
cls.pg_container = f"{cls.prefix}-pg"
|
||||||
|
cls.pg_volume = f"{cls.prefix}-pg-vol"
|
||||||
|
cls.containers = [cls.pg_container]
|
||||||
|
cls.volumes = [cls.pg_volume]
|
||||||
|
|
||||||
|
run(["docker", "volume", "create", cls.pg_volume])
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"-d",
|
||||||
|
"--name",
|
||||||
|
cls.pg_container,
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_PASSWORD=pgpw",
|
||||||
|
"-v",
|
||||||
|
f"{cls.pg_volume}:{POSTGRES_DATA_DIR}",
|
||||||
|
POSTGRES_IMAGE,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
wait_for_postgres(cls.pg_container, user="postgres")
|
||||||
|
|
||||||
|
for statement in SEED_SQL:
|
||||||
|
cls._psql("postgres", statement)
|
||||||
|
cls._psql("first", FIRST_SQL)
|
||||||
|
cls._psql("second", SECOND_SQL)
|
||||||
|
|
||||||
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
|
write_databases_csv(
|
||||||
|
cls.databases_csv, [(cls.pg_container, "*", "postgres", "pgpw")]
|
||||||
|
)
|
||||||
|
|
||||||
|
backup_run(
|
||||||
|
backups_dir=cls.backups_dir,
|
||||||
|
repo_name=cls.repo_name,
|
||||||
|
compose_dir=cls.compose_dir,
|
||||||
|
databases_csv=cls.databases_csv,
|
||||||
|
database_containers=[cls.pg_container],
|
||||||
|
images_no_stop_required=[POSTGRES_IMAGE],
|
||||||
|
)
|
||||||
|
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||||
|
cls.dump = (
|
||||||
|
backup_path(cls.backups_dir, cls.repo_name, cls.version, cls.pg_volume)
|
||||||
|
/ "sql"
|
||||||
|
/ f"{cls.pg_container}.cluster.backup.sql"
|
||||||
|
)
|
||||||
|
|
||||||
|
for statement in DROP_SQL:
|
||||||
|
cls._psql("postgres", statement)
|
||||||
|
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"baudolo-restore",
|
||||||
|
"cluster",
|
||||||
|
cls.pg_volume,
|
||||||
|
cls.hash,
|
||||||
|
cls.version,
|
||||||
|
"--backups-dir",
|
||||||
|
cls.backups_dir,
|
||||||
|
"--repo-name",
|
||||||
|
cls.repo_name,
|
||||||
|
"--container",
|
||||||
|
cls.pg_container,
|
||||||
|
"--instance",
|
||||||
|
cls.pg_container,
|
||||||
|
"--db-user",
|
||||||
|
"postgres",
|
||||||
|
"--db-password",
|
||||||
|
"pgpw",
|
||||||
|
"--empty",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls) -> None:
|
||||||
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _psql(cls, database: str, sql: str) -> str:
|
||||||
|
p = run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
cls.pg_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
f'psql -U postgres -d {database} -t -A -c "{sql}"',
|
||||||
|
]
|
||||||
|
)
|
||||||
|
return (p.stdout or "").strip()
|
||||||
|
|
||||||
|
def test_the_backup_wrote_a_cluster_dump(self) -> None:
|
||||||
|
self.assertTrue(self.dump.is_file(), f"no cluster dump at {self.dump}")
|
||||||
|
|
||||||
|
def test_both_databases_are_back(self) -> None:
|
||||||
|
listed = self._psql(
|
||||||
|
"postgres",
|
||||||
|
"SELECT datname FROM pg_database WHERE datname IN ('first','second') ORDER BY 1",
|
||||||
|
)
|
||||||
|
self.assertEqual(listed.split(), ["first", "second"])
|
||||||
|
|
||||||
|
def test_each_database_carries_its_own_payload(self) -> None:
|
||||||
|
self.assertEqual(self._psql("first", "SELECT v FROM t"), "first-payload")
|
||||||
|
self.assertEqual(self._psql("second", "SELECT v FROM t"), "second-payload")
|
||||||
|
|
||||||
|
def test_the_superusers_own_create_was_filtered(self) -> None:
|
||||||
|
self.assertEqual(
|
||||||
|
self._psql(
|
||||||
|
"postgres", "SELECT rolsuper FROM pg_roles WHERE rolname = 'postgres'"
|
||||||
|
),
|
||||||
|
"t",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_the_owning_role_is_back(self) -> None:
|
||||||
|
self.assertEqual(
|
||||||
|
self._psql(
|
||||||
|
"postgres", "SELECT rolname FROM pg_roles WHERE rolname = 'app'"
|
||||||
|
),
|
||||||
|
"app",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_ownership_survived(self) -> None:
|
||||||
|
self.assertEqual(
|
||||||
|
self._psql(
|
||||||
|
"postgres",
|
||||||
|
"SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = 'first'",
|
||||||
|
),
|
||||||
|
"app",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -1,9 +1,8 @@
|
|||||||
# tests/e2e/test_e2e_postgres_empty_drop_hard.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
POSTGRES_IMAGE,
|
|
||||||
POSTGRES_DATA_DIR,
|
POSTGRES_DATA_DIR,
|
||||||
|
POSTGRES_IMAGE,
|
||||||
backup_run,
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
|
|||||||
@@ -1,20 +1,19 @@
|
|||||||
# tests/e2e/test_e2e_postgres_full.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
POSTGRES_IMAGE,
|
|
||||||
POSTGRES_DATA_DIR,
|
POSTGRES_DATA_DIR,
|
||||||
backup_run,
|
POSTGRES_IMAGE,
|
||||||
backup_path,
|
backup_path,
|
||||||
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
ensure_empty_dir,
|
ensure_empty_dir,
|
||||||
latest_version_dir,
|
latest_version_dir,
|
||||||
require_docker,
|
require_docker,
|
||||||
unique,
|
|
||||||
write_databases_csv,
|
|
||||||
run,
|
run,
|
||||||
|
unique,
|
||||||
wait_for_postgres,
|
wait_for_postgres,
|
||||||
|
write_databases_csv,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -55,7 +54,6 @@ class TestE2EPostgresFull(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||||
|
|
||||||
# Create a table + data
|
|
||||||
run(
|
run(
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
|
|||||||
@@ -1,20 +1,19 @@
|
|||||||
# tests/e2e/test_e2e_postgres_no_copy.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
POSTGRES_IMAGE,
|
|
||||||
POSTGRES_DATA_DIR,
|
POSTGRES_DATA_DIR,
|
||||||
backup_run,
|
POSTGRES_IMAGE,
|
||||||
backup_path,
|
backup_path,
|
||||||
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
ensure_empty_dir,
|
ensure_empty_dir,
|
||||||
latest_version_dir,
|
latest_version_dir,
|
||||||
require_docker,
|
require_docker,
|
||||||
unique,
|
|
||||||
write_databases_csv,
|
|
||||||
run,
|
run,
|
||||||
|
unique,
|
||||||
wait_for_postgres,
|
wait_for_postgres,
|
||||||
|
write_databases_csv,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
# tests/e2e/test_e2e_postgres_single_transaction_live_writer.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
POSTGRES_IMAGE,
|
|
||||||
POSTGRES_DATA_DIR,
|
POSTGRES_DATA_DIR,
|
||||||
|
POSTGRES_IMAGE,
|
||||||
backup_run,
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
POSTGRES_IMAGE,
|
|
||||||
POSTGRES_DATA_DIR,
|
POSTGRES_DATA_DIR,
|
||||||
|
POSTGRES_IMAGE,
|
||||||
backup_path,
|
backup_path,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ LOOP_FS = {
|
|||||||
"ext4": "mkdir -p /subject/docker",
|
"ext4": "mkdir -p /subject/docker",
|
||||||
}
|
}
|
||||||
# A container carries no /lib/modules, so modprobe fails even on a loaded module.
|
# A container carries no /lib/modules, so modprobe fails even on a loaded module.
|
||||||
ZFS_READY = '{ [ -c /dev/zfs ] || modprobe zfs 2>/dev/null; }; [ -c /dev/zfs ]'
|
ZFS_READY = "{ [ -c /dev/zfs ] || modprobe zfs 2>/dev/null; }; [ -c /dev/zfs ]"
|
||||||
|
|
||||||
|
|
||||||
def mount_script(fstype: str) -> str:
|
def mount_script(fstype: str) -> str:
|
||||||
@@ -51,7 +51,13 @@ def zfs_usable() -> bool:
|
|||||||
"""Whether this host's kernel can serve zfs to a privileged container."""
|
"""Whether this host's kernel can serve zfs to a privileged container."""
|
||||||
proc = run(
|
proc = run(
|
||||||
[
|
[
|
||||||
"docker", "run", "--rm", "--privileged", IMAGE, "sh", "-lc",
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"--privileged",
|
||||||
|
IMAGE,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
f"apk add -q zfs >/dev/null 2>&1 && {ZFS_READY}",
|
f"apk add -q zfs >/dev/null 2>&1 && {ZFS_READY}",
|
||||||
],
|
],
|
||||||
capture=True,
|
capture=True,
|
||||||
@@ -87,11 +93,20 @@ def drive(fstype: str, kind: str, expect: str) -> str:
|
|||||||
try:
|
try:
|
||||||
proc = run(
|
proc = run(
|
||||||
[
|
[
|
||||||
"docker", "run", "--rm", "--privileged",
|
"docker",
|
||||||
"--name", staged.name,
|
"run",
|
||||||
"-v", f"{staged / 'src'}:/src:ro",
|
"--rm",
|
||||||
"-v", f"{staged / 'driver.py'}:/driver.py:ro",
|
"--privileged",
|
||||||
IMAGE, "sh", "-lc", script,
|
"--name",
|
||||||
|
staged.name,
|
||||||
|
"-v",
|
||||||
|
f"{staged / 'src'}:/src:ro",
|
||||||
|
"-v",
|
||||||
|
f"{staged / 'driver.py'}:/driver.py:ro",
|
||||||
|
IMAGE,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
script,
|
||||||
],
|
],
|
||||||
capture=True,
|
capture=True,
|
||||||
check=False,
|
check=False,
|
||||||
@@ -99,7 +114,9 @@ def drive(fstype: str, kind: str, expect: str) -> str:
|
|||||||
finally:
|
finally:
|
||||||
shutil.rmtree(staged, ignore_errors=True)
|
shutil.rmtree(staged, ignore_errors=True)
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
raise AssertionError(f"{fstype}/{kind} driver failed:\n{proc.stdout}\n{proc.stderr}")
|
raise AssertionError(
|
||||||
|
f"{fstype}/{kind} driver failed:\n{proc.stdout}\n{proc.stderr}"
|
||||||
|
)
|
||||||
return proc.stdout
|
return proc.stdout
|
||||||
|
|
||||||
|
|
||||||
@@ -125,7 +142,9 @@ class TestE2ESnapshot(unittest.TestCase):
|
|||||||
"E2E_REQUIRE_FILESYSTEMS demands zfs, but this kernel provides no "
|
"E2E_REQUIRE_FILESYSTEMS demands zfs, but this kernel provides no "
|
||||||
"zfs module; load it before running the suite"
|
"zfs module; load it before running the suite"
|
||||||
)
|
)
|
||||||
self.skipTest("this kernel provides no zfs module, so no pool can be created")
|
self.skipTest(
|
||||||
|
"this kernel provides no zfs module, so no pool can be created"
|
||||||
|
)
|
||||||
self.assert_freezes("zfs")
|
self.assert_freezes("zfs")
|
||||||
|
|
||||||
def test_ext4_has_no_snapshot_and_says_so(self) -> None:
|
def test_ext4_has_no_snapshot_and_says_so(self) -> None:
|
||||||
|
|||||||
@@ -68,11 +68,20 @@ class TestE2ESnapshotDatabase(unittest.TestCase):
|
|||||||
try:
|
try:
|
||||||
proc = run(
|
proc = run(
|
||||||
[
|
[
|
||||||
"docker", "run", "--rm", "--privileged",
|
"docker",
|
||||||
"--name", staged.name,
|
"run",
|
||||||
"-v", f"{staged / 'src'}:/src:ro",
|
"--rm",
|
||||||
"-v", f"{staged / 'driver.py'}:/driver.py:ro",
|
"--privileged",
|
||||||
IMAGE, "sh", "-lc", SCRIPT,
|
"--name",
|
||||||
|
staged.name,
|
||||||
|
"-v",
|
||||||
|
f"{staged / 'src'}:/src:ro",
|
||||||
|
"-v",
|
||||||
|
f"{staged / 'driver.py'}:/driver.py:ro",
|
||||||
|
IMAGE,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
SCRIPT,
|
||||||
],
|
],
|
||||||
capture=True,
|
capture=True,
|
||||||
check=False,
|
check=False,
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
# tests/e2e/test_e2e_swarm_task_skip.py
|
|
||||||
#
|
|
||||||
# Reproduces the swarm flake fixed on this branch: baudolo used to stop a
|
# Reproduces the swarm flake fixed on this branch: baudolo used to stop a
|
||||||
# swarm task container around the volume file backup because its image was
|
# swarm task container around the volume file backup because its image was
|
||||||
# not whitelisted; the orchestrator immediately replaced the stopped task and
|
# not whitelisted; the orchestrator immediately replaced the stopped task and
|
||||||
|
|||||||
125
tests/e2e/test_e2e_volumes_no_backup_required_early_skip.py
Normal file
125
tests/e2e/test_e2e_volumes_no_backup_required_early_skip.py
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from .helpers import (
|
||||||
|
backup_path,
|
||||||
|
cleanup_docker,
|
||||||
|
create_minimal_compose_dir,
|
||||||
|
ensure_empty_dir,
|
||||||
|
latest_version_dir,
|
||||||
|
require_docker,
|
||||||
|
run,
|
||||||
|
unique,
|
||||||
|
write_databases_csv,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestE2EVolumesNoBackupRequiredEarlySkip(unittest.TestCase):
|
||||||
|
"""Both volumes hang off the same container, so an image-level exclusion
|
||||||
|
could only drop both. Only the named one may disappear."""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
require_docker()
|
||||||
|
|
||||||
|
cls.prefix = unique("baudolo-e2e-early-skip-no-backup-volume")
|
||||||
|
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||||
|
ensure_empty_dir(cls.backups_dir)
|
||||||
|
|
||||||
|
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||||
|
cls.repo_name = cls.prefix
|
||||||
|
|
||||||
|
cls.container = f"{cls.prefix}-app"
|
||||||
|
cls.excluded_volume = f"{cls.prefix}-derived-vol"
|
||||||
|
cls.kept_volume = f"{cls.prefix}-state-vol"
|
||||||
|
|
||||||
|
cls.containers = [cls.container]
|
||||||
|
cls.volumes = [cls.excluded_volume, cls.kept_volume]
|
||||||
|
|
||||||
|
run(["docker", "volume", "create", cls.excluded_volume])
|
||||||
|
run(["docker", "volume", "create", cls.kept_volume])
|
||||||
|
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"-v",
|
||||||
|
f"{cls.excluded_volume}:/derived",
|
||||||
|
"-v",
|
||||||
|
f"{cls.kept_volume}:/state",
|
||||||
|
"alpine:3.20",
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
"echo derived > /derived/derived.txt && echo state > /state/state.txt",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"-d",
|
||||||
|
"--name",
|
||||||
|
cls.container,
|
||||||
|
"-v",
|
||||||
|
f"{cls.excluded_volume}:/derived",
|
||||||
|
"-v",
|
||||||
|
f"{cls.kept_volume}:/state",
|
||||||
|
"alpine:3.20",
|
||||||
|
"sleep",
|
||||||
|
"600",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
|
write_databases_csv(cls.databases_csv, [])
|
||||||
|
|
||||||
|
cmd = [
|
||||||
|
"baudolo",
|
||||||
|
"--compose-dir",
|
||||||
|
cls.compose_dir,
|
||||||
|
"--repo-name",
|
||||||
|
cls.repo_name,
|
||||||
|
"--databases-csv",
|
||||||
|
cls.databases_csv,
|
||||||
|
"--backups-dir",
|
||||||
|
cls.backups_dir,
|
||||||
|
"--images-no-stop-required",
|
||||||
|
"alpine:3.20",
|
||||||
|
"--volumes-no-backup-required",
|
||||||
|
cls.excluded_volume,
|
||||||
|
]
|
||||||
|
cp = run(cmd, capture=True, check=True)
|
||||||
|
cls.stdout = cp.stdout or ""
|
||||||
|
cls.stderr = cp.stderr or ""
|
||||||
|
|
||||||
|
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls) -> None:
|
||||||
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
|
||||||
|
def test_excluded_volume_has_no_backup_directory_at_all(self) -> None:
|
||||||
|
p = backup_path(
|
||||||
|
self.backups_dir,
|
||||||
|
self.repo_name,
|
||||||
|
self.version,
|
||||||
|
self.excluded_volume,
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
p.exists(),
|
||||||
|
f"Expected NO backup directory for the excluded volume, but found: {p}",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_sibling_volume_of_the_same_container_is_still_backed_up(self) -> None:
|
||||||
|
p = (
|
||||||
|
backup_path(
|
||||||
|
self.backups_dir,
|
||||||
|
self.repo_name,
|
||||||
|
self.version,
|
||||||
|
self.kept_volume,
|
||||||
|
)
|
||||||
|
/ "files"
|
||||||
|
/ "state.txt"
|
||||||
|
)
|
||||||
|
self.assertTrue(p.is_file(), f"Expected backed up file at: {p}")
|
||||||
@@ -9,7 +9,6 @@ import pandas as pd
|
|||||||
# Adjust if your package name/import path differs.
|
# Adjust if your package name/import path differs.
|
||||||
from baudolo.backup.dumps import load_databases_df
|
from baudolo.backup.dumps import load_databases_df
|
||||||
|
|
||||||
|
|
||||||
EXPECTED_COLUMNS = ["instance", "database", "username", "password"]
|
EXPECTED_COLUMNS = ["instance", "database", "username", "password"]
|
||||||
|
|
||||||
|
|
||||||
@@ -33,7 +32,6 @@ class TestLoadDatabasesDf(unittest.TestCase):
|
|||||||
def test_empty_csv_is_handled_with_warning_and_empty_df(self) -> None:
|
def test_empty_csv_is_handled_with_warning_and_empty_df(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as td:
|
with tempfile.TemporaryDirectory() as td:
|
||||||
empty_path = os.path.join(td, "databases.csv")
|
empty_path = os.path.join(td, "databases.csv")
|
||||||
# Create an empty file (0 bytes)
|
|
||||||
with open(empty_path, "w", encoding="utf-8") as f:
|
with open(empty_path, "w", encoding="utf-8") as f:
|
||||||
f.write("")
|
f.write("")
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from baudolo.backup.snapshot import volume_snapshot
|
|||||||
def stubbed_snapshot(kind: str, subject: str, tag: str):
|
def stubbed_snapshot(kind: str, subject: str, tag: str):
|
||||||
return volume_snapshot(kind, subject, tag, run=lambda command: [])
|
return volume_snapshot(kind, subject, tag, run=lambda command: [])
|
||||||
|
|
||||||
|
|
||||||
ARGV = [
|
ARGV = [
|
||||||
"baudolo",
|
"baudolo",
|
||||||
"--compose-dir",
|
"--compose-dir",
|
||||||
|
|||||||
82
tests/unit/backup/test_app_volumes_no_backup_required.py
Normal file
82
tests/unit/backup/test_app_volumes_no_backup_required.py
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
"""Contract of --volumes-no-backup-required: exclusion is per volume name,
|
||||||
|
independent of which containers use it."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
from baudolo.backup import app
|
||||||
|
|
||||||
|
ARGV = [
|
||||||
|
"baudolo",
|
||||||
|
"--compose-dir",
|
||||||
|
"/compose",
|
||||||
|
"--backups-dir",
|
||||||
|
"/backups",
|
||||||
|
"--volumes-no-backup-required",
|
||||||
|
"derived",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def drive() -> tuple[list[str], list[str], list[str]]:
|
||||||
|
backed_up: list[str] = []
|
||||||
|
created: list[str] = []
|
||||||
|
inspected: list[str] = []
|
||||||
|
|
||||||
|
def record_backup(versions_dir, volume_name, volume_dir, *, authoritative, source):
|
||||||
|
backed_up.append(volume_name)
|
||||||
|
|
||||||
|
with (
|
||||||
|
mock.patch("sys.argv", ARGV),
|
||||||
|
mock.patch.object(app, "get_machine_id", return_value="machine"),
|
||||||
|
mock.patch.object(app, "create_version_directory", return_value="/gen"),
|
||||||
|
mock.patch.object(
|
||||||
|
app,
|
||||||
|
"create_volume_directory",
|
||||||
|
side_effect=lambda _version_dir, name: created.append(name) or "/gen/vol",
|
||||||
|
),
|
||||||
|
mock.patch.object(app, "load_databases_df", return_value=None),
|
||||||
|
mock.patch.object(
|
||||||
|
app, "docker_volume_names", return_value=["derived", "state"]
|
||||||
|
),
|
||||||
|
mock.patch.object(
|
||||||
|
app,
|
||||||
|
"containers_using_volume",
|
||||||
|
side_effect=lambda name: inspected.append(name) or ["app"],
|
||||||
|
),
|
||||||
|
mock.patch.object(app, "volume_is_fully_ignored", return_value=False),
|
||||||
|
mock.patch.object(app, "backup_dumps_for_volume", return_value=(False, False)),
|
||||||
|
mock.patch.object(app, "get_storage_path", return_value="/data/"),
|
||||||
|
mock.patch.object(app, "stamp_directory"),
|
||||||
|
mock.patch.object(app, "handle_docker_compose_services"),
|
||||||
|
mock.patch.object(app.os.path, "isdir", return_value=True),
|
||||||
|
mock.patch.object(app, "backup_volume", side_effect=record_backup),
|
||||||
|
mock.patch.object(app, "filter_stoppable", return_value=[]),
|
||||||
|
mock.patch.object(app, "requires_stop", return_value=False),
|
||||||
|
mock.patch.object(app, "change_containers_status"),
|
||||||
|
):
|
||||||
|
app.main()
|
||||||
|
return backed_up, created, inspected
|
||||||
|
|
||||||
|
|
||||||
|
class TestVolumesNoBackupRequired(unittest.TestCase):
|
||||||
|
def test_the_named_volume_is_never_backed_up(self) -> None:
|
||||||
|
backed_up, _created, _inspected = drive()
|
||||||
|
self.assertNotIn("derived", backed_up)
|
||||||
|
|
||||||
|
def test_a_sibling_volume_of_the_same_container_survives(self) -> None:
|
||||||
|
backed_up, _created, _inspected = drive()
|
||||||
|
self.assertEqual(backed_up, ["state"])
|
||||||
|
|
||||||
|
def test_no_generation_directory_is_created_for_it(self) -> None:
|
||||||
|
_backed_up, created, _inspected = drive()
|
||||||
|
self.assertEqual(created, ["state"])
|
||||||
|
|
||||||
|
def test_the_skip_precedes_the_container_inspection(self) -> None:
|
||||||
|
_backed_up, _created, inspected = drive()
|
||||||
|
self.assertEqual(inspected, ["state"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -39,7 +39,9 @@ class TestSnapshotFlags(unittest.TestCase):
|
|||||||
parse("--snapshot", "ext4", "--snapshot-subject", "/var/lib/docker")
|
parse("--snapshot", "ext4", "--snapshot-subject", "/var/lib/docker")
|
||||||
|
|
||||||
def test_zfs_is_accepted(self) -> None:
|
def test_zfs_is_accepted(self) -> None:
|
||||||
self.assertEqual(parse("--snapshot", "zfs", "--snapshot-subject", "/d").snapshot, "zfs")
|
self.assertEqual(
|
||||||
|
parse("--snapshot", "zfs", "--snapshot-subject", "/d").snapshot, "zfs"
|
||||||
|
)
|
||||||
|
|
||||||
def test_shutdown_is_rejected_because_nothing_is_stopped(self) -> None:
|
def test_shutdown_is_rejected_because_nothing_is_stopped(self) -> None:
|
||||||
with self.assertRaises(SystemExit):
|
with self.assertRaises(SystemExit):
|
||||||
@@ -67,14 +69,18 @@ class TestSnapshotFlags(unittest.TestCase):
|
|||||||
|
|
||||||
class TestRequiredFlags(unittest.TestCase):
|
class TestRequiredFlags(unittest.TestCase):
|
||||||
def test_backups_dir_is_required(self) -> None:
|
def test_backups_dir_is_required(self) -> None:
|
||||||
with mock.patch("sys.argv", ["baudolo", "--compose-dir", "/compose"]):
|
with (
|
||||||
with self.assertRaises(SystemExit):
|
mock.patch("sys.argv", ["baudolo", "--compose-dir", "/compose"]),
|
||||||
parse_args()
|
self.assertRaises(SystemExit),
|
||||||
|
):
|
||||||
|
parse_args()
|
||||||
|
|
||||||
def test_compose_dir_is_required(self) -> None:
|
def test_compose_dir_is_required(self) -> None:
|
||||||
with mock.patch("sys.argv", ["baudolo", "--backups-dir", "/backups"]):
|
with (
|
||||||
with self.assertRaises(SystemExit):
|
mock.patch("sys.argv", ["baudolo", "--backups-dir", "/backups"]),
|
||||||
parse_args()
|
self.assertRaises(SystemExit),
|
||||||
|
):
|
||||||
|
parse_args()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ from __future__ import annotations
|
|||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import List
|
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from .compose_fixture import setup_compose_dir as _setup_compose_dir
|
from .compose_fixture import setup_compose_dir as _setup_compose_dir
|
||||||
@@ -99,7 +98,7 @@ class TestCompose(unittest.TestCase):
|
|||||||
str(d), ["up", "-d", "--force-recreate"]
|
str(d), ["up", "-d", "--force-recreate"]
|
||||||
)
|
)
|
||||||
|
|
||||||
expected: List[str] = [
|
expected: list[str] = [
|
||||||
"/usr/bin/docker",
|
"/usr/bin/docker",
|
||||||
"compose",
|
"compose",
|
||||||
"--chdir",
|
"--chdir",
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
from typing import List
|
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
|
||||||
@@ -11,7 +10,7 @@ class HardRestartArgTests(unittest.TestCase):
|
|||||||
the dir is a stack whose overlay network collides with compose up, pass
|
the dir is a stack whose overlay network collides with compose up, pass
|
||||||
nothing."""
|
nothing."""
|
||||||
|
|
||||||
def _parse(self, extra: List[str]):
|
def _parse(self, extra: list[str]):
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from baudolo.backup import cli
|
from baudolo.backup import cli
|
||||||
|
|||||||
@@ -20,15 +20,17 @@ def _capture_commands(*, db_type, rows, container):
|
|||||||
captured.append(cmd)
|
captured.append(cmd)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as td:
|
with (
|
||||||
with patch.object(db_mod, "execute_shell_command", side_effect=_capture):
|
tempfile.TemporaryDirectory() as td,
|
||||||
db_mod.backup_database(
|
patch.object(db_mod, "execute_shell_command", side_effect=_capture),
|
||||||
container=container,
|
):
|
||||||
volume_dir=td,
|
db_mod.backup_database(
|
||||||
db_type=db_type,
|
container=container,
|
||||||
databases_df=_df(rows),
|
volume_dir=td,
|
||||||
database_containers=[container],
|
db_type=db_type,
|
||||||
)
|
databases_df=_df(rows),
|
||||||
|
database_containers=[container],
|
||||||
|
)
|
||||||
return captured
|
return captured
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
50
tests/unit/backup/test_docker_image_name.py
Normal file
50
tests/unit/backup/test_docker_image_name.py
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from baudolo.backup import docker as docker_mod
|
||||||
|
|
||||||
|
|
||||||
|
def _with_image(reference: str):
|
||||||
|
return patch.object(docker_mod, "execute_shell_command", return_value=[reference])
|
||||||
|
|
||||||
|
|
||||||
|
class TestImageName(unittest.TestCase):
|
||||||
|
def test_plain_reference(self) -> None:
|
||||||
|
with _with_image("postgres:16"):
|
||||||
|
self.assertEqual(docker_mod.image_name("c1"), "postgres")
|
||||||
|
|
||||||
|
def test_registry_host_is_dropped(self) -> None:
|
||||||
|
with _with_image("svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5"):
|
||||||
|
self.assertEqual(docker_mod.image_name("c1"), "postgres_custom")
|
||||||
|
|
||||||
|
def test_pull_through_path_is_kept(self) -> None:
|
||||||
|
with _with_image(
|
||||||
|
"svc-db-mariadb-swarm-mgr-01:5000/ghcr.io/x/mirror/docker.io/postgres:16"
|
||||||
|
):
|
||||||
|
self.assertEqual(
|
||||||
|
docker_mod.image_name("c1"), "ghcr.io/x/mirror/docker.io/postgres"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_digest_is_dropped(self) -> None:
|
||||||
|
with _with_image("registry:5000/postgres@sha256:" + "0" * 64):
|
||||||
|
self.assertEqual(docker_mod.image_name("c1"), "postgres")
|
||||||
|
|
||||||
|
|
||||||
|
class TestHasImage(unittest.TestCase):
|
||||||
|
def test_registry_hostname_does_not_decide_the_engine(self) -> None:
|
||||||
|
with _with_image("svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5"):
|
||||||
|
self.assertFalse(docker_mod.has_image("c1", "mariadb"))
|
||||||
|
with _with_image("svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5"):
|
||||||
|
self.assertTrue(docker_mod.has_image("c1", "postgres"))
|
||||||
|
|
||||||
|
def test_tag_does_not_decide_the_engine(self) -> None:
|
||||||
|
with _with_image("registry:5000/xwiki_custom:lts-postgres-tomcat"):
|
||||||
|
self.assertFalse(docker_mod.has_image("c1", "postgres"))
|
||||||
|
|
||||||
|
def test_mirrored_mariadb_still_matches(self) -> None:
|
||||||
|
with _with_image("registry:5000/ghcr.io/x/mirror/docker.io/mariadb:11"):
|
||||||
|
self.assertTrue(docker_mod.has_image("c1", "mariadb"))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -8,6 +8,7 @@ from pathlib import Path
|
|||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
from baudolo.backup import layout as mod
|
from baudolo.backup import layout as mod
|
||||||
|
from baudolo.backup.shell import BackupException
|
||||||
|
|
||||||
|
|
||||||
class TestVersionDirectory(unittest.TestCase):
|
class TestVersionDirectory(unittest.TestCase):
|
||||||
@@ -17,11 +18,12 @@ class TestVersionDirectory(unittest.TestCase):
|
|||||||
self.assertTrue(Path(created).is_dir())
|
self.assertTrue(Path(created).is_dir())
|
||||||
self.assertEqual(Path(created).name, "20260731020304")
|
self.assertEqual(Path(created).name, "20260731020304")
|
||||||
|
|
||||||
def test_it_is_idempotent(self) -> None:
|
def test_it_refuses_a_generation_another_run_already_claimed(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
first = mod.create_version_directory(tmp, "20260731")
|
mod.create_version_directory(tmp, "20260731")
|
||||||
second = mod.create_version_directory(tmp, "20260731")
|
with self.assertRaises(BackupException) as caught:
|
||||||
self.assertEqual(first, second)
|
mod.create_version_directory(tmp, "20260731")
|
||||||
|
self.assertIn("20260731", str(caught.exception))
|
||||||
|
|
||||||
def test_it_creates_missing_parents(self) -> None:
|
def test_it_creates_missing_parents(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
|||||||
@@ -35,11 +35,15 @@ class TestBtrfs(unittest.TestCase):
|
|||||||
run = Runner()
|
run = Runner()
|
||||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
||||||
pass
|
pass
|
||||||
self.assertEqual(run.calls[-1], "btrfs subvolume delete /var/lib/docker/.baudolo-20260731")
|
self.assertEqual(
|
||||||
|
run.calls[-1], "btrfs subvolume delete /var/lib/docker/.baudolo-20260731"
|
||||||
|
)
|
||||||
|
|
||||||
def test_it_maps_a_volume_path_into_the_snapshot(self) -> None:
|
def test_it_maps_a_volume_path_into_the_snapshot(self) -> None:
|
||||||
run = Runner()
|
run = Runner()
|
||||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
with volume_snapshot(
|
||||||
|
"btrfs", "/var/lib/docker", "20260731", run=run
|
||||||
|
) as resolve:
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
resolve("/var/lib/docker/volumes/postgres_data/_data"),
|
resolve("/var/lib/docker/volumes/postgres_data/_data"),
|
||||||
"/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data",
|
"/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data",
|
||||||
@@ -47,7 +51,9 @@ class TestBtrfs(unittest.TestCase):
|
|||||||
|
|
||||||
def test_it_keeps_the_trailing_slash_rsync_reads_as_contents(self) -> None:
|
def test_it_keeps_the_trailing_slash_rsync_reads_as_contents(self) -> None:
|
||||||
run = Runner()
|
run = Runner()
|
||||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
with volume_snapshot(
|
||||||
|
"btrfs", "/var/lib/docker", "20260731", run=run
|
||||||
|
) as resolve:
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
resolve("/var/lib/docker/volumes/postgres_data/_data/"),
|
resolve("/var/lib/docker/volumes/postgres_data/_data/"),
|
||||||
"/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data/",
|
"/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data/",
|
||||||
@@ -55,9 +61,11 @@ class TestBtrfs(unittest.TestCase):
|
|||||||
|
|
||||||
def test_it_removes_the_snapshot_even_when_the_body_raises(self) -> None:
|
def test_it_removes_the_snapshot_even_when_the_body_raises(self) -> None:
|
||||||
run = Runner()
|
run = Runner()
|
||||||
with self.assertRaises(ZeroDivisionError):
|
with (
|
||||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
self.assertRaises(ZeroDivisionError),
|
||||||
raise ZeroDivisionError
|
volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run),
|
||||||
|
):
|
||||||
|
raise ZeroDivisionError
|
||||||
self.assertTrue(run.calls[-1].startswith("btrfs subvolume delete"))
|
self.assertTrue(run.calls[-1].startswith("btrfs subvolume delete"))
|
||||||
|
|
||||||
|
|
||||||
@@ -87,29 +95,39 @@ class TestZfs(unittest.TestCase):
|
|||||||
|
|
||||||
def test_an_unmounted_dataset_is_an_error(self) -> None:
|
def test_an_unmounted_dataset_is_an_error(self) -> None:
|
||||||
run = Runner({"zfs list": [""]})
|
run = Runner({"zfs list": [""]})
|
||||||
with self.assertRaises(SnapshotError):
|
with (
|
||||||
with volume_snapshot("zfs", "/var/lib/docker", "20260731", run=run):
|
self.assertRaises(SnapshotError),
|
||||||
pass
|
volume_snapshot("zfs", "/var/lib/docker", "20260731", run=run),
|
||||||
|
):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
class TestRejections(unittest.TestCase):
|
class TestRejections(unittest.TestCase):
|
||||||
def test_an_unknown_kind_is_rejected(self) -> None:
|
def test_an_unknown_kind_is_rejected(self) -> None:
|
||||||
run = Runner()
|
run = Runner()
|
||||||
with self.assertRaises(SnapshotError):
|
with (
|
||||||
with volume_snapshot("ext4", "/var/lib/docker", "20260731", run=run):
|
self.assertRaises(SnapshotError),
|
||||||
pass
|
volume_snapshot("ext4", "/var/lib/docker", "20260731", run=run),
|
||||||
|
):
|
||||||
|
pass
|
||||||
self.assertEqual(run.calls, [])
|
self.assertEqual(run.calls, [])
|
||||||
|
|
||||||
def test_a_path_outside_the_subject_is_rejected(self) -> None:
|
def test_a_path_outside_the_subject_is_rejected(self) -> None:
|
||||||
run = Runner()
|
run = Runner()
|
||||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
with (
|
||||||
with self.assertRaises(SnapshotError):
|
volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve,
|
||||||
resolve("/etc/passwd")
|
self.assertRaises(SnapshotError),
|
||||||
|
):
|
||||||
|
resolve("/etc/passwd")
|
||||||
|
|
||||||
def test_the_subject_itself_resolves_to_the_snapshot_root(self) -> None:
|
def test_the_subject_itself_resolves_to_the_snapshot_root(self) -> None:
|
||||||
run = Runner()
|
run = Runner()
|
||||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
with volume_snapshot(
|
||||||
self.assertEqual(resolve("/var/lib/docker"), "/var/lib/docker/.baudolo-20260731")
|
"btrfs", "/var/lib/docker", "20260731", run=run
|
||||||
|
) as resolve:
|
||||||
|
self.assertEqual(
|
||||||
|
resolve("/var/lib/docker"), "/var/lib/docker/.baudolo-20260731"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Busy(Runner):
|
class Busy(Runner):
|
||||||
@@ -125,9 +143,11 @@ class TestRemovalFailure(unittest.TestCase):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
def test_a_failed_removal_does_not_mask_the_body(self) -> None:
|
def test_a_failed_removal_does_not_mask_the_body(self) -> None:
|
||||||
with self.assertRaises(ZeroDivisionError):
|
with (
|
||||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=Busy()):
|
self.assertRaises(ZeroDivisionError),
|
||||||
raise ZeroDivisionError
|
volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=Busy()),
|
||||||
|
):
|
||||||
|
raise ZeroDivisionError
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -43,6 +43,14 @@ class TestBackupVolume(unittest.TestCase):
|
|||||||
def test_it_always_deletes_what_the_source_no_longer_has(self) -> None:
|
def test_it_always_deletes_what_the_source_no_longer_has(self) -> None:
|
||||||
self.assertIn("--delete", self.copy())
|
self.assertIn("--delete", self.copy())
|
||||||
|
|
||||||
|
def test_it_carries_no_kernel_objects_into_a_generation(self) -> None:
|
||||||
|
self.assertIn("--no-D", self.copy())
|
||||||
|
|
||||||
|
def test_it_keeps_no_twin_of_what_the_second_pass_replaces(self) -> None:
|
||||||
|
command = self.copy(authoritative=True)
|
||||||
|
self.assertIn("rsync -aP ", command)
|
||||||
|
self.assertNotIn("--backup", command)
|
||||||
|
|
||||||
def test_it_creates_the_destination(self) -> None:
|
def test_it_creates_the_destination(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
dest = Path(tmp) / "gen" / "demo"
|
dest = Path(tmp) / "gen" / "demo"
|
||||||
|
|||||||
121
tests/unit/restore/test_cluster_replay.py
Normal file
121
tests/unit/restore/test_cluster_replay.py
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from baudolo.restore.db import cluster as cluster_mod
|
||||||
|
from baudolo.restore.paths import BackupPaths
|
||||||
|
|
||||||
|
|
||||||
|
class TestClusterReplay(unittest.TestCase):
|
||||||
|
def _replay(self, *, empty: bool):
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def _capture(container, argv, **kwargs):
|
||||||
|
calls.append((argv, kwargs.get("stdin")))
|
||||||
|
return MagicMock()
|
||||||
|
|
||||||
|
with tempfile.NamedTemporaryFile(suffix=".sql") as sql:
|
||||||
|
sql.write(b"CREATE ROLE app;\nCREATE DATABASE app OWNER app;\n")
|
||||||
|
sql.flush()
|
||||||
|
with patch.object(cluster_mod, "docker_exec", side_effect=_capture):
|
||||||
|
cluster_mod.restore_cluster_sql(
|
||||||
|
container="db",
|
||||||
|
user="postgres",
|
||||||
|
password="pw",
|
||||||
|
sql_path=sql.name,
|
||||||
|
empty=empty,
|
||||||
|
)
|
||||||
|
return calls
|
||||||
|
|
||||||
|
def test_the_replay_is_not_wrapped_in_a_transaction(self) -> None:
|
||||||
|
argv, _ = self._replay(empty=False)[0]
|
||||||
|
self.assertNotIn(
|
||||||
|
"--single-transaction",
|
||||||
|
argv,
|
||||||
|
"CREATE DATABASE cannot run inside a transaction block, so unlike the "
|
||||||
|
"single-database replay this stream must not be wrapped in one",
|
||||||
|
)
|
||||||
|
self.assertIn("ON_ERROR_STOP=1", argv)
|
||||||
|
|
||||||
|
def test_the_replay_targets_the_control_database(self) -> None:
|
||||||
|
argv, _ = self._replay(empty=False)[0]
|
||||||
|
self.assertEqual(argv[argv.index("-d") + 1], cluster_mod.CONTROL_DB)
|
||||||
|
self.assertEqual(argv[argv.index("-U") + 1], "postgres")
|
||||||
|
|
||||||
|
def test_without_empty_nothing_is_dropped_first(self) -> None:
|
||||||
|
self.assertEqual(len(self._replay(empty=False)), 1)
|
||||||
|
|
||||||
|
def test_empty_drops_databases_before_their_owners(self) -> None:
|
||||||
|
calls = self._replay(empty=True)
|
||||||
|
self.assertEqual(len(calls), 2, f"expected pre-clean + replay: {calls}")
|
||||||
|
preclean = calls[0][1].decode()
|
||||||
|
self.assertLess(
|
||||||
|
preclean.index("DROP DATABASE"),
|
||||||
|
preclean.index("DROP ROLE"),
|
||||||
|
"a role cannot be dropped while it still owns a database",
|
||||||
|
)
|
||||||
|
self.assertIn("DROP OWNED BY", preclean)
|
||||||
|
self.assertIn("ORDER BY phase", preclean)
|
||||||
|
|
||||||
|
def test_the_preclean_spares_what_the_dump_does_not_recreate(self) -> None:
|
||||||
|
preclean = self._replay(empty=True)[0][1].decode()
|
||||||
|
self.assertIn("NOT datistemplate", preclean)
|
||||||
|
self.assertIn("datname <> current_database()", preclean)
|
||||||
|
self.assertIn("starts_with(rolname, 'pg_')", preclean)
|
||||||
|
self.assertIn("rolname <> current_user", preclean)
|
||||||
|
|
||||||
|
def test_only_the_connecting_role_loses_its_create(self) -> None:
|
||||||
|
# Captured from pg_dumpall 17.
|
||||||
|
dump = [
|
||||||
|
b"CREATE ROLE app;\n",
|
||||||
|
b"ALTER ROLE app WITH NOSUPERUSER INHERIT LOGIN PASSWORD 'SCRAM-SHA-256$...';\n",
|
||||||
|
b"CREATE ROLE postgres;\n",
|
||||||
|
b"ALTER ROLE postgres WITH SUPERUSER INHERIT LOGIN PASSWORD 'SCRAM-SHA-256$...';\n",
|
||||||
|
b'CREATE ROLE "odd-name";\n',
|
||||||
|
]
|
||||||
|
kept = list(cluster_mod.filter_own_role_creation(dump, "postgres"))
|
||||||
|
self.assertNotIn(b"CREATE ROLE postgres;\n", kept)
|
||||||
|
self.assertIn(b"CREATE ROLE app;\n", kept)
|
||||||
|
self.assertIn(b'CREATE ROLE "odd-name";\n', kept)
|
||||||
|
self.assertEqual(
|
||||||
|
sum(1 for line in kept if line.startswith(b"ALTER ROLE postgres")),
|
||||||
|
1,
|
||||||
|
"the ALTER re-applies the superuser's attributes and password",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_a_quoted_connecting_role_is_matched_too(self) -> None:
|
||||||
|
kept = list(
|
||||||
|
cluster_mod.filter_own_role_creation(
|
||||||
|
[b'CREATE ROLE "odd-name";\n'], "odd-name"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertEqual(kept, [])
|
||||||
|
|
||||||
|
def test_a_role_whose_name_merely_starts_the_same_is_kept(self) -> None:
|
||||||
|
kept = list(
|
||||||
|
cluster_mod.filter_own_role_creation(
|
||||||
|
[b"CREATE ROLE postgresql;\n"], "postgres"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertEqual(kept, [b"CREATE ROLE postgresql;\n"])
|
||||||
|
|
||||||
|
def test_a_missing_dump_is_reported_as_such(self) -> None:
|
||||||
|
with self.assertRaises(FileNotFoundError):
|
||||||
|
cluster_mod.restore_cluster_sql(
|
||||||
|
container="db",
|
||||||
|
user="postgres",
|
||||||
|
password="pw",
|
||||||
|
sql_path="/nonexistent/x.cluster.backup.sql",
|
||||||
|
empty=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_the_path_helper_names_the_dumpall_file(self) -> None:
|
||||||
|
paths = BackupPaths("vol", "hash", "v1", repo_name="repo", backups_dir="/B")
|
||||||
|
self.assertEqual(
|
||||||
|
paths.cluster_file("bigbluebutton"),
|
||||||
|
"/B/hash/repo/v1/vol/sql/bigbluebutton.cluster.backup.sql",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -131,7 +131,6 @@ class TestSeedMain(unittest.TestCase):
|
|||||||
warning_calls,
|
warning_calls,
|
||||||
"Expected a WARNING print when databases.csv is empty, but none was found.",
|
"Expected a WARNING print when databases.csv is empty, but none was found.",
|
||||||
)
|
)
|
||||||
# Ensure the warning goes to stderr
|
|
||||||
_, warn_kwargs = warning_calls[0]
|
_, warn_kwargs = warning_calls[0]
|
||||||
self.assertEqual(warn_kwargs.get("file"), seed_main.sys.stderr)
|
self.assertEqual(warn_kwargs.get("file"), seed_main.sys.stderr)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user