mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2026-08-24 06:44:33 +00:00
Compare commits
27 Commits
v3.4.2
...
44b1f16f7c
| Author | SHA1 | Date | |
|---|---|---|---|
| 44b1f16f7c | |||
| 7f51748486 | |||
| e0f89c86ec | |||
| f97efb10c4 | |||
| 704481a505 | |||
| b1ee8f5fac | |||
| 8dbd5e89ea | |||
| efcfe88e7f | |||
| 94637c32aa | |||
| 03da186a06 | |||
| 37a07fe100 | |||
| 1d86277a94 | |||
| 03013b6c76 | |||
| df1c65ccac | |||
| f791046c02 | |||
| c03329e4ca | |||
| 5f3ee0a669 | |||
| 8dac7371cb | |||
| 23cfc3b7e2 | |||
| da9c3a1e6f | |||
| e80f11d5e4 | |||
| bb647c66ec | |||
| f437787e64 | |||
| 2129c5e362 | |||
| a0204fd3ea | |||
| 90d289d92f | |||
| 57fc7c96bc |
13
.dockerignore
Normal file
13
.dockerignore
Normal file
@@ -0,0 +1,13 @@
|
||||
.git
|
||||
.github
|
||||
__pycache__
|
||||
**/__pycache__
|
||||
*.egg-info
|
||||
**/*.egg-info
|
||||
artifacts/
|
||||
dist/
|
||||
build/
|
||||
.venv
|
||||
.ruff_cache
|
||||
.pytest_cache
|
||||
.mcp.json
|
||||
242
CHANGELOG.md
242
CHANGELOG.md
@@ -1,5 +1,247 @@
|
||||
# Changelog
|
||||
|
||||
## [7.0.1] - 2026-08-18
|
||||
|
||||
- Restore: *--empty* no longer aborts on a database that carries an extension.
|
||||
The pre-clean picked its candidates by owner, on the stated assumption that
|
||||
extension members are superuser-owned and would therefore never be selected.
|
||||
That holds only when a superuser installed the extension: a role that
|
||||
installs one itself owns its functions, so they were listed for a one-by-one
|
||||
*DROP* that postgres refuses — *cannot drop function
|
||||
vector_in(cstring,oid,integer) because extension vector requires it*. Under
|
||||
*ON_ERROR_STOP* that ends the whole restore, which is how a generation of an
|
||||
application declaring the *vector* extension became unreplayable. Membership
|
||||
now comes from *pg_depend* rather than from ownership; each branch carries
|
||||
its oid and classid so one *NOT EXISTS* covers all seven instead of seven
|
||||
separate predicates, and the schema branch is guarded too because an
|
||||
extension can own a schema. Skipping the members suffices — the dump's
|
||||
*CREATE EXTENSION IF NOT EXISTS* finds the surviving extension either way.
|
||||
|
||||
## [7.0.0] - 2026-08-18
|
||||
|
||||
Breaking:
|
||||
- Backup: *mariadb* and *mysql* join the suffix tokens, so a container named
|
||||
*<app>-mariadb* or *<app>-mysql* now resolves to the instance *<app>* instead
|
||||
of to its own name. A *databases.csv* keyed on the full container name has to
|
||||
move to the application name, or name the container in
|
||||
*--database-containers*. This narrows what 6.0.0 broke rather than widening
|
||||
it: a container named exactly *postgres*, *mariadb*, *mysql*, *db* or
|
||||
*database* resolves again without any declaration, which is the shape a
|
||||
compose file writes as *container_name: postgres* and the most common
|
||||
configuration there is.
|
||||
|
||||
Fixed:
|
||||
- Backup: a container named exactly after its engine is dumped again. The
|
||||
suffix match needs a hyphen or underscore in front of the token, which a bare
|
||||
name does not carry, so 6.0.0 resolved *container_name: postgres* to nothing
|
||||
and stopped dumping it without saying so. *ENGINE_NAMES* now states the set
|
||||
once and serves both readings — carried as a suffix it makes the rest the
|
||||
instance, being one outright makes the container its own instance.
|
||||
- Backup: a central MariaDB under swarm is dumped for the first time. Swarm
|
||||
names its task *mariadb_mariadb.1.<id>*, which matches neither the static
|
||||
*mariadb* passed through *--database-containers* nor any token the suffix
|
||||
match knew, since *_mariadb* is not *_db*. The database was silently absent
|
||||
from every swarm backup this tool has ever written, before 6.0.0 as well.
|
||||
- Backup: an application container is no longer recorded as a database.
|
||||
*container_engine* recognises an engine by its client tools, which an
|
||||
application image frequently ships, so refusing its dump alone would have
|
||||
written the volume to the manifest as *database: true, dumped: false* — the
|
||||
exact shape a restore drill reads as a database that was missed. Without an
|
||||
instance there is no database to record, and the volume is a file backup like
|
||||
any other.
|
||||
|
||||
## [6.0.0] - 2026-08-18
|
||||
|
||||
Breaking:
|
||||
- Backup: a database container whose name carries no *database*, *db* or
|
||||
*postgres* token — preceded by a hyphen or underscore — must now be named in
|
||||
*--database-containers*. Without that declaration its *databases.csv* rows no
|
||||
longer match and no dump is written, silently, because nothing fails. The
|
||||
shape this hits hardest is a container named exactly *postgres* or *mariadb*:
|
||||
the token needs a separator in front of it, which a bare name does not have.
|
||||
*app-database*, *app_database.1.<task>* from swarm and *app-postgres-1* are
|
||||
unaffected, as is any container already declared.
|
||||
|
||||
Fixed:
|
||||
- Backup: *docker exec* now forwards *PGPASSWORD* into the container.
|
||||
*execute_to_file* set the variable on baudolo's own process, but nothing
|
||||
carried it across the container boundary, so an engine whose *pg_hba* demands
|
||||
a password on TCP loopback refused every dump — which is every dedicated
|
||||
Postgres instance on a real host. The name travels as a bare *-e NAME* so
|
||||
docker copies the value out of this process's environment; spelling
|
||||
*-e NAME=value* instead would publish the secret in the host's process list.
|
||||
- Backup: *get_instance* no longer claims an instance it never derived. It
|
||||
returned the container name unchanged when that name carried no database
|
||||
token, so an application container answered the same *databases.csv* row as
|
||||
its own dedicated engine. Application images frequently ship the engine's
|
||||
client tools, so the dump command started and wrote a file that looked like a
|
||||
backup and held none of the data: measured against Discourse, 1,680 bytes
|
||||
from the application where the engine produced 10,469,439. The regex stays a
|
||||
normaliser — *<app>-database* from compose and *<app>_database.1.<task>* from
|
||||
swarm still resolve to one instance. Only the fallthrough changed.
|
||||
|
||||
New:
|
||||
- Tests: *get_instance* has unit coverage for the first time. Eleven cases pin
|
||||
the container names that compose, swarm and explicitly-named engines produce,
|
||||
so a future change to the regex has to state which shape it gives up.
|
||||
- Tests: two e2e modules cover shapes the suite structurally could not see.
|
||||
Every fixture passed its container in *--database-containers*, which left the
|
||||
regex branch — the only one a dedicated database ever takes — dead code under
|
||||
test, and no scenario made a password mandatory, because stock
|
||||
*postgres:alpine* grants trust on loopback.
|
||||
*test_e2e_postgres_password_required* starts an engine with
|
||||
*--auth-host=scram-sha-256* and carries a negative control asserting the
|
||||
server refuses an unauthenticated dump; without it the module would pass
|
||||
whether or not the password is forwarded at all.
|
||||
*test_e2e_app_container_ships_client_tools* places an application container
|
||||
beside its engine with neither declared, and requires the engine dumped, the
|
||||
application volume copied as files, and no dump written from the application.
|
||||
|
||||
## [5.0.0] - 2026-08-18
|
||||
|
||||
**[5.0.0] - 2026-08-18**
|
||||
|
||||
Breaking:
|
||||
- Library: *BackupException* is now *BackupError* and *VersionMismatch* is now
|
||||
*VersionMismatchError*. Both names violated the convention that an exception
|
||||
class ends in *Error*; the first is imported by six modules, so the rename is
|
||||
atomic across the package.
|
||||
- Library: *backup_dumps_for_volume* and *backup_mariadb_or_postgres* return a
|
||||
*VolumeOutcome* instead of a *(bool, bool)* tuple. The pair could not carry
|
||||
the detected engine, which the caller needs for the manifest.
|
||||
|
||||
New:
|
||||
- Backup: every generation carries a *manifest.json* stating its layout and,
|
||||
per volume, *database* (it held one), *dumped* (a dump was produced) and
|
||||
*engine* (which one was detected). A volume with *database* and no *dumped*
|
||||
was copied as raw engine files — under *--only-sql* that fallback is the
|
||||
documented behaviour, and until now nothing in the finished tree said it had
|
||||
happened. Restoring such a volume replays engine files instead of a dump.
|
||||
- Library: *baudolo.generation* states the generation layout once — *files*,
|
||||
*sql*, the dump suffixes, the manifest name. *BackupPaths*, the dump writer
|
||||
and the volume copier stop spelling them out separately. The module is
|
||||
import-free on purpose, so a consumer can read a manifest with nothing but
|
||||
*json* on a host where this package is not installed.
|
||||
|
||||
Changed:
|
||||
- Build: the test targets no longer depend on *clean*. *clean* is
|
||||
*git clean -fdX .*, so running the unit tests deleted every git-ignored file
|
||||
in the working tree. It was compensating for a missing *.dockerignore*, which
|
||||
now keeps *__pycache__*, egg-info and build output out of the image context
|
||||
where that belongs. *clean* remains available as its own target.
|
||||
- Lint: a ruff configuration is declared. The package ran on ruff's defaults
|
||||
while its consumer held itself to a far wider selection; measured against
|
||||
that selection the tree had 224 findings and now has none. Includes a full
|
||||
*os.path* to *pathlib* migration, with two deliberate exceptions: *abspath*
|
||||
stays where *Path.resolve()* would follow symlinks and let a symlinked volume
|
||||
test as inside the snapshot subject, and the rsync trailing separator is kept
|
||||
explicit where *Path* would drop it.
|
||||
|
||||
## [4.0.0] - 2026-08-17
|
||||
|
||||
Breaking:
|
||||
- CLI: *--repo-name* and *--databases-csv* are required. One default was the
|
||||
literal *backup-docker-to-local* while its help promised the git folder name;
|
||||
the other pointed into the installed package, so a forgotten flag ran the
|
||||
whole backup silently without a single dump.
|
||||
- CLI: *--dump-only-sql* is now *--only-sql*; the old spelling exits 2.
|
||||
- CLI: *--everything* is withdrawn. Its only real effect was to ignore
|
||||
*--images-no-stop-required*, which leaving that list empty already does.
|
||||
- Library: the runner injected into *volume_snapshot* receives an argv list
|
||||
instead of a command string.
|
||||
|
||||
New:
|
||||
- Backup: *--only-files* — no dumps at all, every volume as files, for hosts
|
||||
that hold no database credentials. Needs no *--databases-csv*; mutually
|
||||
exclusive with *--only-sql*.
|
||||
- Backup: the engine is detected by executing the dump tool in the container
|
||||
(*pg_dumpall*, *mariadb-dump*, *mysqldump*), not by reading the image name.
|
||||
A dedicated Postgres tagged *<app>-database* is finally dumped; an image
|
||||
merely named like an engine no longer kills the run with exit 127. Probed
|
||||
once per image ID, and an image shipping only *mysqldump* is dumped with it.
|
||||
- Library: *baudolo.databases* states the databases.csv contract once —
|
||||
columns, delimiter, cluster marker, validator, *read_rows()* — for the seed,
|
||||
the backup, and external consumers.
|
||||
|
||||
Changed:
|
||||
- Backup: every command is an argv list; *shell=True* is gone. A database name
|
||||
is validated on read as strictly as the seed writes it, *PGPASSWORD* travels
|
||||
in the child's environment instead of the command string, and a failing dump
|
||||
deletes its partial file instead of leaving it behind.
|
||||
|
||||
## [3.6.1] - 2026-08-17
|
||||
|
||||
- Restore: *--empty* on a cluster dump is a catalog-wide sweep — it drops every
|
||||
non-template database and every non-pg_ role of the instance. On a dedicated
|
||||
instance that is right, because the dump recreates all of it; on a shared one
|
||||
it destroys databases the dump does not carry, with nothing to restore them
|
||||
from. No test had ever executed that sweep: the e2e dropped the cluster by
|
||||
hand first and left the pre-clean with zero rows to generate.
|
||||
- Restore: the instance is checked instead of the sweep being narrowed.
|
||||
*--empty* refuses when the instance holds a database the dump does not carry,
|
||||
names it, and touches nothing. Narrowing the sweep is the obvious fix and is
|
||||
worse — a surviving database that owns or merely grants to one of the dump's
|
||||
roles pins it in *pg_shdepend*, *DROP OWNED BY* reaches only the control
|
||||
database the pre-clean is connected to, so *DROP ROLE* fails after the dump's
|
||||
own databases are already gone and the replay never starts.
|
||||
- Restore: the dump's inventory is read with a real identifier parser. A quoted
|
||||
name may hold spaces, and psql options precede the target of a *\connect*
|
||||
line, so a character class that stops at whitespace read *"odd name"* as
|
||||
*odd* and *-reuse-previous=on* as a database.
|
||||
- Tests: the cluster e2e no longer empties the instance itself, so *--empty*
|
||||
has to do it and the replay has to put it back. A second pass adds a foreign
|
||||
database and requires the refusal to leave both it and the restored data
|
||||
untouched.
|
||||
|
||||
## [3.6.0] - 2026-08-17
|
||||
|
||||
- Restore: *--empty* drops the schema in one session and replays in the next,
|
||||
with no rollback across the two, so a dump the engine could not parse left an
|
||||
emptied database behind. The dump's header is now checked against the running
|
||||
engine before anything is dropped, and a newer dump is refused.
|
||||
Forward across a major version stays allowed; *--no-version-check* is the way out.
|
||||
- Restore: a volume with driver options — NFS, a bind device, tmpfs — keeps the
|
||||
usual *_data* path, but docker mounts its real storage over it only while a
|
||||
container holds it. Restoring meanwhile landed under the mount, stayed hidden
|
||||
there, and rsync reported success. That volume is now refused until something
|
||||
mounts it.
|
||||
- Backup: the same volume sits in a snapshot as an empty directory, so it was
|
||||
copied empty and the generation stamped complete. Capture is decided per volume
|
||||
now — an uncaptured one is copied live, the rest keep their snapshot. A single
|
||||
NFS volume no longer costs the whole host its consistent backup.
|
||||
|
||||
## [3.5.0] - 2026-08-17
|
||||
|
||||
- Restore: a *database = '*'* row makes the backup write
|
||||
*<instance>.cluster.backup.sql* via *pg_dumpall*, and nothing could read it
|
||||
back — the CLI knew *files*, *postgres* and *mariadb*, so that dump was
|
||||
stored and unrestorable. *baudolo-restore cluster* replays it against the
|
||||
control database, deliberately without *--single-transaction* because
|
||||
CREATE DATABASE is forbidden inside a transaction block, and filters out the
|
||||
CREATE ROLE of the connecting role, which the pre-clean cannot drop while it
|
||||
holds the session. *--empty* drops the cluster's databases first, then
|
||||
releases what its roles still own, then the roles themselves.
|
||||
|
||||
- Lint: ruff was never wired into the repository — no target, no CI step, no
|
||||
pin — and reported 45 findings across sources and tests. *make ruff* and
|
||||
*make lint* now run it over every file, *make test* gates on a clean run as a
|
||||
fourth parallel spur, and the linter is pinned in a *lint* extra because a
|
||||
minor bump changes which rules fire.
|
||||
|
||||
## [3.4.3] - 2026-08-16
|
||||
|
||||
- Backup: *create_version_directory* carried *exist_ok=True*, so a run starting
|
||||
in the same wall-clock second as its predecessor claimed that predecessor's
|
||||
generation. Generation names carry seconds, and a host with little to copy
|
||||
finishes inside one — rsync *--delete* then overwrote a finished generation,
|
||||
and only afterwards did *create_stamp_file* refuse the already-stamped
|
||||
directory and exit 2. The guard reported the damage instead of preventing it.
|
||||
- Backup: the generation directory is claimed exclusively. Claiming it is the
|
||||
first filesystem action of a run, so a collision aborts before the first
|
||||
write and names the second it collided on.
|
||||
- Tests: the idempotence test asserted the reuse and gave way to one that
|
||||
requires the refusal.
|
||||
|
||||
## [3.4.2] - 2026-08-15
|
||||
|
||||
- Backup: *has_image* matched the raw *.Config.Image*, so the registry host and
|
||||
|
||||
35
Makefile
35
Makefile
@@ -1,4 +1,4 @@
|
||||
.PHONY: install build clean \
|
||||
.PHONY: install install-lint build clean lint ruff ruff-fix \
|
||||
test test-unit test-integration test-e2e \
|
||||
test-unit-run test-integration-run test-e2e-run
|
||||
|
||||
@@ -34,19 +34,36 @@ build:
|
||||
clean:
|
||||
git clean -fdX .
|
||||
|
||||
# clean + build run once and in order, then the three suites run concurrently
|
||||
# via -j3; the *-run targets carry no clean/build prereq so the sub-make cannot
|
||||
# race a second clean against build.
|
||||
# Separate from `install` so the test image does not have to carry the linter.
|
||||
install-lint:
|
||||
@$(PY_DEFAULT) -m pip install -q -e ".[lint]"
|
||||
|
||||
# Runs on the host, not in the image, so it also covers what the Dockerfile
|
||||
# does not copy.
|
||||
ruff: install-lint
|
||||
@echo ">> Running ruff over the whole repository"
|
||||
@$(PY_DEFAULT) -m ruff check .
|
||||
@$(PY_DEFAULT) -m ruff format --check .
|
||||
|
||||
ruff-fix: install-lint
|
||||
@$(PY_DEFAULT) -m ruff check --fix .
|
||||
@$(PY_DEFAULT) -m ruff format .
|
||||
|
||||
lint: ruff
|
||||
|
||||
# build runs once, then lint and the three suites run concurrently via -j4; the
|
||||
# *-run targets carry no build prereq so the sub-make cannot race a second build.
|
||||
# `clean` is deliberately not a prerequisite; .dockerignore keeps the image
|
||||
# context clean instead.
|
||||
test:
|
||||
@$(MAKE) clean
|
||||
@$(MAKE) build
|
||||
@$(MAKE) -j3 test-unit-run test-integration-run test-e2e-run
|
||||
@$(MAKE) -j4 lint test-unit-run test-integration-run test-e2e-run
|
||||
|
||||
test-unit: clean build test-unit-run
|
||||
test-unit: build test-unit-run
|
||||
|
||||
test-integration: clean build test-integration-run
|
||||
test-integration: build test-integration-run
|
||||
|
||||
test-e2e: clean build test-e2e-run
|
||||
test-e2e: build test-e2e-run
|
||||
|
||||
test-unit-run:
|
||||
@echo ">> Running unit tests"
|
||||
|
||||
11
README.md
11
README.md
@@ -123,6 +123,8 @@ This information is used by `baudolo` to execute
|
||||
```bash
|
||||
baudolo \
|
||||
--compose-dir /srv/docker \
|
||||
--backups-dir /Backups \
|
||||
--repo-name my-repo \
|
||||
--databases-csv /etc/baudolo/databases.csv \
|
||||
--database-containers central-postgres central-mariadb \
|
||||
--images-no-stop-required alpine postgres mariadb mysql \
|
||||
@@ -133,11 +135,12 @@ baudolo \
|
||||
|
||||
| Flag | Description |
|
||||
| --------------- | ------------------------------------------- |
|
||||
| `--everything` | Always stop containers and re-run rsync |
|
||||
| `--dump-only-sql`| Skip file backups only for DB volumes when dumps succeed; non-DB volumes are still backed up; fallback to files if no dump. |
|
||||
| `--only-sql` | Skip file backups only for DB volumes when dumps succeed; non-DB volumes are still backed up; fallback to files if no dump. |
|
||||
| `--only-files` | Take no dumps at all; every volume is backed up as files. Needs no `--databases-csv`. Mutually exclusive with `--only-sql`. |
|
||||
| `--shutdown` | Do not restart containers after backup |
|
||||
| `--backups-dir` | Backup root directory (default: `/Backups`) |
|
||||
| `--repo-name` | Backup namespace under machine hash |
|
||||
| `--backups-dir` | Backup root directory (required) |
|
||||
| `--repo-name` | Backup namespace under machine hash (required) |
|
||||
| `--databases-csv`| Path to `databases.csv` (required) |
|
||||
|
||||
## ♻️ Restore Operations
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "backup-docker-to-local"
|
||||
version = "3.4.2"
|
||||
version = "7.0.1"
|
||||
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
@@ -16,6 +16,11 @@ dependencies = [
|
||||
"dirval",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
# Pinned: a ruff minor bump changes which rules fire, and `make test` gates on
|
||||
# a clean run, so an unpinned lint would fail the suite on an unrelated day.
|
||||
lint = ["ruff==0.16.1"]
|
||||
|
||||
[project.scripts]
|
||||
baudolo = "baudolo.backup.__main__:main"
|
||||
baudolo-restore = "baudolo.restore.__main__:main"
|
||||
@@ -30,3 +35,52 @@ exclude = ["tests*"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
"baudolo.restore.db" = ["*.sql"]
|
||||
|
||||
[tool.ruff]
|
||||
respect-gitignore = true
|
||||
# The package still declares >=3.9, so pyupgrade must not propose 3.10+ syntax.
|
||||
target-version = "py39"
|
||||
exclude = ["build", "dist", "*.egg-info", ".venv", "venv"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
# Adopted from infinito-nexus-core so both repositories are held to one bar;
|
||||
# see that project's pyproject.toml for what each selector buys.
|
||||
select = [
|
||||
"E", "F", "I", "B", "UP", "RUF", "SIM", "C4", "PERF", "RET", "PIE",
|
||||
"T10", "PGH", "EXE", "RSE", "ICN", "DTZ",
|
||||
"TID", "LOG", "G",
|
||||
"S",
|
||||
"PTH",
|
||||
"FURB", "W", "FA", "YTT", "A", "ISC", "SLOT", "FLY",
|
||||
"PYI",
|
||||
"TC", "N",
|
||||
"PLE0605",
|
||||
"PLW1510", "PLW2901", "PLW0108", "PLW0603",
|
||||
"PLR5501", "PLC0207", "PLR1722", "PLR1714",
|
||||
"TRY002", "TRY004", "TRY300", "TRY301",
|
||||
"BLE001",
|
||||
]
|
||||
|
||||
# E501: `ruff format` reflows what it can; the rest is unsplittable literals.
|
||||
# RUF001/002/003: the prose uses em-dashes deliberately, not homoglyphs.
|
||||
# S603/S607: this tool's whole job is running `docker` / dump binaries from
|
||||
# PATH in list form, which is already injection-safe.
|
||||
# PTH207/PTH208: changing `glob.glob`/`os.listdir` return shapes needs a
|
||||
# per-call-site review, not a blanket rewrite.
|
||||
ignore = [
|
||||
"E501",
|
||||
"RUF001", "RUF002", "RUF003",
|
||||
"S603", "S607",
|
||||
"PTH207", "PTH208",
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
# Test code legitimately uses what flake8-bandit flags in production code:
|
||||
# asserts, dummy credentials, /tmp fixtures, broad excepts in teardown, and
|
||||
# SQL built from fixture names (S608) to set the databases under test up.
|
||||
"tests/**" = [
|
||||
"S101", "S102", "S105", "S106", "S108", "S110", "S112", "S608", "BLE001",
|
||||
]
|
||||
|
||||
# The e2e helpers package is a deliberate re-export aggregator.
|
||||
"tests/e2e/helpers/__init__.py" = ["F403"]
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .app import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from contextlib import ExitStack
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from .cli import parse_args
|
||||
from .compose import handle_docker_compose_services
|
||||
@@ -14,37 +14,36 @@ from .docker import (
|
||||
docker_volume_names,
|
||||
filter_stoppable,
|
||||
)
|
||||
from .dumps import backup_dumps_for_volume, load_databases_df
|
||||
from .dumps import VolumeOutcome, backup_dumps_for_volume, load_databases_df
|
||||
from .layout import (
|
||||
create_version_directory,
|
||||
create_volume_directory,
|
||||
get_machine_id,
|
||||
stamp_directory,
|
||||
write_manifest,
|
||||
)
|
||||
from .policy import requires_stop, volume_is_fully_ignored
|
||||
from .snapshot import volume_snapshot
|
||||
from .volume import backup_volume, get_storage_path
|
||||
from .snapshot import snapshot_source, volume_snapshot
|
||||
from .volume import backup_volume, inspect_backing
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
machine_id = get_machine_id()
|
||||
backup_time = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
# Local wall clock on purpose: generations sort by this name, and UTC would
|
||||
# order new ones before the existing ones wherever the offset is positive.
|
||||
backup_time = datetime.now().strftime("%Y%m%d%H%M%S") # noqa: DTZ005
|
||||
|
||||
versions_dir = os.path.join(args.backups_dir, machine_id, args.repo_name)
|
||||
versions_dir = str(Path(args.backups_dir) / machine_id / args.repo_name)
|
||||
version_dir = create_version_directory(versions_dir, backup_time)
|
||||
|
||||
# IMPORTANT:
|
||||
# - keep_default_na=False prevents empty fields from turning into NaN
|
||||
# - dtype=str keeps all columns stable for comparisons/validation
|
||||
#
|
||||
# Robust behavior:
|
||||
# - if the file is missing or empty, we continue without DB dumps.
|
||||
databases_df = load_databases_df(args.databases_csv)
|
||||
databases_df = None if args.only_files else load_databases_df(args.databases_csv)
|
||||
|
||||
print("💾 Start volume backups...", flush=True)
|
||||
|
||||
outcomes: dict[str, VolumeOutcome] = {}
|
||||
|
||||
with ExitStack() as stack:
|
||||
resolve_source = None
|
||||
if args.snapshot:
|
||||
@@ -73,57 +72,59 @@ def main() -> int:
|
||||
|
||||
vol_dir = create_volume_directory(version_dir, volume_name)
|
||||
|
||||
found_db, dumped_any = backup_dumps_for_volume(
|
||||
containers=containers,
|
||||
vol_dir=vol_dir,
|
||||
databases_df=databases_df,
|
||||
database_containers=args.database_containers,
|
||||
)
|
||||
outcome = VolumeOutcome(database=False, dumped=False)
|
||||
if not args.only_files:
|
||||
outcome = backup_dumps_for_volume(
|
||||
containers=containers,
|
||||
vol_dir=vol_dir,
|
||||
databases_df=databases_df,
|
||||
database_containers=args.database_containers,
|
||||
)
|
||||
outcomes[volume_name] = outcome
|
||||
|
||||
if args.dump_only_sql:
|
||||
if found_db:
|
||||
if not dumped_any:
|
||||
print(
|
||||
f"WARNING: dump-only-sql requested but no DB dump was produced for DB volume '{volume_name}'. "
|
||||
"Falling back to file backup.",
|
||||
flush=True,
|
||||
)
|
||||
else:
|
||||
continue
|
||||
if args.only_sql and outcome.database:
|
||||
if not outcome.dumped:
|
||||
print(
|
||||
f"WARNING: only-sql requested but no DB dump was produced for DB volume '{volume_name}'. "
|
||||
"Falling back to file backup.",
|
||||
flush=True,
|
||||
)
|
||||
else:
|
||||
continue
|
||||
|
||||
live_source = get_storage_path(volume_name)
|
||||
backing = inspect_backing(volume_name)
|
||||
live_source = backing.source
|
||||
|
||||
def copy(*, authoritative: bool, source: str = live_source) -> None:
|
||||
def copy(
|
||||
*,
|
||||
authoritative: bool,
|
||||
source: str = live_source,
|
||||
volume: str = volume_name,
|
||||
target: str = vol_dir,
|
||||
) -> None:
|
||||
backup_volume(
|
||||
versions_dir,
|
||||
volume_name,
|
||||
vol_dir,
|
||||
volume,
|
||||
target,
|
||||
authoritative=authoritative,
|
||||
source=source,
|
||||
)
|
||||
|
||||
if resolve_source is not None:
|
||||
snapshot_source = resolve_source(live_source)
|
||||
if os.path.isdir(snapshot_source):
|
||||
copy(authoritative=True, source=snapshot_source)
|
||||
source, reason = snapshot_source(
|
||||
resolve_source, backing, args.snapshot_subject
|
||||
)
|
||||
if source is not None:
|
||||
copy(authoritative=True, source=source)
|
||||
else:
|
||||
print(
|
||||
f"WARNING: volume '{volume_name}' is not in the snapshot "
|
||||
"(created after it was taken); copying it live instead.",
|
||||
f"({reason}); copying it live instead.",
|
||||
flush=True,
|
||||
)
|
||||
copy(authoritative=False)
|
||||
continue
|
||||
|
||||
if args.everything:
|
||||
stoppable = filter_stoppable(containers)
|
||||
copy(authoritative=False)
|
||||
change_containers_status(stoppable, "stop")
|
||||
copy(authoritative=True)
|
||||
if not args.shutdown:
|
||||
change_containers_status(stoppable, "start")
|
||||
continue
|
||||
|
||||
copy(authoritative=False)
|
||||
if requires_stop(containers, args.images_no_stop_required):
|
||||
stoppable = filter_stoppable(containers)
|
||||
@@ -132,6 +133,7 @@ def main() -> int:
|
||||
if not args.shutdown:
|
||||
change_containers_status(stoppable, "start")
|
||||
|
||||
write_manifest(version_dir, outcomes)
|
||||
stamp_directory(version_dir)
|
||||
print("Finished volume backups.", flush=True)
|
||||
|
||||
|
||||
@@ -1,13 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
dirname = os.path.dirname(__file__)
|
||||
default_databases_csv = os.path.join(dirname, "databases.csv")
|
||||
|
||||
p = argparse.ArgumentParser(description="Backup Docker volumes.")
|
||||
|
||||
p.add_argument(
|
||||
@@ -25,13 +21,12 @@ def parse_args() -> argparse.Namespace:
|
||||
|
||||
p.add_argument(
|
||||
"--repo-name",
|
||||
default="backup-docker-to-local",
|
||||
help="Backup repo folder name under <backups-dir>/<machine-id>/ (default: git repo folder name)",
|
||||
required=True,
|
||||
help="Backup repo folder name under <backups-dir>/<machine-id>/",
|
||||
)
|
||||
p.add_argument(
|
||||
"--databases-csv",
|
||||
default=default_databases_csv,
|
||||
help=f"Path to databases.csv (default: {default_databases_csv})",
|
||||
help="Path to databases.csv; required unless --only-files is given",
|
||||
)
|
||||
p.add_argument(
|
||||
"--backups-dir",
|
||||
@@ -75,19 +70,15 @@ def parse_args() -> argparse.Namespace:
|
||||
help="Exact volume names that are never backed up, whatever containers use them. For derived trees a restore cannot reproduce, above all a nested docker data root",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--everything",
|
||||
action="store_true",
|
||||
help="Force file backup for all volumes and also execute database dumps (like old script)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--shutdown",
|
||||
action="store_true",
|
||||
help="Do not restart containers after backup",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--dump-only-sql",
|
||||
scope = p.add_mutually_exclusive_group()
|
||||
scope.add_argument(
|
||||
"--only-sql",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Create database dumps only for DB volumes. "
|
||||
@@ -96,7 +87,19 @@ def parse_args() -> argparse.Namespace:
|
||||
"If a DB dump cannot be produced, baudolo falls back to a file backup."
|
||||
),
|
||||
)
|
||||
scope.add_argument(
|
||||
"--only-files",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Take no database dumps at all and back up every volume as files. "
|
||||
"For hosts that hold no database credentials. A database's files "
|
||||
"are only consistent if its containers are stopped for the second "
|
||||
"pass, so keep its image off --images-no-stop-required."
|
||||
),
|
||||
)
|
||||
args = p.parse_args()
|
||||
if not args.only_files and not args.databases_csv:
|
||||
p.error("--databases-csv is required unless --only-files is given")
|
||||
if bool(args.snapshot) != bool(args.snapshot_subject):
|
||||
p.error("--snapshot and --snapshot-subject must be given together")
|
||||
if args.snapshot and args.shutdown:
|
||||
|
||||
@@ -4,10 +4,9 @@ import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
def _build_compose_cmd(project_dir: str, passthrough: List[str]) -> List[str]:
|
||||
def _build_compose_cmd(project_dir: str, passthrough: list[str]) -> list[str]:
|
||||
"""
|
||||
Build the compose command for this project directory.
|
||||
|
||||
@@ -30,7 +29,7 @@ def _build_compose_cmd(project_dir: str, passthrough: List[str]) -> List[str]:
|
||||
raise RuntimeError("Neither 'compose' nor 'docker' found in PATH")
|
||||
|
||||
|
||||
def _find_compose_file(project_dir: str) -> Optional[Path]:
|
||||
def _find_compose_file(project_dir: str) -> Path | None:
|
||||
"""
|
||||
Detect a compose file in `project_dir` (case-insensitive).
|
||||
|
||||
@@ -83,7 +82,7 @@ def handle_docker_compose_services(
|
||||
continue
|
||||
|
||||
dir_path = entry.path
|
||||
name = os.path.basename(dir_path)
|
||||
name = Path(dir_path).name
|
||||
|
||||
print(f"Checking directory: {dir_path}", flush=True)
|
||||
|
||||
|
||||
@@ -1,55 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import logging
|
||||
import pathlib
|
||||
import re
|
||||
import logging
|
||||
from typing import Optional
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pandas
|
||||
from baudolo.databases import CLUSTER_ROW, validate_database
|
||||
from baudolo.generation import CLUSTER_SUFFIX, DUMP_SUFFIX, SQL_DIR
|
||||
|
||||
from .shell import BackupException, execute_shell_command
|
||||
from .docker import docker_exec_argv
|
||||
from .shell import BackupError, execute_to_file
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import pandas as pd
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ENGINE_NAMES = ("database", "postgres", "mariadb", "mysql", "db")
|
||||
_SUFFIX_RE = re.compile(rf"(_|-)({'|'.join(ENGINE_NAMES)})")
|
||||
|
||||
def get_instance(container: str, database_containers: list[str]) -> str:
|
||||
"""
|
||||
Derive a stable instance name from the container name.
|
||||
|
||||
def get_instance(container: str, database_containers: list[str]) -> str | None:
|
||||
"""The databases.csv instance a container serves, or None for no database.
|
||||
|
||||
A declared container is its own instance. Every other name is read against
|
||||
ENGINE_NAMES: carrying one as a suffix makes the rest the instance, which
|
||||
maps `<app>-database` from compose and `<app>_database.1.<task>` from swarm
|
||||
onto the same one; being one outright makes the container its own instance,
|
||||
the shape a compose file writes as `container_name: postgres`.
|
||||
|
||||
Args:
|
||||
container: the running container's name.
|
||||
database_containers: names passed via --database-containers, taken as
|
||||
declared engines whatever they are called.
|
||||
|
||||
Returns:
|
||||
The instance name, or None when the name neither carries nor is an
|
||||
engine name: an application container is not an engine, even when it
|
||||
ships the client tools that would let a dump command start.
|
||||
"""
|
||||
if container in database_containers:
|
||||
return container
|
||||
return re.split(r"(_|-)(database|db|postgres)", container)[0]
|
||||
|
||||
|
||||
def _validate_database_value(value: Optional[str], *, instance: str) -> str:
|
||||
"""
|
||||
Enforce explicit database semantics:
|
||||
|
||||
- "*" => dump ALL databases (cluster dump for Postgres)
|
||||
- "<name>" => dump exactly this database
|
||||
- "" => invalid configuration (would previously result in NaN / nan.backup.sql)
|
||||
"""
|
||||
v = (value or "").strip()
|
||||
if v == "":
|
||||
raise ValueError(
|
||||
f"Invalid databases.csv entry for instance '{instance}': "
|
||||
"column 'database' must be '*' or a concrete database name (not empty)."
|
||||
)
|
||||
return v
|
||||
|
||||
|
||||
def _atomic_write_cmd(cmd: str, out_file: str) -> None:
|
||||
"""
|
||||
Write dump output atomically:
|
||||
- write to <file>.tmp
|
||||
- rename to <file> only on success
|
||||
|
||||
This prevents empty or partial dump files from being treated as valid backups.
|
||||
"""
|
||||
tmp = f"{out_file}.tmp"
|
||||
execute_shell_command(f"{cmd} > {tmp}")
|
||||
execute_shell_command(f"mv {tmp} {out_file}")
|
||||
parts = _SUFFIX_RE.split(container)
|
||||
if len(parts) > 1:
|
||||
return parts[0]
|
||||
return container if container in ENGINE_NAMES else None
|
||||
|
||||
|
||||
def fallback_pg_dumpall(
|
||||
@@ -58,11 +53,16 @@ def fallback_pg_dumpall(
|
||||
"""
|
||||
Perform a full Postgres cluster dump using pg_dumpall.
|
||||
"""
|
||||
cmd = (
|
||||
f"PGPASSWORD={password} docker exec -i {container} "
|
||||
f"pg_dumpall -U {username} -h localhost"
|
||||
execute_to_file(
|
||||
docker_exec_argv(
|
||||
container,
|
||||
["pg_dumpall", "-U", username, "-h", "localhost"],
|
||||
interactive=True,
|
||||
forward_env=["PGPASSWORD"],
|
||||
),
|
||||
out_file,
|
||||
env={"PGPASSWORD": password},
|
||||
)
|
||||
_atomic_write_cmd(cmd, out_file)
|
||||
|
||||
|
||||
def backup_database(
|
||||
@@ -70,23 +70,31 @@ def backup_database(
|
||||
container: str,
|
||||
volume_dir: str,
|
||||
db_type: str,
|
||||
databases_df: "pandas.DataFrame",
|
||||
dump_tool: str,
|
||||
databases_df: pd.DataFrame,
|
||||
database_containers: list[str],
|
||||
) -> bool:
|
||||
"""
|
||||
Backup databases for a given DB container.
|
||||
|
||||
Args:
|
||||
dump_tool: the MariaDB client found in the container, so an image
|
||||
that ships only mysqldump is dumped with the tool it has.
|
||||
|
||||
Returns True if at least one dump was produced.
|
||||
"""
|
||||
instance_name = get_instance(container, database_containers)
|
||||
if instance_name is None:
|
||||
log.debug("Container '%s' carries no database token", container)
|
||||
return False
|
||||
|
||||
entries = databases_df[databases_df["instance"] == instance_name]
|
||||
if entries.empty:
|
||||
log.debug("No database entries for instance '%s'", instance_name)
|
||||
return False
|
||||
|
||||
out_dir = os.path.join(volume_dir, "sql")
|
||||
pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True)
|
||||
out_dir = pathlib.Path(volume_dir) / SQL_DIR
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
produced = False
|
||||
|
||||
@@ -95,53 +103,74 @@ def backup_database(
|
||||
user = (getattr(row, "username", "") or "").strip()
|
||||
password = (getattr(row, "password", "") or "").strip()
|
||||
|
||||
db_value = _validate_database_value(raw_db, instance=instance_name)
|
||||
db_value = validate_database(raw_db, instance=instance_name)
|
||||
|
||||
# Explicit: dump ALL databases
|
||||
if db_value == "*":
|
||||
if db_value == CLUSTER_ROW:
|
||||
if db_type != "postgres":
|
||||
raise ValueError(
|
||||
f"databases.csv entry for instance '{instance_name}': "
|
||||
"'*' is currently only supported for Postgres."
|
||||
f"'{CLUSTER_ROW}' is currently only supported for Postgres."
|
||||
)
|
||||
|
||||
cluster_file = os.path.join(out_dir, f"{instance_name}.cluster.backup.sql")
|
||||
cluster_file = str(out_dir / f"{instance_name}{CLUSTER_SUFFIX}")
|
||||
fallback_pg_dumpall(container, user, password, cluster_file)
|
||||
produced = True
|
||||
continue
|
||||
|
||||
# Concrete database dump
|
||||
db_name = db_value
|
||||
dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
|
||||
dump_file = str(out_dir / f"{db_name}{DUMP_SUFFIX}")
|
||||
|
||||
if db_type == "mariadb":
|
||||
# Force TCP so auth matches '<user>'@'%' instead of socket -> 'localhost'.
|
||||
cmd = (
|
||||
f"docker exec {container} /usr/bin/mariadb-dump "
|
||||
f"-h 127.0.0.1 --protocol=tcp "
|
||||
f"-u {user} -p{password} {db_name}"
|
||||
execute_to_file(
|
||||
docker_exec_argv(
|
||||
container,
|
||||
[
|
||||
dump_tool,
|
||||
"-h",
|
||||
"127.0.0.1",
|
||||
"--protocol=tcp",
|
||||
"-u",
|
||||
user,
|
||||
f"-p{password}",
|
||||
db_name,
|
||||
],
|
||||
),
|
||||
dump_file,
|
||||
)
|
||||
_atomic_write_cmd(cmd, dump_file)
|
||||
produced = True
|
||||
continue
|
||||
|
||||
if db_type == "postgres":
|
||||
try:
|
||||
cmd = (
|
||||
f"PGPASSWORD={password} docker exec -i {container} "
|
||||
f"pg_dump -U {user} -d {db_name} -h localhost "
|
||||
f"--no-owner --no-privileges"
|
||||
execute_to_file(
|
||||
docker_exec_argv(
|
||||
container,
|
||||
[
|
||||
"pg_dump",
|
||||
"-U",
|
||||
user,
|
||||
"-d",
|
||||
db_name,
|
||||
"-h",
|
||||
"localhost",
|
||||
"--no-owner",
|
||||
"--no-privileges",
|
||||
],
|
||||
interactive=True,
|
||||
forward_env=["PGPASSWORD"],
|
||||
),
|
||||
dump_file,
|
||||
env={"PGPASSWORD": password},
|
||||
)
|
||||
_atomic_write_cmd(cmd, dump_file)
|
||||
produced = True
|
||||
except BackupException as e:
|
||||
# Explicit DB dump failed -> hard error
|
||||
raise BackupException(
|
||||
except BackupError as e:
|
||||
raise BackupError(
|
||||
f"Postgres dump failed for instance '{instance_name}', "
|
||||
f"database '{db_name}'. This database was explicitly configured "
|
||||
"and therefore must succeed.\n"
|
||||
f"{e}"
|
||||
)
|
||||
) from e
|
||||
continue
|
||||
|
||||
return produced
|
||||
|
||||
@@ -1,46 +1,86 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .shell import BackupException, execute_shell_command
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from .shell import BackupError, execute_shell_command
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
|
||||
def docker_exec_argv(
|
||||
container: str,
|
||||
argv: Sequence[str],
|
||||
*,
|
||||
interactive: bool = False,
|
||||
forward_env: Sequence[str] = (),
|
||||
) -> list[str]:
|
||||
"""The argv that runs *argv* inside *container*.
|
||||
|
||||
Args:
|
||||
container: the container to run in.
|
||||
argv: the command, already split.
|
||||
interactive: keep stdin open, for a command that is fed a dump.
|
||||
forward_env: names of environment variables to hand to the container.
|
||||
Passed as bare ``-e NAME``, so docker copies the value out of this
|
||||
process's own environment; spelling ``-e NAME=value`` instead would
|
||||
publish a secret in the host's process list.
|
||||
|
||||
Returns:
|
||||
The argv list.
|
||||
"""
|
||||
forwarded = [arg for name in forward_env for arg in ("-e", name)]
|
||||
return [
|
||||
"docker",
|
||||
"exec",
|
||||
*(["-i"] if interactive else []),
|
||||
*forwarded,
|
||||
container,
|
||||
*argv,
|
||||
]
|
||||
|
||||
|
||||
def get_image_info(container: str) -> str:
|
||||
return execute_shell_command(
|
||||
f"docker inspect --format '{{{{.Config.Image}}}}' {container}"
|
||||
["docker", "inspect", "--format", "{{.Config.Image}}", container]
|
||||
)[0]
|
||||
|
||||
|
||||
def image_name(container: str) -> str:
|
||||
"""The image's repository path, without registry host, tag or digest.
|
||||
def image_id(container: str) -> str:
|
||||
"""The container's image ID, identical for every replica of one image."""
|
||||
return execute_shell_command(
|
||||
["docker", "inspect", "--format", "{{.Image}}", container]
|
||||
)[0].strip()
|
||||
|
||||
A swarm node that hosts the local registry puts its own hostname in front
|
||||
of every pull, so the raw reference of a Postgres container can read
|
||||
`svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5`. Matching the
|
||||
whole reference finds "mariadb" there and dumps the database with
|
||||
mariadb-dump, which the Postgres image does not ship (exit 127). Tags bite
|
||||
the same way: `xwiki_custom:lts-postgres-tomcat`.
|
||||
|
||||
def has_tool(container: str, tool: str) -> bool:
|
||||
"""Whether *tool* runs inside the container.
|
||||
|
||||
Executes the binary rather than asking a shell for it: a distroless image
|
||||
has no shell, and `sh -c 'command -v'` would answer "absent" for every
|
||||
tool it ships.
|
||||
"""
|
||||
reference = get_image_info(container).strip().split("@", 1)[0]
|
||||
head, _, tail = reference.rpartition("/")
|
||||
tail = tail.split(":", 1)[0]
|
||||
if head:
|
||||
registry = head.split("/", 1)[0]
|
||||
if "." in registry or ":" in registry or registry == "localhost":
|
||||
head = head.partition("/")[2]
|
||||
return f"{head}/{tail}" if head else tail
|
||||
|
||||
|
||||
def has_image(container: str, pattern: str) -> bool:
|
||||
"""Return True if the container's image name contains the pattern."""
|
||||
return pattern in image_name(container)
|
||||
try:
|
||||
execute_shell_command(docker_exec_argv(container, [tool, "--version"]))
|
||||
except BackupError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def docker_volume_names() -> list[str]:
|
||||
return execute_shell_command("docker volume ls --format '{{.Name}}'")
|
||||
return execute_shell_command(["docker", "volume", "ls", "--format", "{{.Name}}"])
|
||||
|
||||
|
||||
def containers_using_volume(volume_name: str) -> list[str]:
|
||||
return execute_shell_command(
|
||||
f"docker ps --filter volume=\"{volume_name}\" --format '{{{{.Names}}}}'"
|
||||
[
|
||||
"docker",
|
||||
"ps",
|
||||
"--filter",
|
||||
f"volume={volume_name}",
|
||||
"--format",
|
||||
"{{.Names}}",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -54,12 +94,25 @@ def is_swarm_task(container: str) -> bool:
|
||||
keeps failing the run loudly instead of silently skipping the stop."""
|
||||
try:
|
||||
out = execute_shell_command(
|
||||
"docker inspect --format "
|
||||
f"'{{{{index .Config.Labels \"com.docker.swarm.task.id\"}}}}' {container}"
|
||||
[
|
||||
"docker",
|
||||
"inspect",
|
||||
"--format",
|
||||
'{{index .Config.Labels "com.docker.swarm.task.id"}}',
|
||||
container,
|
||||
]
|
||||
)
|
||||
except BackupException:
|
||||
except BackupError:
|
||||
still_listed = execute_shell_command(
|
||||
f"docker ps -a --filter name=^{container}$ --format '{{{{.Names}}}}'"
|
||||
[
|
||||
"docker",
|
||||
"ps",
|
||||
"-a",
|
||||
"--filter",
|
||||
f"name=^{container}$",
|
||||
"--format",
|
||||
"{{.Names}}",
|
||||
]
|
||||
)
|
||||
if still_listed and still_listed[0].strip():
|
||||
raise
|
||||
@@ -86,17 +139,5 @@ def change_containers_status(containers: list[str], status: str) -> None:
|
||||
if not containers:
|
||||
print(f"No containers to {status}.", flush=True)
|
||||
return
|
||||
names = " ".join(containers)
|
||||
print(f"{status.capitalize()} containers: {names}...", flush=True)
|
||||
execute_shell_command(f"docker {status} {names}")
|
||||
|
||||
|
||||
def docker_volume_exists(volume: str) -> bool:
|
||||
# Avoid throwing exceptions for exists checks.
|
||||
try:
|
||||
execute_shell_command(
|
||||
f"docker volume inspect {volume} >/dev/null 2>&1 && echo OK"
|
||||
)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
print(f"{status.capitalize()} containers: {' '.join(containers)}...", flush=True)
|
||||
execute_shell_command(["docker", status, *containers])
|
||||
|
||||
@@ -3,48 +3,104 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from typing import NamedTuple
|
||||
|
||||
import pandas
|
||||
import pandas as pd
|
||||
from pandas.errors import EmptyDataError
|
||||
|
||||
from .db import backup_database
|
||||
from .docker import has_image
|
||||
from baudolo.databases import COLUMNS, DELIMITER
|
||||
|
||||
from .db import backup_database, get_instance
|
||||
from .docker import has_tool, image_id
|
||||
|
||||
DUMP_TOOLS: tuple[tuple[str, str], ...] = (
|
||||
("postgres", "pg_dumpall"),
|
||||
("mariadb", "mariadb-dump"),
|
||||
("mariadb", "mysqldump"),
|
||||
)
|
||||
|
||||
_ENGINE_BY_IMAGE: dict[str, tuple[str, str] | None] = {}
|
||||
|
||||
|
||||
class VolumeOutcome(NamedTuple):
|
||||
"""What a dump attempt established about one volume.
|
||||
|
||||
``database`` says a container serving the volume speaks an engine this
|
||||
tool can dump; ``dumped`` says a dump was actually written. ``engine`` is
|
||||
the engine that was detected, or None when none was.
|
||||
"""
|
||||
|
||||
database: bool
|
||||
dumped: bool
|
||||
engine: str | None = None
|
||||
|
||||
|
||||
def container_engine(container: str) -> tuple[str, str] | None:
|
||||
"""The (engine, dump tool) a container can serve, or None for neither.
|
||||
|
||||
Asks the container what it can run instead of reading its image name. A
|
||||
dedicated Postgres is tagged `<app>-database` or `postgis/postgis` and
|
||||
carries no engine token at all, while a swarm registry host such as
|
||||
`svc-db-mariadb-swarm-mgr-01:5000` carries the wrong one.
|
||||
|
||||
Args:
|
||||
container: must be running - `docker exec` is the probe, and a
|
||||
stopped container would be cached as "no engine" for its whole
|
||||
image. The only caller feeds it `docker ps` output.
|
||||
|
||||
Returns:
|
||||
The engine and the tool that dumps it, cached per image ID so that
|
||||
replicas of one image are probed once.
|
||||
"""
|
||||
image = image_id(container)
|
||||
if image not in _ENGINE_BY_IMAGE:
|
||||
_ENGINE_BY_IMAGE[image] = next(
|
||||
(
|
||||
(engine, tool)
|
||||
for engine, tool in DUMP_TOOLS
|
||||
if has_tool(container, tool)
|
||||
),
|
||||
None,
|
||||
)
|
||||
return _ENGINE_BY_IMAGE[image]
|
||||
|
||||
|
||||
def backup_mariadb_or_postgres(
|
||||
*,
|
||||
container: str,
|
||||
volume_dir: str,
|
||||
databases_df: "pandas.DataFrame",
|
||||
databases_df: pd.DataFrame,
|
||||
database_containers: list[str],
|
||||
) -> tuple[bool, bool]:
|
||||
"""
|
||||
Returns (is_db_container, dumped_any)
|
||||
"""
|
||||
for img in ["mariadb", "postgres"]:
|
||||
if has_image(container, img):
|
||||
dumped = backup_database(
|
||||
container=container,
|
||||
volume_dir=volume_dir,
|
||||
db_type=img,
|
||||
databases_df=databases_df,
|
||||
database_containers=database_containers,
|
||||
)
|
||||
return True, dumped
|
||||
return False, False
|
||||
) -> VolumeOutcome:
|
||||
"""What this container contributes to its volume's outcome."""
|
||||
engine = container_engine(container)
|
||||
if engine is None:
|
||||
return VolumeOutcome(database=False, dumped=False)
|
||||
if get_instance(container, database_containers) is None:
|
||||
return VolumeOutcome(database=False, dumped=False)
|
||||
db_type, dump_tool = engine
|
||||
dumped = backup_database(
|
||||
container=container,
|
||||
volume_dir=volume_dir,
|
||||
db_type=db_type,
|
||||
dump_tool=dump_tool,
|
||||
databases_df=databases_df,
|
||||
database_containers=database_containers,
|
||||
)
|
||||
return VolumeOutcome(database=True, dumped=dumped, engine=db_type)
|
||||
|
||||
|
||||
def _empty_databases_df() -> "pandas.DataFrame":
|
||||
def _empty_databases_df() -> pd.DataFrame:
|
||||
"""
|
||||
Create an empty DataFrame with the expected schema for databases.csv.
|
||||
|
||||
This allows the backup to continue without DB dumps when the CSV is missing
|
||||
or empty (pandas EmptyDataError).
|
||||
"""
|
||||
return pandas.DataFrame(columns=["instance", "database", "username", "password"])
|
||||
return pd.DataFrame(columns=list(COLUMNS))
|
||||
|
||||
|
||||
def load_databases_df(csv_path: str) -> "pandas.DataFrame":
|
||||
def load_databases_df(csv_path: str) -> pd.DataFrame:
|
||||
"""
|
||||
Load databases.csv robustly.
|
||||
|
||||
@@ -53,7 +109,7 @@ def load_databases_df(csv_path: str) -> "pandas.DataFrame":
|
||||
- Valid CSV -> return dataframe
|
||||
"""
|
||||
try:
|
||||
return pandas.read_csv(csv_path, sep=";", keep_default_na=False, dtype=str)
|
||||
return pd.read_csv(csv_path, sep=DELIMITER, keep_default_na=False, dtype=str)
|
||||
except FileNotFoundError:
|
||||
print(
|
||||
f"WARNING: databases.csv not found: {csv_path}. Continuing without database dumps.",
|
||||
@@ -74,25 +130,26 @@ def backup_dumps_for_volume(
|
||||
*,
|
||||
containers: list[str],
|
||||
vol_dir: str,
|
||||
databases_df: "pandas.DataFrame",
|
||||
databases_df: pd.DataFrame,
|
||||
database_containers: list[str],
|
||||
) -> tuple[bool, bool]:
|
||||
"""
|
||||
Returns (found_db_container, dumped_any)
|
||||
"""
|
||||
) -> VolumeOutcome:
|
||||
"""The volume's outcome across every container that mounts it."""
|
||||
found_db = False
|
||||
dumped_any = False
|
||||
engine: str | None = None
|
||||
|
||||
for c in containers:
|
||||
is_db, dumped = backup_mariadb_or_postgres(
|
||||
outcome = backup_mariadb_or_postgres(
|
||||
container=c,
|
||||
volume_dir=vol_dir,
|
||||
databases_df=databases_df,
|
||||
database_containers=database_containers,
|
||||
)
|
||||
if is_db:
|
||||
if outcome.database:
|
||||
found_db = True
|
||||
if dumped:
|
||||
if outcome.dumped:
|
||||
dumped_any = True
|
||||
if engine is None:
|
||||
engine = outcome.engine
|
||||
|
||||
return found_db, dumped_any
|
||||
return VolumeOutcome(database=found_db, dumped=dumped_any, engine=engine)
|
||||
|
||||
@@ -2,16 +2,18 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import json
|
||||
import pathlib
|
||||
|
||||
from dirval import create_stamp_file
|
||||
|
||||
from .shell import execute_shell_command
|
||||
from baudolo.generation import MANIFEST_FILE, manifest_document
|
||||
|
||||
from .shell import BackupError, execute_shell_command
|
||||
|
||||
|
||||
def get_machine_id() -> str:
|
||||
return execute_shell_command("sha256sum /etc/machine-id")[0][0:64]
|
||||
return execute_shell_command(["sha256sum", "/etc/machine-id"])[0][0:64]
|
||||
|
||||
|
||||
def stamp_directory(version_dir: str) -> None:
|
||||
@@ -22,12 +24,38 @@ def stamp_directory(version_dir: str) -> None:
|
||||
|
||||
|
||||
def create_version_directory(versions_dir: str, backup_time: str) -> str:
|
||||
version_dir = os.path.join(versions_dir, backup_time)
|
||||
pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
|
||||
version_dir = str(pathlib.Path(versions_dir) / backup_time)
|
||||
try:
|
||||
pathlib.Path(version_dir).mkdir(parents=True)
|
||||
except FileExistsError:
|
||||
raise BackupError(
|
||||
f"generation {backup_time} already exists at {version_dir}; "
|
||||
"another run claimed this second - refusing to write into it, "
|
||||
"since rsync --delete would overwrite that generation"
|
||||
) from None
|
||||
return version_dir
|
||||
|
||||
|
||||
def create_volume_directory(version_dir: str, volume_name: str) -> str:
|
||||
path = os.path.join(version_dir, volume_name)
|
||||
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
path = pathlib.Path(version_dir) / volume_name
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return str(path)
|
||||
|
||||
|
||||
def write_manifest(version_dir: str, volumes: dict[str, dict[str, bool]]) -> str:
|
||||
"""Record the generation's layout and per-volume outcome.
|
||||
|
||||
Written before the directory is stamped, so the stamp covers it.
|
||||
|
||||
Args:
|
||||
version_dir: the generation directory.
|
||||
volumes: per volume name, ``database`` and ``dumped``.
|
||||
|
||||
Returns:
|
||||
The path written.
|
||||
"""
|
||||
path = pathlib.Path(version_dir) / MANIFEST_FILE
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
json.dump(manifest_document(volumes), handle, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
return str(path)
|
||||
|
||||
@@ -1,26 +1,75 @@
|
||||
"""Running external commands without a shell.
|
||||
|
||||
Every command is an argv list. A database name, a password or a container name
|
||||
therefore cannot close a quote and start a second command, which a formatted
|
||||
string handed to ``shell=True`` allowed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
||||
|
||||
class BackupException(Exception):
|
||||
class BackupError(Exception):
|
||||
"""Generic exception for backup errors."""
|
||||
|
||||
|
||||
def execute_shell_command(command: str) -> list[str]:
|
||||
"""Execute a shell command and return its output lines."""
|
||||
print(command, flush=True)
|
||||
def _child_env(env: Mapping[str, str] | None) -> dict[str, str] | None:
|
||||
return None if env is None else {**os.environ, **env}
|
||||
|
||||
|
||||
def _fail(command: Sequence[str], returncode: int, out: bytes, err: bytes) -> None:
|
||||
raise BackupError(
|
||||
f"Error in command: {' '.join(command)}\n"
|
||||
f"Output: {out}\nError: {err}\n"
|
||||
f"Exit code: {returncode}"
|
||||
)
|
||||
|
||||
|
||||
def execute_shell_command(
|
||||
command: Sequence[str], *, env: Mapping[str, str] | None = None
|
||||
) -> list[str]:
|
||||
"""Run *command* and return its stdout lines.
|
||||
|
||||
Args:
|
||||
command: argv, the program first.
|
||||
env: variables added to the child's environment, for values that must
|
||||
not appear in the argv of a process listing.
|
||||
"""
|
||||
command = list(command)
|
||||
print(" ".join(command), flush=True)
|
||||
process = subprocess.Popen(
|
||||
[command],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
shell=True,
|
||||
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=_child_env(env)
|
||||
)
|
||||
out, err = process.communicate()
|
||||
if process.returncode != 0:
|
||||
raise BackupException(
|
||||
f"Error in command: {command}\n"
|
||||
f"Output: {out}\nError: {err}\n"
|
||||
f"Exit code: {process.returncode}"
|
||||
)
|
||||
_fail(command, process.returncode, out, err)
|
||||
return [line.decode("utf-8") for line in out.splitlines()]
|
||||
|
||||
|
||||
def execute_to_file(
|
||||
command: Sequence[str], out_file: str, *, env: Mapping[str, str] | None = None
|
||||
) -> None:
|
||||
"""Run *command*, writing its stdout to *out_file* only once it succeeded.
|
||||
|
||||
The output goes to a sibling temporary file first, so a partial or empty
|
||||
stream from a failing dump never takes the place of a valid backup.
|
||||
"""
|
||||
command = list(command)
|
||||
print(" ".join(command), flush=True)
|
||||
tmp = Path(f"{out_file}.tmp")
|
||||
with tmp.open("wb") as handle:
|
||||
process = subprocess.Popen(
|
||||
command, stdout=handle, stderr=subprocess.PIPE, env=_child_env(env)
|
||||
)
|
||||
_, err = process.communicate()
|
||||
if process.returncode != 0:
|
||||
tmp.unlink()
|
||||
_fail(command, process.returncode, b"", err)
|
||||
tmp.replace(out_file)
|
||||
|
||||
@@ -9,15 +9,28 @@ built for. That also removes the reason to stop containers at all.
|
||||
The snapshot kind is stated by the caller rather than probed, because falling
|
||||
back to a live copy when a probe is inconclusive would hand out backups that
|
||||
look consistent and are not.
|
||||
|
||||
Which volumes a snapshot of the subject contains is a different question, and
|
||||
it is decided per volume: a volume with a backing store of its own appears
|
||||
inside the snapshot as an existing empty directory, so copying from there
|
||||
succeeds and stores nothing. Such a volume is copied live instead - correct
|
||||
data without the point in time - while every other volume of the same run
|
||||
keeps its snapshot.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from .shell import BackupException, execute_shell_command
|
||||
from .shell import BackupError, execute_shell_command
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Iterator
|
||||
|
||||
from .volume import Backing
|
||||
|
||||
KINDS = ("btrfs", "zfs")
|
||||
|
||||
@@ -28,10 +41,15 @@ class SnapshotError(RuntimeError):
|
||||
|
||||
def _resolver(subject: str, root: str) -> Callable[[str], str]:
|
||||
def resolve(path: str) -> str:
|
||||
relative = os.path.relpath(os.path.abspath(path), os.path.abspath(subject))
|
||||
# Exception: abspath, not Path.resolve() - resolve() follows symlinks,
|
||||
# which would let a symlinked volume test as inside the subject.
|
||||
relative = os.path.relpath(
|
||||
os.path.abspath(path), # noqa: PTH100
|
||||
os.path.abspath(subject), # noqa: PTH100
|
||||
)
|
||||
if relative.startswith(".."):
|
||||
raise SnapshotError(f"{path} lies outside the snapshot subject {subject}")
|
||||
resolved = root if relative == "." else os.path.join(root, relative)
|
||||
resolved = root if relative == "." else str(Path(root) / relative)
|
||||
|
||||
# abspath drops a trailing separator, and rsync reads "dir/" as its
|
||||
# contents where "dir" means the directory itself.
|
||||
@@ -40,34 +58,94 @@ def _resolver(subject: str, root: str) -> Callable[[str], str]:
|
||||
return resolve
|
||||
|
||||
|
||||
def _btrfs(subject: str, name: str, run: Callable[[str], list[str]]) -> tuple[str, str]:
|
||||
def _btrfs(
|
||||
subject: str, name: str, run: Callable[[list[str]], list[str]]
|
||||
) -> tuple[str, list[str]]:
|
||||
# The snapshot goes inside the subject, never beside it: the kernel rejects
|
||||
# a snapshot whose destination is on another filesystem, which is exactly
|
||||
# what the parent directory is when the subject is a mountpoint of its own.
|
||||
target = os.path.join(os.path.abspath(subject), f".{name}")
|
||||
run(f"btrfs subvolume snapshot -r {subject} {target}")
|
||||
return target, f"btrfs subvolume delete {target}"
|
||||
target = str(Path(os.path.abspath(subject)) / f".{name}") # noqa: PTH100 - see _resolver
|
||||
run(["btrfs", "subvolume", "snapshot", "-r", subject, target])
|
||||
return target, ["btrfs", "subvolume", "delete", target]
|
||||
|
||||
|
||||
def _zfs(subject: str, name: str, run: Callable[[str], list[str]]) -> tuple[str, str]:
|
||||
output = run(f"zfs list -H -o name {subject}")
|
||||
def _zfs(
|
||||
subject: str, name: str, run: Callable[[list[str]], list[str]]
|
||||
) -> tuple[str, list[str]]:
|
||||
output = run(["zfs", "list", "-H", "-o", "name", subject])
|
||||
dataset = (output[0] if output else "").strip()
|
||||
if not dataset:
|
||||
raise SnapshotError(f"no zfs dataset is mounted at {subject}")
|
||||
run(f"zfs snapshot {dataset}@{name}")
|
||||
root = os.path.join(subject, ".zfs", "snapshot", name)
|
||||
return root, f"zfs destroy {dataset}@{name}"
|
||||
run(["zfs", "snapshot", f"{dataset}@{name}"])
|
||||
root = str(Path(subject) / ".zfs" / "snapshot" / name)
|
||||
return root, ["zfs", "destroy", f"{dataset}@{name}"]
|
||||
|
||||
|
||||
_CREATE = {"btrfs": _btrfs, "zfs": _zfs}
|
||||
|
||||
|
||||
def unsnapshotted(backing: Backing, subject: str) -> str | None:
|
||||
"""Return why a snapshot of ``subject`` does not hold this volume's data.
|
||||
|
||||
Docker mounts a volume's own backing store lazily and unmounts it when the
|
||||
last consumer stops, so the declaration is what gets checked: it is true at
|
||||
every moment, where the mount table is only true while a container happens
|
||||
to hold the volume.
|
||||
|
||||
Args:
|
||||
backing: the volume as the daemon describes it.
|
||||
subject: the snapshot subject, e.g. ``/var/lib/docker``.
|
||||
|
||||
Returns:
|
||||
The reason, or None when the snapshot holds the volume.
|
||||
"""
|
||||
if backing.driver != "local":
|
||||
return f"it uses the {backing.driver} driver"
|
||||
if backing.options:
|
||||
return f"it declares its own backing store {backing.options}"
|
||||
if not backing.mountpoint:
|
||||
return "it reports no mountpoint"
|
||||
real = os.path.realpath(backing.mountpoint)
|
||||
if os.path.ismount(real):
|
||||
return f"its mountpoint {backing.mountpoint} sits on its own mount"
|
||||
try:
|
||||
crosses = (
|
||||
Path(real).stat().st_dev != Path(os.path.realpath(subject)).stat().st_dev
|
||||
)
|
||||
except OSError as error:
|
||||
return f"its mountpoint {backing.mountpoint} could not be read: {error}"
|
||||
if crosses:
|
||||
return f"its mountpoint {backing.mountpoint} crosses a filesystem boundary"
|
||||
return None
|
||||
|
||||
|
||||
def snapshot_source(
|
||||
resolve: Callable[[str], str], backing: Backing, subject: str
|
||||
) -> tuple[str | None, str]:
|
||||
"""Resolve where to read a volume from, and why if not from the snapshot.
|
||||
|
||||
Returns:
|
||||
``(path, "")`` to copy from the snapshot, or ``(None, reason)`` to copy
|
||||
it live.
|
||||
"""
|
||||
reason = unsnapshotted(backing, subject)
|
||||
if reason:
|
||||
return None, reason
|
||||
try:
|
||||
source = resolve(backing.source)
|
||||
except SnapshotError as error:
|
||||
return None, str(error)
|
||||
if not Path(source).is_dir():
|
||||
return None, "it was created after the snapshot was taken"
|
||||
return source, ""
|
||||
|
||||
|
||||
@contextmanager
|
||||
def volume_snapshot(
|
||||
kind: str,
|
||||
subject: str,
|
||||
tag: str,
|
||||
run: Callable[[str], list[str]] = execute_shell_command,
|
||||
run: Callable[[list[str]], list[str]] = execute_shell_command,
|
||||
) -> Iterator[Callable[[str], str]]:
|
||||
"""Yield a resolver mapping a path under ``subject`` into a snapshot of it.
|
||||
|
||||
@@ -93,6 +171,6 @@ def volume_snapshot(
|
||||
finally:
|
||||
try:
|
||||
run(remove)
|
||||
except BackupException as error:
|
||||
except BackupError as error:
|
||||
# Raising here would also mask whatever the body raised.
|
||||
print(f"WARNING: {root} could not be removed: {error}", flush=True)
|
||||
|
||||
@@ -1,16 +1,45 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .shell import BackupException, execute_shell_command
|
||||
from baudolo.generation import FILES_DIR
|
||||
|
||||
from .shell import BackupError, execute_shell_command
|
||||
|
||||
|
||||
def get_storage_path(volume_name: str) -> str:
|
||||
path = execute_shell_command(
|
||||
f"docker volume inspect --format '{{{{ .Mountpoint }}}}' {volume_name}"
|
||||
@dataclass(frozen=True)
|
||||
class Backing:
|
||||
"""Where a docker volume actually keeps its data.
|
||||
|
||||
Args:
|
||||
mountpoint: the path the daemon reports.
|
||||
driver: the volume driver, ``local`` for the built-in one.
|
||||
options: the driver options; a non-empty map means the mountpoint is a
|
||||
mount target rather than the storage itself.
|
||||
"""
|
||||
|
||||
mountpoint: str
|
||||
driver: str = "local"
|
||||
options: dict = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def source(self) -> str:
|
||||
return f"{self.mountpoint}/"
|
||||
|
||||
|
||||
def inspect_backing(volume_name: str) -> Backing:
|
||||
reported = execute_shell_command(
|
||||
["docker", "volume", "inspect", "--format", "{{json .}}", volume_name]
|
||||
)[0]
|
||||
return f"{path}/"
|
||||
data = json.loads(reported)
|
||||
return Backing(
|
||||
data.get("Mountpoint") or "",
|
||||
data.get("Driver") or "",
|
||||
data.get("Options") or {},
|
||||
)
|
||||
|
||||
|
||||
def get_last_backup_dir(
|
||||
@@ -18,8 +47,8 @@ def get_last_backup_dir(
|
||||
) -> str | None:
|
||||
versions = sorted(os.listdir(versions_dir), reverse=True)
|
||||
for version in versions:
|
||||
candidate = os.path.join(versions_dir, version, volume_name, "files", "")
|
||||
if candidate != current_backup_dir and os.path.isdir(candidate):
|
||||
candidate = f"{pathlib.Path(versions_dir) / version / volume_name / FILES_DIR}/"
|
||||
if candidate != current_backup_dir and pathlib.Path(candidate).is_dir():
|
||||
return candidate
|
||||
return None
|
||||
|
||||
@@ -42,21 +71,20 @@ def backup_volume(
|
||||
source: directory to read from - the volume's mountpoint, or its path
|
||||
inside a snapshot.
|
||||
"""
|
||||
dest = os.path.join(volume_dir, "files") + "/"
|
||||
dest = f"{pathlib.Path(volume_dir) / FILES_DIR}/"
|
||||
pathlib.Path(dest).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
last = get_last_backup_dir(versions_dir, volume_name, dest)
|
||||
link_dest = f"--link-dest='{last}'" if last else ""
|
||||
verify = "--checksum " if authoritative else ""
|
||||
|
||||
cmd = (
|
||||
f"rsync -aP --no-D --delete --delete-excluded "
|
||||
f"{verify}{link_dest} {source} {dest}"
|
||||
)
|
||||
cmd = ["rsync", "-aP", "--no-D", "--delete", "--delete-excluded"]
|
||||
if authoritative:
|
||||
cmd.append("--checksum")
|
||||
if last:
|
||||
cmd.append(f"--link-dest={last}")
|
||||
cmd += [source, dest]
|
||||
|
||||
try:
|
||||
execute_shell_command(cmd)
|
||||
except BackupException as e:
|
||||
except BackupError as e:
|
||||
if "file has vanished" in str(e):
|
||||
print(
|
||||
"Warning: Some files vanished before transfer. Continuing.", flush=True
|
||||
|
||||
103
src/baudolo/databases.py
Normal file
103
src/baudolo/databases.py
Normal file
@@ -0,0 +1,103 @@
|
||||
"""The databases.csv contract: its columns, its delimiter, and what a row means.
|
||||
|
||||
``baudolo-seed`` writes the file, the backup reads it to learn which dumps to
|
||||
take, and a restore consumer reads it again to replay them. Stating the schema
|
||||
once keeps a column or a convention added here from being invisible to the
|
||||
other two.
|
||||
|
||||
Field values are handed back exactly as they stand in the file. A password may
|
||||
legitimately begin or end with a space, so stripping belongs to the caller that
|
||||
compares, never to the reader.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple
|
||||
|
||||
COLUMNS = ("instance", "database", "username", "password")
|
||||
DELIMITER = ";"
|
||||
CLUSTER_ROW = "*"
|
||||
|
||||
_NAME_RE = re.compile(r"^[a-zA-Z0-9_][a-zA-Z0-9_-]*$")
|
||||
|
||||
|
||||
class DatabasesCsvError(ValueError):
|
||||
"""A row does not match the contract."""
|
||||
|
||||
|
||||
class Row(NamedTuple):
|
||||
"""One databases.csv row, verbatim.
|
||||
|
||||
``database`` holds :data:`CLUSTER_ROW` when the whole instance is dumped.
|
||||
"""
|
||||
|
||||
instance: str
|
||||
database: str
|
||||
username: str
|
||||
password: str
|
||||
|
||||
@property
|
||||
def is_cluster(self) -> bool:
|
||||
return self.database.strip() == CLUSTER_ROW
|
||||
|
||||
|
||||
def validate_database(value: str | None, *, instance: str) -> str:
|
||||
"""The database column of one row, or raise.
|
||||
|
||||
The name reaches a shell as part of the dump command, so it is checked
|
||||
where it is read as well as where it is written: a file edited by hand
|
||||
never passed the seed.
|
||||
|
||||
Args:
|
||||
value: the raw column.
|
||||
instance: named in the error, so a bad row can be found.
|
||||
|
||||
Raises:
|
||||
DatabasesCsvError: the column is empty, literally ``nan``, or holds
|
||||
anything but letters, numbers, ``_`` and ``-``.
|
||||
"""
|
||||
text = (value or "").strip()
|
||||
if not text:
|
||||
raise DatabasesCsvError(
|
||||
f"Invalid databases.csv entry for instance '{instance}': column "
|
||||
f"'database' must be '{CLUSTER_ROW}' or a concrete database name "
|
||||
"(not empty)."
|
||||
)
|
||||
if text == CLUSTER_ROW:
|
||||
return CLUSTER_ROW
|
||||
if text.lower() == "nan":
|
||||
raise DatabasesCsvError(
|
||||
f"Invalid databases.csv entry for instance '{instance}': "
|
||||
"database must not be 'nan'."
|
||||
)
|
||||
if not _NAME_RE.match(text):
|
||||
raise DatabasesCsvError(
|
||||
f"Invalid databases.csv entry for instance '{instance}': invalid "
|
||||
f"database name '{text}'. Allowed: letters, numbers, '_' and '-'."
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def read_rows(csv_path: str) -> list[Row]:
|
||||
"""Every row of the file in file order, header skipped, blank rows dropped.
|
||||
|
||||
Raises:
|
||||
DatabasesCsvError: a row holds fewer columns than :data:`COLUMNS`.
|
||||
"""
|
||||
rows: list[Row] = []
|
||||
with Path(csv_path).open(newline="", encoding="utf-8") as handle:
|
||||
reader = csv.reader(handle, delimiter=DELIMITER)
|
||||
next(reader, None)
|
||||
for raw in reader:
|
||||
if not any(field.strip() for field in raw):
|
||||
continue
|
||||
if len(raw) < len(COLUMNS):
|
||||
raise DatabasesCsvError(
|
||||
f"{csv_path} has a row with {len(raw)} column(s), "
|
||||
f"expected {len(COLUMNS)}"
|
||||
)
|
||||
rows.append(Row(*raw[: len(COLUMNS)]))
|
||||
return rows
|
||||
54
src/baudolo/generation.py
Normal file
54
src/baudolo/generation.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""The on-disk shape of a generation, and the manifest that states it.
|
||||
|
||||
Every name a reader needs to find payload in a generation is declared here
|
||||
once, and written into each generation's own manifest. A consumer therefore
|
||||
never has to hardcode the layout or match this package's version: it reads
|
||||
what the run that produced the tree recorded.
|
||||
|
||||
The manifest also carries what only the run itself can know: per volume,
|
||||
``database`` (it held one), ``dumped`` (a dump was produced for it) and
|
||||
``engine`` (which one was detected). Both flags true is a replayable dump;
|
||||
``database`` without ``dumped`` is a raw copy of live engine files.
|
||||
|
||||
Kept import-free: consumers read the manifest with nothing but ``json``, on
|
||||
hosts that do not have this package installed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
FILES_DIR = "files"
|
||||
SQL_DIR = "sql"
|
||||
DUMP_SUFFIX = ".backup.sql"
|
||||
CLUSTER_SUFFIX = ".cluster.backup.sql"
|
||||
|
||||
MANIFEST_FILE = "manifest.json"
|
||||
MANIFEST_SCHEMA = 1
|
||||
|
||||
|
||||
def manifest_document(volumes: dict[str, object]) -> dict[str, object]:
|
||||
"""The manifest a finished run writes.
|
||||
|
||||
Args:
|
||||
volumes: per volume name, an object carrying ``database``, ``dumped``
|
||||
and ``engine`` -- a ``baudolo.backup.dumps.VolumeOutcome``.
|
||||
|
||||
Returns:
|
||||
The document, ready for ``json.dump``.
|
||||
"""
|
||||
return {
|
||||
"schema": MANIFEST_SCHEMA,
|
||||
"layout": {
|
||||
"files_dir": FILES_DIR,
|
||||
"sql_dir": SQL_DIR,
|
||||
"dump_suffix": DUMP_SUFFIX,
|
||||
"cluster_suffix": CLUSTER_SUFFIX,
|
||||
},
|
||||
"volumes": {
|
||||
name: {
|
||||
"database": bool(outcome.database),
|
||||
"dumped": bool(outcome.dumped),
|
||||
"engine": outcome.engine,
|
||||
}
|
||||
for name, outcome in sorted(volumes.items())
|
||||
},
|
||||
}
|
||||
@@ -3,10 +3,11 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
from .paths import BackupPaths
|
||||
from .files import restore_volume_files
|
||||
from .db.postgres import restore_postgres_sql
|
||||
from .db.cluster import restore_cluster_sql
|
||||
from .db.mariadb import restore_mariadb_sql
|
||||
from .db.postgres import restore_postgres_sql
|
||||
from .files import restore_volume_files
|
||||
from .paths import BackupPaths
|
||||
|
||||
|
||||
def _add_common_backup_args(p: argparse.ArgumentParser) -> None:
|
||||
@@ -21,8 +22,22 @@ def _add_common_backup_args(p: argparse.ArgumentParser) -> None:
|
||||
)
|
||||
p.add_argument(
|
||||
"--repo-name",
|
||||
default="backup-docker-to-local",
|
||||
help="Backup repo folder name under <backups-dir>/<hash>/ (default: backup-docker-to-local)",
|
||||
required=True,
|
||||
help="Backup repo folder name under <backups-dir>/<hash>/",
|
||||
)
|
||||
|
||||
|
||||
def _add_common_engine_args(p: argparse.ArgumentParser) -> None:
|
||||
p.add_argument("--container", required=True)
|
||||
p.add_argument("--db-password", required=True)
|
||||
p.add_argument("--empty", action="store_true")
|
||||
p.add_argument(
|
||||
"--no-version-check",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Replay even if the dump comes from a newer engine than the target. "
|
||||
"With --empty this can leave an emptied database behind."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -33,9 +48,6 @@ def main(argv: list[str] | None = None) -> int:
|
||||
)
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# files
|
||||
# ------------------------------------------------------------------
|
||||
p_files = sub.add_parser("files", help="Restore files into a docker volume")
|
||||
_add_common_backup_args(p_files)
|
||||
p_files.add_argument(
|
||||
@@ -48,36 +60,40 @@ def main(argv: list[str] | None = None) -> int:
|
||||
),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# postgres
|
||||
# ------------------------------------------------------------------
|
||||
p_pg = sub.add_parser("postgres", help="Restore a single PostgreSQL database dump")
|
||||
_add_common_backup_args(p_pg)
|
||||
p_pg.add_argument("--container", required=True)
|
||||
_add_common_engine_args(p_pg)
|
||||
p_pg.add_argument("--db-name", required=True)
|
||||
p_pg.add_argument("--db-user", default=None, help="Defaults to db-name if omitted")
|
||||
p_pg.add_argument("--db-password", required=True)
|
||||
p_pg.add_argument("--empty", action="store_true")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# mariadb
|
||||
# ------------------------------------------------------------------
|
||||
p_cluster = sub.add_parser(
|
||||
"cluster", help="Restore a full PostgreSQL cluster dump (pg_dumpall)"
|
||||
)
|
||||
_add_common_backup_args(p_cluster)
|
||||
_add_common_engine_args(p_cluster)
|
||||
p_cluster.add_argument(
|
||||
"--instance",
|
||||
required=True,
|
||||
help="Instance the dump was taken from; names <instance>.cluster.backup.sql",
|
||||
)
|
||||
p_cluster.add_argument(
|
||||
"--db-user",
|
||||
required=True,
|
||||
help="Superuser of the instance; the dump creates roles and databases",
|
||||
)
|
||||
|
||||
p_mdb = sub.add_parser(
|
||||
"mariadb", help="Restore a single MariaDB/MySQL-compatible dump"
|
||||
)
|
||||
_add_common_backup_args(p_mdb)
|
||||
p_mdb.add_argument("--container", required=True)
|
||||
_add_common_engine_args(p_mdb)
|
||||
p_mdb.add_argument("--db-name", required=True)
|
||||
p_mdb.add_argument("--db-user", default=None, help="Defaults to db-name if omitted")
|
||||
p_mdb.add_argument("--db-password", required=True)
|
||||
p_mdb.add_argument("--empty", action="store_true")
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
if args.cmd == "files":
|
||||
# target volume = args.volume_name
|
||||
# source volume (backup key) defaults to target volume
|
||||
source_volume = args.source_volume or args.volume_name
|
||||
|
||||
bp_files = BackupPaths(
|
||||
@@ -108,6 +124,24 @@ def main(argv: list[str] | None = None) -> int:
|
||||
backups_dir=args.backups_dir,
|
||||
).sql_file(args.db_name),
|
||||
empty=args.empty,
|
||||
check_version=not args.no_version_check,
|
||||
)
|
||||
return 0
|
||||
|
||||
if args.cmd == "cluster":
|
||||
restore_cluster_sql(
|
||||
container=args.container,
|
||||
user=args.db_user,
|
||||
password=args.db_password,
|
||||
sql_path=BackupPaths(
|
||||
args.volume_name,
|
||||
args.backup_hash,
|
||||
args.version,
|
||||
repo_name=args.repo_name,
|
||||
backups_dir=args.backups_dir,
|
||||
).cluster_file(args.instance),
|
||||
empty=args.empty,
|
||||
check_version=not args.no_version_check,
|
||||
)
|
||||
return 0
|
||||
|
||||
@@ -126,13 +160,14 @@ def main(argv: list[str] | None = None) -> int:
|
||||
backups_dir=args.backups_dir,
|
||||
).sql_file(args.db_name),
|
||||
empty=args.empty,
|
||||
check_version=not args.no_version_check,
|
||||
)
|
||||
return 0
|
||||
|
||||
parser.error("Unhandled command")
|
||||
return 2
|
||||
return 2 # noqa: TRY300 - the try wraps the whole dispatch on purpose
|
||||
|
||||
except Exception as e:
|
||||
except Exception as e: # noqa: BLE001 - CLI boundary: any failure becomes exit 1
|
||||
print(f"ERROR: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
252
src/baudolo/restore/db/cluster.py
Normal file
252
src/baudolo/restore/db/cluster.py
Normal file
@@ -0,0 +1,252 @@
|
||||
"""Replay a full PostgreSQL cluster dump produced by ``pg_dumpall``.
|
||||
|
||||
The backup side writes one when a databases.csv row asks for every database of
|
||||
an instance (``database = '*'``, see ``backup/db.py``). Until now nothing read
|
||||
it back, so that dump was stored and unrestorable - a format whose producer has
|
||||
no consumer.
|
||||
|
||||
A cluster stream differs from a single-database one in three ways that decide
|
||||
the implementation:
|
||||
|
||||
* it recreates roles and databases, so it must be replayed against the control
|
||||
database rather than into a target database;
|
||||
* ``CREATE DATABASE`` cannot run inside a transaction block, so unlike
|
||||
:mod:`baudolo.restore.db.postgres` the replay must not be wrapped in
|
||||
``--single-transaction``;
|
||||
* it is replayed as a superuser, so the superuser-only statements that the
|
||||
single-database path filters out are exactly the ones that have to survive.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from baudolo.restore.run import docker_exec
|
||||
|
||||
from .version import guard
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable, Iterator
|
||||
|
||||
CONTROL_DB = "postgres"
|
||||
_CLUSTER_PRECLEAN_SQL = Path(__file__).parent / "cluster_preclean.sql"
|
||||
_CREATE_ROLE = re.compile(rb'^CREATE ROLE "?([^";]+)"?;\s*$')
|
||||
_CREATE_DATABASE = re.compile(rb"^CREATE DATABASE\s+(.*)$")
|
||||
_CREATE_ROLE_LINE = re.compile(rb"^CREATE ROLE\s+(.*)$")
|
||||
_CONNECT = re.compile(rb"^\\connect\s+(.*)$")
|
||||
_NO_ROWS = "SELECT ''::text WHERE false"
|
||||
|
||||
|
||||
def _first_identifier(rest: str) -> str | None:
|
||||
"""The first SQL identifier in *rest*, quoted or bare.
|
||||
|
||||
A quoted identifier may hold spaces and doubled quotes, so it cannot be
|
||||
read with a character class that stops at whitespace - which is how a
|
||||
database called ``odd name`` used to leave the inventory as ``odd``.
|
||||
"""
|
||||
text = rest.strip()
|
||||
if not text:
|
||||
return None
|
||||
if text.startswith('"'):
|
||||
out = []
|
||||
index = 1
|
||||
while index < len(text):
|
||||
char = text[index]
|
||||
if char == '"':
|
||||
if index + 1 < len(text) and text[index + 1] == '"':
|
||||
out.append('"')
|
||||
index += 2
|
||||
continue
|
||||
return "".join(out)
|
||||
out.append(char)
|
||||
index += 1
|
||||
return None
|
||||
return re.split(r"[\s;(]", text, maxsplit=1)[0] or None
|
||||
|
||||
|
||||
def _connect_target(rest: str) -> str | None:
|
||||
"""The database a ``\\connect`` line switches to.
|
||||
|
||||
psql options precede the name (``\\connect -reuse-previous=on dbname=x``),
|
||||
and the name may arrive as a ``dbname=`` assignment rather than bare.
|
||||
"""
|
||||
for token in rest.strip().split():
|
||||
if token.startswith("-"):
|
||||
continue
|
||||
if token.startswith("dbname="):
|
||||
return _first_identifier(token[len("dbname=") :])
|
||||
return _first_identifier(rest.strip()[rest.strip().index(token) :])
|
||||
return None
|
||||
|
||||
|
||||
def dump_inventory(sql_path: str) -> tuple[list[str], list[str]]:
|
||||
"""The databases and roles a cluster dump recreates.
|
||||
|
||||
Args:
|
||||
sql_path: the ``pg_dumpall`` stream.
|
||||
|
||||
Returns:
|
||||
``(databases, roles)``, each in the order the dump names them. The
|
||||
pre-clean is scoped to these: everything else in the instance belongs
|
||||
to no backup this restore holds, and dropping it would destroy data
|
||||
the replay cannot bring back.
|
||||
"""
|
||||
databases: list[str] = []
|
||||
roles: list[str] = []
|
||||
with Path(sql_path).open("rb") as handle:
|
||||
for raw in handle:
|
||||
line = raw.decode("utf-8", "replace")
|
||||
for pattern, sink, read in (
|
||||
(_CREATE_DATABASE, databases, _first_identifier),
|
||||
(_CONNECT, databases, _connect_target),
|
||||
(_CREATE_ROLE_LINE, roles, _first_identifier),
|
||||
):
|
||||
found = pattern.match(raw)
|
||||
if not found:
|
||||
continue
|
||||
name = read(line[found.start(1) :])
|
||||
if name and name not in sink:
|
||||
sink.append(name)
|
||||
return databases, roles
|
||||
|
||||
|
||||
def preclean_sql() -> str:
|
||||
"""The catalog-wide pre-clean, safe only behind the instance check."""
|
||||
with _CLUSTER_PRECLEAN_SQL.open(encoding="utf-8") as preclean:
|
||||
return preclean.read()
|
||||
|
||||
|
||||
def instance_databases(container: str, user: str, docker_env: dict) -> list[str]:
|
||||
"""The instance's own databases, templates and control database aside."""
|
||||
listed = docker_exec(
|
||||
container,
|
||||
[
|
||||
"psql",
|
||||
"-U",
|
||||
user,
|
||||
"-d",
|
||||
CONTROL_DB,
|
||||
"-tAc",
|
||||
(
|
||||
"SELECT datname FROM pg_database "
|
||||
"WHERE NOT datistemplate AND datname <> current_database()"
|
||||
),
|
||||
],
|
||||
capture=True,
|
||||
docker_env=docker_env,
|
||||
).stdout
|
||||
text = listed.decode() if isinstance(listed, bytes) else listed
|
||||
return [name for name in text.split() if name]
|
||||
|
||||
|
||||
def assert_instance_matches_dump(
|
||||
container: str, user: str, sql_path: str, docker_env: dict
|
||||
) -> None:
|
||||
"""Refuse ``--empty`` on an instance holding anything the dump lacks.
|
||||
|
||||
The pre-clean is a catalog-wide sweep, so a foreign database would be
|
||||
destroyed with no way back. Scoping the sweep instead is not a fix: a
|
||||
surviving database that owns or grants to one of the dump's roles pins
|
||||
that role in pg_shdepend, and DROP ROLE then fails after the dump's own
|
||||
databases are already gone.
|
||||
|
||||
Raises:
|
||||
RuntimeError: the instance carries databases this dump cannot restore.
|
||||
"""
|
||||
dumped, _roles = dump_inventory(sql_path)
|
||||
present = instance_databases(container, user, docker_env)
|
||||
foreign = sorted(set(present) - set(dumped))
|
||||
if foreign:
|
||||
raise RuntimeError(
|
||||
f"{container} also holds {', '.join(foreign)}, which "
|
||||
f"{Path(sql_path).name} does not carry. --empty wipes the "
|
||||
"instance, so those would be destroyed with nothing to restore "
|
||||
"them from. Move them off this instance, or drop them yourself if "
|
||||
"they are disposable."
|
||||
)
|
||||
|
||||
|
||||
def _psql(user: str) -> list[str]:
|
||||
"""The replay client: no --single-transaction, CREATE DATABASE forbids it."""
|
||||
return ["psql", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", CONTROL_DB]
|
||||
|
||||
|
||||
def filter_own_role_creation(lines: Iterable[bytes], user: str) -> Iterator[bytes]:
|
||||
"""Drop the ``CREATE ROLE`` of the role holding this session.
|
||||
|
||||
A pg_dumpall stream recreates every role of the cluster, the bootstrap
|
||||
superuser included, and the pre-clean cannot drop the one it is connected
|
||||
as - so that single statement always collides. Its ``ALTER ROLE`` is kept:
|
||||
that is what re-applies the attributes and the password the dump captured.
|
||||
|
||||
Args:
|
||||
lines: dump lines including their trailing newlines.
|
||||
user: the connecting role.
|
||||
|
||||
Yields:
|
||||
Every line except that one CREATE.
|
||||
"""
|
||||
for line in lines:
|
||||
found = _CREATE_ROLE.match(line)
|
||||
if found and found.group(1).decode() == user:
|
||||
continue
|
||||
yield line
|
||||
|
||||
|
||||
def restore_cluster_sql(
|
||||
*,
|
||||
container: str,
|
||||
user: str,
|
||||
password: str,
|
||||
sql_path: str,
|
||||
empty: bool,
|
||||
check_version: bool = True,
|
||||
) -> None:
|
||||
"""Replay a pg_dumpall stream into a running instance.
|
||||
|
||||
Args:
|
||||
container: the running engine to replay into.
|
||||
user: a superuser of that instance; the dump creates roles and
|
||||
databases, which an application role may not do.
|
||||
password: its password, handed to psql through the container's env.
|
||||
sql_path: the ``<instance>.cluster.backup.sql`` of a generation.
|
||||
empty: drop the cluster's databases and roles first. Without it the
|
||||
replay stops at the first object that already exists, which is the
|
||||
honest outcome: recreating a cluster over a populated one is a
|
||||
decision, not a default.
|
||||
check_version: refuse a dump from a newer major version than the
|
||||
running engine before anything is dropped.
|
||||
"""
|
||||
if not Path(sql_path).is_file():
|
||||
raise FileNotFoundError(sql_path)
|
||||
|
||||
if check_version:
|
||||
guard(
|
||||
sql_path=sql_path,
|
||||
engine="postgres",
|
||||
container=container,
|
||||
user=user,
|
||||
password=password,
|
||||
)
|
||||
|
||||
docker_env = {"PGPASSWORD": password}
|
||||
|
||||
if empty:
|
||||
assert_instance_matches_dump(container, user, sql_path, docker_env)
|
||||
docker_exec(
|
||||
container,
|
||||
_psql(user),
|
||||
stdin=preclean_sql().encode(),
|
||||
docker_env=docker_env,
|
||||
)
|
||||
|
||||
with Path(sql_path).open("rb") as src, tempfile.TemporaryFile() as filtered:
|
||||
for line in filter_own_role_creation(src, user):
|
||||
filtered.write(line)
|
||||
filtered.seek(0)
|
||||
docker_exec(container, _psql(user), stdin=filtered, docker_env=docker_env)
|
||||
|
||||
print(f"PostgreSQL cluster restore complete from '{Path(sql_path).name}'.")
|
||||
33
src/baudolo/restore/db/cluster_preclean.sql
Normal file
33
src/baudolo/restore/db/cluster_preclean.sql
Normal file
@@ -0,0 +1,33 @@
|
||||
-- Pre-clean for `restore cluster --empty`. A pg_dumpall stream recreates roles
|
||||
-- and databases, so replaying it into a populated cluster dies on the first
|
||||
-- CREATE ROLE. Emitted as one DROP per row and run via \gexec so each executes
|
||||
-- as its own top-level statement: DROP DATABASE cannot run inside a
|
||||
-- transaction block, which rules out a single DO block.
|
||||
-- The phase column pins the order: databases must be gone before their owners
|
||||
-- can be dropped, and DROP OWNED BY releases what a role still holds in the
|
||||
-- control database. Template databases, the control database itself, the pg_*
|
||||
-- system roles and the connecting role are kept - the dump does not recreate
|
||||
-- them and dropping them would end the session.
|
||||
-- The sweep stays catalog-wide on purpose: a scoped one leaves databases that
|
||||
-- pin a dumped role in pg_shdepend, and phase 3 then fails after phase 1 has
|
||||
-- already dropped. assert_instance_matches_dump refuses before this runs.
|
||||
SELECT statement
|
||||
FROM (
|
||||
SELECT 1 AS phase,
|
||||
format('DROP DATABASE IF EXISTS %I', datname) AS statement
|
||||
FROM pg_database
|
||||
WHERE NOT datistemplate
|
||||
AND datname <> current_database()
|
||||
UNION ALL
|
||||
SELECT 2, format('DROP OWNED BY %I', rolname)
|
||||
FROM pg_roles
|
||||
WHERE NOT starts_with(rolname, 'pg_')
|
||||
AND rolname <> current_user
|
||||
UNION ALL
|
||||
SELECT 3, format('DROP ROLE IF EXISTS %I', rolname)
|
||||
FROM pg_roles
|
||||
WHERE NOT starts_with(rolname, 'pg_')
|
||||
AND rolname <> current_user
|
||||
) drops
|
||||
ORDER BY phase
|
||||
\gexec
|
||||
@@ -3,8 +3,12 @@
|
||||
-- would run every DROP in one transaction and exhaust max_locks_per_transaction on
|
||||
-- large schemas (e.g. gitlab). Also drops user-owned non-public schemas so a dump
|
||||
-- that CREATE SCHEMAs (e.g. discourse's discourse_functions) does not fail on an
|
||||
-- already-existing schema. Extension members (pg_trgm's set_limit) are
|
||||
-- superuser-owned; IF EXISTS absorbs the CASCADE fallout.
|
||||
-- already-existing schema. Objects belonging to an extension are skipped: postgres
|
||||
-- refuses to drop them one by one ("cannot drop function vector_in(...) because
|
||||
-- extension vector requires it"), and the dump's CREATE EXTENSION IF NOT EXISTS
|
||||
-- finds the surviving extension either way. Owning them is not enough to make them
|
||||
-- droppable - an extension a role installed itself is owned by that role, so the
|
||||
-- owner filter alone lets pgvector's members through.
|
||||
SELECT format('DROP %s IF EXISTS public.%s CASCADE', obj.type, obj.name)
|
||||
FROM (
|
||||
SELECT format('%I', c.relname) AS name,
|
||||
@@ -13,7 +17,8 @@ SELECT format('DROP %s IF EXISTS public.%s CASCADE', obj.type, obj.name)
|
||||
WHEN 'm' THEN 'MATERIALIZED VIEW'
|
||||
WHEN 'f' THEN 'FOREIGN TABLE'
|
||||
ELSE 'TABLE'
|
||||
END AS type
|
||||
END AS type,
|
||||
c.oid AS objid, 'pg_class'::regclass AS classid
|
||||
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p', 'v', 'm', 'f')
|
||||
AND pg_get_userbyid(c.relowner) = current_user
|
||||
@@ -21,17 +26,20 @@ SELECT format('DROP %s IF EXISTS public.%s CASCADE', obj.type, obj.name)
|
||||
-- Overloaded functions share a proname; DROP needs the identity
|
||||
-- signature or psql aborts with "function name is not unique".
|
||||
SELECT format('%I(%s)', p.proname, pg_get_function_identity_arguments(p.oid)) AS name,
|
||||
CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END AS type
|
||||
CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END AS type,
|
||||
p.oid AS objid, 'pg_proc'::regclass AS classid
|
||||
FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace
|
||||
WHERE n.nspname = 'public' AND p.prokind IN ('f', 'p', 'w')
|
||||
AND pg_get_userbyid(p.proowner) = current_user
|
||||
UNION ALL
|
||||
SELECT format('%I', c.relname) AS name, 'SEQUENCE' AS type
|
||||
SELECT format('%I', c.relname) AS name, 'SEQUENCE' AS type,
|
||||
c.oid AS objid, 'pg_class'::regclass AS classid
|
||||
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = 'public' AND c.relkind = 'S'
|
||||
AND pg_get_userbyid(c.relowner) = current_user
|
||||
UNION ALL
|
||||
SELECT format('%I', t.typname) AS name, 'TYPE' AS type
|
||||
SELECT format('%I', t.typname) AS name, 'TYPE' AS type,
|
||||
t.oid AS objid, 'pg_type'::regclass AS classid
|
||||
FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace
|
||||
WHERE n.nspname = 'public'
|
||||
AND pg_get_userbyid(t.typowner) = current_user
|
||||
@@ -40,25 +48,36 @@ SELECT format('DROP %s IF EXISTS public.%s CASCADE', obj.type, obj.name)
|
||||
SELECT 1 FROM pg_class c2
|
||||
WHERE c2.oid = t.typrelid AND c2.relkind = 'c')))
|
||||
UNION ALL
|
||||
SELECT format('%I', col.collname) AS name, 'COLLATION' AS type
|
||||
SELECT format('%I', col.collname) AS name, 'COLLATION' AS type,
|
||||
col.oid AS objid, 'pg_collation'::regclass AS classid
|
||||
FROM pg_collation col JOIN pg_namespace n ON n.oid = col.collnamespace
|
||||
WHERE n.nspname = 'public'
|
||||
AND pg_get_userbyid(col.collowner) = current_user
|
||||
UNION ALL
|
||||
SELECT format('%I', ts.cfgname) AS name, 'TEXT SEARCH CONFIGURATION' AS type
|
||||
SELECT format('%I', ts.cfgname) AS name, 'TEXT SEARCH CONFIGURATION' AS type,
|
||||
ts.oid AS objid, 'pg_ts_config'::regclass AS classid
|
||||
FROM pg_ts_config ts JOIN pg_namespace n ON n.oid = ts.cfgnamespace
|
||||
WHERE n.nspname = 'public'
|
||||
AND pg_get_userbyid(ts.cfgowner) = current_user
|
||||
UNION ALL
|
||||
SELECT format('%I', d.dictname) AS name, 'TEXT SEARCH DICTIONARY' AS type
|
||||
SELECT format('%I', d.dictname) AS name, 'TEXT SEARCH DICTIONARY' AS type,
|
||||
d.oid AS objid, 'pg_ts_dict'::regclass AS classid
|
||||
FROM pg_ts_dict d JOIN pg_namespace n ON n.oid = d.dictnamespace
|
||||
WHERE n.nspname = 'public'
|
||||
AND pg_get_userbyid(d.dictowner) = current_user
|
||||
) obj
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM pg_depend dep
|
||||
WHERE dep.classid = obj.classid AND dep.objid = obj.objid
|
||||
AND dep.deptype = 'e')
|
||||
UNION ALL
|
||||
SELECT format('DROP SCHEMA IF EXISTS %I CASCADE', n.nspname)
|
||||
FROM pg_namespace n
|
||||
WHERE NOT starts_with(n.nspname, 'pg_')
|
||||
AND n.nspname NOT IN ('public', 'information_schema')
|
||||
AND pg_get_userbyid(n.nspowner) = current_user
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM pg_depend dep
|
||||
WHERE dep.classid = 'pg_namespace'::regclass AND dep.objid = n.oid
|
||||
AND dep.deptype = 'e')
|
||||
\gexec
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from ..run import docker_exec, docker_exec_sh
|
||||
from baudolo.restore.run import docker_exec, docker_exec_sh
|
||||
|
||||
from .version import guard
|
||||
|
||||
_NO_CLIENT = "ERROR: neither 'mariadb' nor 'mysql' found in container."
|
||||
|
||||
|
||||
def _pick_client(container: str) -> str:
|
||||
@@ -19,14 +23,13 @@ exit 42
|
||||
"""
|
||||
try:
|
||||
out = docker_exec_sh(container, script, capture=True).stdout.decode().strip()
|
||||
if not out:
|
||||
raise RuntimeError("empty client detection output")
|
||||
return out
|
||||
except Exception as e:
|
||||
print(
|
||||
"ERROR: neither 'mariadb' nor 'mysql' found in container.", file=sys.stderr
|
||||
)
|
||||
raise e
|
||||
except Exception:
|
||||
print(_NO_CLIENT, file=sys.stderr)
|
||||
raise
|
||||
if not out:
|
||||
print(_NO_CLIENT, file=sys.stderr)
|
||||
raise RuntimeError("empty client detection output")
|
||||
return out
|
||||
|
||||
|
||||
def restore_mariadb_sql(
|
||||
@@ -37,16 +40,25 @@ def restore_mariadb_sql(
|
||||
password: str,
|
||||
sql_path: str,
|
||||
empty: bool,
|
||||
check_version: bool = True,
|
||||
) -> None:
|
||||
client = _pick_client(container)
|
||||
|
||||
if not os.path.isfile(sql_path):
|
||||
if not Path(sql_path).is_file():
|
||||
raise FileNotFoundError(sql_path)
|
||||
|
||||
if check_version:
|
||||
guard(
|
||||
sql_path=sql_path,
|
||||
engine="mariadb",
|
||||
container=container,
|
||||
user=user,
|
||||
password=password,
|
||||
client=client,
|
||||
)
|
||||
|
||||
if empty:
|
||||
# IMPORTANT:
|
||||
# Do NOT hardcode 'mysql' here. Use the detected client.
|
||||
# MariaDB 11 images may not contain the mysql binary at all.
|
||||
# Do not hardcode 'mysql': MariaDB 11 images may not ship that binary.
|
||||
result = docker_exec(
|
||||
container,
|
||||
[
|
||||
@@ -56,7 +68,7 @@ def restore_mariadb_sql(
|
||||
f"--password={password}",
|
||||
"-N",
|
||||
"-e",
|
||||
f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{db_name}';",
|
||||
f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{db_name}';", # noqa: S608 - validate_database() constrains the name to ^[a-zA-Z0-9_][a-zA-Z0-9_-]*$
|
||||
],
|
||||
capture=True,
|
||||
)
|
||||
@@ -84,7 +96,7 @@ def restore_mariadb_sql(
|
||||
],
|
||||
)
|
||||
|
||||
with open(sql_path, "rb") as f:
|
||||
with Path(sql_path).open("rb") as f:
|
||||
docker_exec(
|
||||
container, [client, "-u", user, f"--password={password}", db_name], stdin=f
|
||||
)
|
||||
|
||||
@@ -1,13 +1,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from collections.abc import Iterable, Iterator
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ..run import docker_exec
|
||||
from baudolo.restore.run import docker_exec
|
||||
|
||||
from .version import guard
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable, Iterator
|
||||
|
||||
_SUPERUSER_ONLY_PREFIXES = (b"COMMENT ON EXTENSION", b"ALTER DEFAULT PRIVILEGES")
|
||||
_EMPTY_PRECLEAN_SQL = os.path.join(os.path.dirname(__file__), "empty_preclean.sql")
|
||||
_EMPTY_PRECLEAN_SQL = Path(__file__).parent / "empty_preclean.sql"
|
||||
|
||||
|
||||
def filter_superuser_only_lines(lines: Iterable[bytes]) -> Iterator[bytes]:
|
||||
@@ -46,15 +51,24 @@ def restore_postgres_sql(
|
||||
password: str,
|
||||
sql_path: str,
|
||||
empty: bool,
|
||||
check_version: bool = True,
|
||||
) -> None:
|
||||
if not os.path.isfile(sql_path):
|
||||
if not Path(sql_path).is_file():
|
||||
raise FileNotFoundError(sql_path)
|
||||
|
||||
# Make password available INSIDE the container for psql.
|
||||
if check_version:
|
||||
guard(
|
||||
sql_path=sql_path,
|
||||
engine="postgres",
|
||||
container=container,
|
||||
user=user,
|
||||
password=password,
|
||||
)
|
||||
|
||||
docker_env = {"PGPASSWORD": password}
|
||||
|
||||
if empty:
|
||||
with open(_EMPTY_PRECLEAN_SQL, encoding="utf-8") as preclean:
|
||||
with _EMPTY_PRECLEAN_SQL.open(encoding="utf-8") as preclean:
|
||||
drop_sql = preclean.read()
|
||||
docker_exec(
|
||||
container,
|
||||
@@ -66,7 +80,7 @@ def restore_postgres_sql(
|
||||
# Filter into a spooled temp file instead of building the whole dump in
|
||||
# memory: production dumps reach many GB and the previous read/splitlines/
|
||||
# join needed roughly three times the dump size in RSS.
|
||||
with open(sql_path, "rb") as src, tempfile.TemporaryFile() as filtered:
|
||||
with Path(sql_path).open("rb") as src, tempfile.TemporaryFile() as filtered:
|
||||
for line in filter_superuser_only_lines(src):
|
||||
filtered.write(line)
|
||||
filtered.seek(0)
|
||||
|
||||
147
src/baudolo/restore/db/version.py
Normal file
147
src/baudolo/restore/db/version.py
Normal file
@@ -0,0 +1,147 @@
|
||||
"""Refuse a dump the target engine is too old to read.
|
||||
|
||||
A restore with ``--empty`` destroys before it replays: the pre-clean drops the
|
||||
schema in one session and the dump goes in the next, with no rollback across
|
||||
the two. A dump the engine cannot parse therefore does not fail harmlessly -
|
||||
it leaves an emptied database behind. Comparing the two versions first turns
|
||||
that into a refusal.
|
||||
|
||||
Both engines state their origin in the dump's own header, and they do not
|
||||
state it the same way. Postgres writes ``-- Dumped from database version``
|
||||
around line seven. MariaDB opens line two with ``-- MariaDB dump 10.19-11.8.8``,
|
||||
where the first number is mariadb-dump's own version, and names the server only
|
||||
further down on the tab-separated ``-- Server version`` line. Matching the first
|
||||
number in the header would read the tool on one engine and the server on the
|
||||
other, so each engine gets its own pattern.
|
||||
|
||||
A ``pg_dumpall`` cluster dump has no version line of its own: its header opens
|
||||
with the cluster banner and the roles section, and the first
|
||||
``-- Dumped from database version`` belongs to the first database's embedded
|
||||
``pg_dump`` output, arbitrarily far down. Hence the scan runs to
|
||||
``SCAN_LINES`` rather than to a header-sized handful.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from baudolo.restore.run import docker_exec, stdout_of
|
||||
|
||||
SCAN_LINES = 2000
|
||||
DUMP_VERSION = {
|
||||
"postgres": re.compile(r"^-- Dumped from database version (\S+)"),
|
||||
"mariadb": re.compile(r"^-- Server version\s+(\S+)"),
|
||||
}
|
||||
|
||||
|
||||
class VersionMismatchError(Exception):
|
||||
"""The dump cannot be replayed into this engine."""
|
||||
|
||||
|
||||
def major_of(version: str) -> int:
|
||||
"""The major number of an engine version string.
|
||||
|
||||
Args:
|
||||
version: as the engine spells it, e.g. ``17.11`` or
|
||||
``11.8.8-MariaDB-ubu2404``.
|
||||
|
||||
Raises:
|
||||
VersionMismatchError: the string does not start with a number.
|
||||
"""
|
||||
leading = re.match(r"(\d+)", version)
|
||||
if not leading:
|
||||
raise VersionMismatchError(f"cannot read a major version from '{version}'")
|
||||
return int(leading.group(1))
|
||||
|
||||
|
||||
def dump_version(sql_path: str, engine: str) -> str:
|
||||
"""Read the engine version a dump was taken from, out of its own header.
|
||||
|
||||
Args:
|
||||
sql_path: the dump to read.
|
||||
engine: ``postgres`` or ``mariadb``.
|
||||
|
||||
Returns:
|
||||
The version string as the dump spells it.
|
||||
|
||||
Raises:
|
||||
VersionMismatchError: no version line within the first ``SCAN_LINES``.
|
||||
"""
|
||||
pattern = DUMP_VERSION[engine]
|
||||
with Path(sql_path).open(encoding="utf-8", errors="replace") as handle:
|
||||
for _ in range(SCAN_LINES):
|
||||
line = handle.readline()
|
||||
if not line:
|
||||
break
|
||||
found = pattern.search(line)
|
||||
if found:
|
||||
return found.group(1)
|
||||
raise VersionMismatchError(
|
||||
f"{sql_path} carries no {engine} version header in its first {SCAN_LINES} lines"
|
||||
)
|
||||
|
||||
|
||||
def server_version(
|
||||
container: str, engine: str, user: str, password: str, client: str = ""
|
||||
) -> str:
|
||||
"""Ask the running engine which version it is."""
|
||||
if engine == "postgres":
|
||||
return stdout_of(
|
||||
docker_exec(
|
||||
container,
|
||||
["psql", "-U", user, "-tAc", "SHOW server_version"],
|
||||
capture=True,
|
||||
docker_env={"PGPASSWORD": password},
|
||||
)
|
||||
)
|
||||
return stdout_of(
|
||||
docker_exec(
|
||||
container,
|
||||
[
|
||||
client or "mariadb",
|
||||
"-u",
|
||||
user,
|
||||
f"--password={password}",
|
||||
"-N",
|
||||
"-B",
|
||||
"-e",
|
||||
"SELECT VERSION()",
|
||||
],
|
||||
capture=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def assert_replayable(sql_path: str, engine: str, dumped: str, serving: str) -> None:
|
||||
"""Refuse a dump from a newer major version than the target engine.
|
||||
|
||||
Restoring forward across a major version is the upgrade path and stays
|
||||
allowed; backward is refused, because a newer dump uses syntax an older
|
||||
server rejects and the pre-clean would already have dropped the schema.
|
||||
|
||||
Raises:
|
||||
VersionMismatchError: the dump is newer than the engine.
|
||||
"""
|
||||
if major_of(dumped) > major_of(serving):
|
||||
raise VersionMismatchError(
|
||||
f"{sql_path} came from {engine} {dumped} but {serving} is running; "
|
||||
"a newer dump does not replay into an older engine, and --empty "
|
||||
"would drop the schema before finding out"
|
||||
)
|
||||
|
||||
|
||||
def guard(
|
||||
*,
|
||||
sql_path: str,
|
||||
engine: str,
|
||||
container: str,
|
||||
user: str,
|
||||
password: str,
|
||||
client: str = "",
|
||||
) -> None:
|
||||
"""Compare the dump's origin against the running engine before replaying."""
|
||||
dumped = dump_version(sql_path, engine)
|
||||
serving = server_version(container, engine, user, password, client)
|
||||
assert_replayable(sql_path, engine, dumped, serving)
|
||||
print(f"OK: dump is from {engine} {dumped}, {serving} is serving.")
|
||||
@@ -1,13 +1,28 @@
|
||||
"""Restore a volume's file tree by writing into its mountpoint.
|
||||
|
||||
That shortcut only holds for a plain local volume, where the mountpoint *is*
|
||||
the storage. A volume with driver options - NFS, a bind device, tmpfs - keeps
|
||||
the same ``/var/lib/docker/volumes/<name>/_data`` path, but docker mounts the
|
||||
real backing store over it on demand and unmounts it again when the last
|
||||
consumer stops. Writing there while nothing has it mounted lands in the empty
|
||||
directory underneath, is hidden by the next mount, and rsync reports success.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from .run import docker_volume_exists, run
|
||||
from .run import docker_volume_exists, run, stdout_of
|
||||
|
||||
INSPECT_FORMAT = (
|
||||
"{{ .Mountpoint }}|{{ .Driver }}|{{ if .Options }}opts{{ else }}plain{{ end }}"
|
||||
)
|
||||
|
||||
|
||||
def restore_volume_files(volume_name: str, backup_files_dir: str) -> int:
|
||||
if not os.path.isdir(backup_files_dir):
|
||||
if not Path(backup_files_dir).is_dir():
|
||||
print(f"ERROR: backup files dir not found: {backup_files_dir}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
@@ -18,11 +33,11 @@ def restore_volume_files(volume_name: str, backup_files_dir: str) -> int:
|
||||
print(f"Volume {volume_name} already exists.")
|
||||
|
||||
cp = run(
|
||||
["docker", "volume", "inspect", "--format", "{{ .Mountpoint }}", volume_name],
|
||||
["docker", "volume", "inspect", "--format", INSPECT_FORMAT, volume_name],
|
||||
capture=True,
|
||||
)
|
||||
raw = cp.stdout or b""
|
||||
mountpoint = (raw.decode() if isinstance(raw, bytes) else raw).strip()
|
||||
fields = stdout_of(cp).split("|")
|
||||
mountpoint = fields[0] if fields else ""
|
||||
if not mountpoint:
|
||||
print(
|
||||
f"ERROR: could not resolve mountpoint for volume {volume_name}",
|
||||
@@ -30,8 +45,20 @@ def restore_volume_files(volume_name: str, backup_files_dir: str) -> int:
|
||||
)
|
||||
return 2
|
||||
|
||||
src = os.path.join(backup_files_dir, "")
|
||||
dest = os.path.join(mountpoint, "")
|
||||
driver, options = ([*fields, "local", "plain"])[1:3]
|
||||
if (driver != "local" or options == "opts") and not os.path.ismount(mountpoint):
|
||||
print(
|
||||
f"ERROR: volume {volume_name} has a backing store of its own "
|
||||
f"(driver {driver}) but nothing has it mounted; writing to "
|
||||
f"{mountpoint} now would land under the mount and be lost. "
|
||||
"Start a container that mounts the volume, then restore again.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
# rsync reads "dir/" as its contents and "dir" as the directory itself.
|
||||
src = f"{Path(backup_files_dir)}{os.sep}"
|
||||
dest = f"{Path(mountpoint)}{os.sep}"
|
||||
run(["rsync", "-avv", "--delete", src, dest])
|
||||
print("File restore complete.")
|
||||
return 0
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from baudolo.generation import CLUSTER_SUFFIX, DUMP_SUFFIX, FILES_DIR, SQL_DIR
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -14,16 +16,20 @@ class BackupPaths:
|
||||
|
||||
def root(self) -> str:
|
||||
# Always build an absolute path under backups_dir
|
||||
return os.path.join(
|
||||
self.backups_dir,
|
||||
self.backup_hash,
|
||||
self.repo_name,
|
||||
self.version,
|
||||
self.volume_name,
|
||||
return str(
|
||||
Path(self.backups_dir)
|
||||
/ self.backup_hash
|
||||
/ self.repo_name
|
||||
/ self.version
|
||||
/ self.volume_name
|
||||
)
|
||||
|
||||
def files_dir(self) -> str:
|
||||
return os.path.join(self.root(), "files")
|
||||
return str(Path(self.root()) / FILES_DIR)
|
||||
|
||||
def sql_file(self, db_name: str) -> str:
|
||||
return os.path.join(self.root(), "sql", f"{db_name}.backup.sql")
|
||||
return str(Path(self.root()) / SQL_DIR / f"{db_name}{DUMP_SUFFIX}")
|
||||
|
||||
def cluster_file(self, instance: str) -> str:
|
||||
"""The pg_dumpall stream a `database = '*'` row produces."""
|
||||
return str(Path(self.root()) / SQL_DIR / f"{instance}{CLUSTER_SUFFIX}")
|
||||
|
||||
@@ -2,7 +2,6 @@ from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def run(
|
||||
@@ -10,7 +9,7 @@ def run(
|
||||
*,
|
||||
stdin=None,
|
||||
capture: bool = False,
|
||||
env: Optional[dict] = None,
|
||||
env: dict | None = None,
|
||||
) -> subprocess.CompletedProcess:
|
||||
try:
|
||||
kwargs: dict = {
|
||||
@@ -26,32 +25,35 @@ def run(
|
||||
else:
|
||||
kwargs["stdin"] = stdin
|
||||
|
||||
return subprocess.run(cmd, **kwargs)
|
||||
return subprocess.run(cmd, **kwargs) # noqa: PLW1510 - check lives in kwargs
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
msg = f"ERROR: command failed ({e.returncode}): {' '.join(cmd)}"
|
||||
print(msg, file=sys.stderr)
|
||||
if e.stdout:
|
||||
for stream in (e.stdout, e.stderr):
|
||||
if not stream:
|
||||
continue
|
||||
try:
|
||||
print(e.stdout.decode(), file=sys.stderr)
|
||||
except Exception:
|
||||
print(e.stdout, file=sys.stderr)
|
||||
if e.stderr:
|
||||
try:
|
||||
print(e.stderr.decode(), file=sys.stderr)
|
||||
except Exception:
|
||||
print(e.stderr, file=sys.stderr)
|
||||
print(stream.decode(), file=sys.stderr)
|
||||
except (UnicodeDecodeError, AttributeError):
|
||||
print(stream, file=sys.stderr)
|
||||
raise
|
||||
|
||||
|
||||
def stdout_of(completed: subprocess.CompletedProcess) -> str:
|
||||
"""The captured stdout as stripped text, whether it came back bytes or str."""
|
||||
raw = completed.stdout or b""
|
||||
return (raw.decode() if isinstance(raw, bytes) else raw).strip()
|
||||
|
||||
|
||||
def docker_exec(
|
||||
container: str,
|
||||
argv: list[str],
|
||||
*,
|
||||
stdin=None,
|
||||
capture: bool = False,
|
||||
env: Optional[dict] = None,
|
||||
docker_env: Optional[dict[str, str]] = None,
|
||||
env: dict | None = None,
|
||||
docker_env: dict[str, str] | None = None,
|
||||
) -> subprocess.CompletedProcess:
|
||||
cmd: list[str] = ["docker", "exec", "-i"]
|
||||
if docker_env:
|
||||
@@ -67,8 +69,8 @@ def docker_exec_sh(
|
||||
*,
|
||||
stdin=None,
|
||||
capture: bool = False,
|
||||
env: Optional[dict] = None,
|
||||
docker_env: Optional[dict[str, str]] = None,
|
||||
env: dict | None = None,
|
||||
docker_env: dict[str, str] | None = None,
|
||||
) -> subprocess.CompletedProcess:
|
||||
return docker_exec(
|
||||
container,
|
||||
@@ -85,5 +87,6 @@ def docker_volume_exists(volume: str) -> bool:
|
||||
["docker", "volume", "inspect", volume],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=False,
|
||||
)
|
||||
return p.returncode == 0
|
||||
|
||||
@@ -1,46 +1,23 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
from typing import Optional
|
||||
from pandas.errors import EmptyDataError
|
||||
|
||||
DB_NAME_RE = re.compile(r"^[a-zA-Z0-9_][a-zA-Z0-9_-]*$")
|
||||
|
||||
|
||||
def _validate_database_value(value: Optional[str], *, instance: str) -> str:
|
||||
v = (value or "").strip()
|
||||
if v == "":
|
||||
raise ValueError(
|
||||
f"Invalid databases.csv entry for instance '{instance}': "
|
||||
"column 'database' must be '*' or a concrete database name (not empty)."
|
||||
)
|
||||
if v == "*":
|
||||
return "*"
|
||||
if v.lower() == "nan":
|
||||
raise ValueError(
|
||||
f"Invalid databases.csv entry for instance '{instance}': database must not be 'nan'."
|
||||
)
|
||||
if not DB_NAME_RE.match(v):
|
||||
raise ValueError(
|
||||
f"Invalid databases.csv entry for instance '{instance}': "
|
||||
f"invalid database name '{v}'. Allowed: letters, numbers, '_' and '-'."
|
||||
)
|
||||
return v
|
||||
from baudolo.databases import COLUMNS, DELIMITER, validate_database
|
||||
|
||||
|
||||
def _empty_df() -> pd.DataFrame:
|
||||
return pd.DataFrame(columns=["instance", "database", "username", "password"])
|
||||
return pd.DataFrame(columns=list(COLUMNS))
|
||||
|
||||
|
||||
def check_and_add_entry(
|
||||
file_path: str,
|
||||
instance: str,
|
||||
database: Optional[str],
|
||||
database: str | None,
|
||||
username: str,
|
||||
password: str,
|
||||
) -> None:
|
||||
@@ -51,13 +28,13 @@ def check_and_add_entry(
|
||||
- database MUST be set
|
||||
- database MUST be '*' or a valid database name
|
||||
"""
|
||||
database = _validate_database_value(database, instance=instance)
|
||||
database = validate_database(database, instance=instance)
|
||||
|
||||
if os.path.exists(file_path):
|
||||
if Path(file_path).exists():
|
||||
try:
|
||||
df = pd.read_csv(
|
||||
file_path,
|
||||
sep=";",
|
||||
sep=DELIMITER,
|
||||
dtype=str,
|
||||
keep_default_na=False,
|
||||
)
|
||||
@@ -78,11 +55,11 @@ def check_and_add_entry(
|
||||
print("Adding new entry.")
|
||||
new_entry = pd.DataFrame(
|
||||
[[instance, database, username, password]],
|
||||
columns=["instance", "database", "username", "password"],
|
||||
columns=list(COLUMNS),
|
||||
)
|
||||
df = pd.concat([df, new_entry], ignore_index=True)
|
||||
|
||||
df.to_csv(file_path, sep=";", index=False)
|
||||
df.to_csv(file_path, sep=DELIMITER, index=False)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
@@ -108,7 +85,7 @@ def main() -> None:
|
||||
username=args.username,
|
||||
password=args.password,
|
||||
)
|
||||
except Exception as exc:
|
||||
except Exception as exc: # noqa: BLE001 - CLI boundary: any failure becomes exit 1
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
90
tests/e2e/faithful_driver.py
Normal file
90
tests/e2e/faithful_driver.py
Normal file
@@ -0,0 +1,90 @@
|
||||
"""Show what a real snapshot holds for a volume that has its own storage.
|
||||
|
||||
Runs inside the privileged container that built the btrfs subject. Prints one
|
||||
PASS/FAIL line per assertion and exits non-zero on the first failure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, "/src")
|
||||
|
||||
from baudolo.backup.snapshot import (
|
||||
SnapshotError,
|
||||
snapshot_source,
|
||||
unsnapshotted,
|
||||
volume_snapshot,
|
||||
)
|
||||
from baudolo.backup.volume import Backing
|
||||
|
||||
SUBJECT = sys.argv[1]
|
||||
|
||||
|
||||
def shell(command: list[str]) -> list[str]:
|
||||
proc = subprocess.run(command, capture_output=True, text=True, check=False)
|
||||
if proc.returncode != 0:
|
||||
raise SnapshotError(
|
||||
f"{' '.join(command)} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||
)
|
||||
return proc.stdout.splitlines()
|
||||
|
||||
|
||||
def check(label: str, condition: bool) -> None:
|
||||
print(f"{'PASS' if condition else 'FAIL'} {label}", flush=True)
|
||||
if not condition:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def volume(name: str, payload: str) -> Path:
|
||||
path = Path(SUBJECT) / "volumes" / name / "_data"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
(path / "state").write_text(payload)
|
||||
return path
|
||||
|
||||
|
||||
plain = volume("plain", "plain-payload")
|
||||
own = Path(SUBJECT) / "volumes" / "own" / "_data"
|
||||
own.mkdir(parents=True, exist_ok=True)
|
||||
shell(["mount", "-t", "tmpfs", "tmpfs", own])
|
||||
(own / "state").write_text("own-payload")
|
||||
|
||||
check("a plain volume is captured", unsnapshotted(Backing(str(plain)), SUBJECT) is None)
|
||||
check(
|
||||
"a volume on a mount of its own is not",
|
||||
unsnapshotted(Backing(str(own)), SUBJECT) is not None,
|
||||
)
|
||||
check(
|
||||
"a declared backing store is not, mounted or not",
|
||||
unsnapshotted(Backing(str(plain), options={"type": "nfs"}), SUBJECT) is not None,
|
||||
)
|
||||
check(
|
||||
"a foreign driver is not",
|
||||
unsnapshotted(Backing(str(plain), driver="rexray"), SUBJECT) is not None,
|
||||
)
|
||||
|
||||
with volume_snapshot("btrfs", SUBJECT, "e2e", run=shell) as resolve:
|
||||
frozen_plain = Path(resolve(str(plain)))
|
||||
frozen_own = Path(resolve(str(own)))
|
||||
|
||||
check(
|
||||
"the snapshot carries the plain volume",
|
||||
(frozen_plain / "state").read_text() == "plain-payload",
|
||||
)
|
||||
check(
|
||||
"the snapshot shows the other volume as an empty directory",
|
||||
frozen_own.is_dir() and not any(frozen_own.iterdir()),
|
||||
)
|
||||
|
||||
source, reason = snapshot_source(resolve, Backing(str(plain)), SUBJECT)
|
||||
check(
|
||||
"the plain volume is read from the snapshot",
|
||||
source is not None and source.rstrip("/") == str(frozen_plain),
|
||||
)
|
||||
|
||||
source, reason = snapshot_source(resolve, Backing(str(own)), SUBJECT)
|
||||
check(f"the other volume degrades to live: {reason[:60]}", source is None)
|
||||
|
||||
print("ALL OK", flush=True)
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Shared e2e helpers, re-exported so tests import one name."""
|
||||
|
||||
from .fixtures import * # noqa: F401,F403
|
||||
from .process import * # noqa: F401,F403
|
||||
from .fixtures import *
|
||||
from .process import *
|
||||
|
||||
@@ -24,7 +24,7 @@ def backup_run(
|
||||
database_containers: list[str],
|
||||
images_no_stop_required: list[str],
|
||||
images_no_backup_required: list[str] | None = None,
|
||||
dump_only_sql: bool = False,
|
||||
only_sql: bool = False,
|
||||
) -> None:
|
||||
cmd = [
|
||||
"baudolo",
|
||||
@@ -45,8 +45,8 @@ def backup_run(
|
||||
]
|
||||
if images_no_backup_required:
|
||||
cmd += ["--images-no-backup-required", *images_no_backup_required]
|
||||
if dump_only_sql:
|
||||
cmd += ["--dump-only-sql"]
|
||||
if only_sql:
|
||||
cmd += ["--only-sql"]
|
||||
|
||||
try:
|
||||
run(cmd, capture=True, check=True)
|
||||
@@ -95,10 +95,9 @@ def write_databases_csv(path: str, rows: list[tuple[str, str, str, str]]) -> Non
|
||||
database may be '' (empty) to trigger pg_dumpall behavior if you want, but here we use db name.
|
||||
"""
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
with Path(path).open("w", encoding="utf-8") as f:
|
||||
f.write("instance;database;username;password\n")
|
||||
for inst, db, user, pw in rows:
|
||||
f.write(f"{inst};{db};{user};{pw}\n")
|
||||
f.writelines(f"{inst};{db};{user};{pw}\n" for inst, db, user, pw in rows)
|
||||
|
||||
|
||||
def cleanup_docker(*, containers: list[str], volumes: list[str]) -> None:
|
||||
|
||||
@@ -12,8 +12,8 @@ import sys
|
||||
|
||||
sys.path.insert(0, "/src")
|
||||
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot # noqa: E402
|
||||
from baudolo.backup.volume import backup_volume # noqa: E402
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
||||
from baudolo.backup.volume import backup_volume
|
||||
|
||||
SUBJECT = "/subject/docker"
|
||||
VOLUME = "mariadb_data"
|
||||
@@ -22,11 +22,11 @@ VERSIONS = "/backups"
|
||||
GENERATION = f"{VERSIONS}/20260731"
|
||||
|
||||
|
||||
def shell(command: str) -> list[str]:
|
||||
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
||||
def shell(command: list[str]) -> list[str]:
|
||||
proc = subprocess.run(command, capture_output=True, text=True, check=False)
|
||||
if proc.returncode != 0:
|
||||
raise SnapshotError(
|
||||
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||
f"{' '.join(command)} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||
)
|
||||
return proc.stdout.splitlines()
|
||||
|
||||
|
||||
@@ -12,18 +12,18 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, "/src")
|
||||
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot # noqa: E402
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
||||
|
||||
KIND = sys.argv[1]
|
||||
SUBJECT = sys.argv[2]
|
||||
EXPECT = sys.argv[3]
|
||||
|
||||
|
||||
def shell(command: str) -> list[str]:
|
||||
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
||||
def shell(command: list[str]) -> list[str]:
|
||||
proc = subprocess.run(command, capture_output=True, text=True, check=False)
|
||||
if proc.returncode != 0:
|
||||
raise SnapshotError(
|
||||
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||
f"{' '.join(command)} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||
)
|
||||
return proc.stdout.splitlines()
|
||||
|
||||
|
||||
184
tests/e2e/test_e2e_app_container_ships_client_tools.py
Normal file
184
tests/e2e/test_e2e_app_container_ships_client_tools.py
Normal file
@@ -0,0 +1,184 @@
|
||||
"""An application container that ships the engine's client tools.
|
||||
|
||||
This is the shape a dedicated database deploys in: the engine runs as
|
||||
`<app>-database` while the application itself runs as `<app>`, and neither is
|
||||
declared through --database-containers, so both names go through the instance
|
||||
regex. `<app>-database` loses its suffix and lands on the instance `<app>` -
|
||||
and `<app>` carries no database token at all, so a fallback that returns the
|
||||
name unchanged lands on that same instance and offers the application container
|
||||
as a second engine for the same row.
|
||||
|
||||
Discourse is the live example: its application container is named `discourse`
|
||||
by its own launcher and ships pg_dumpall, so a dump command starts there and
|
||||
writes a file that looks like a backup and holds none of the data.
|
||||
"""
|
||||
|
||||
import json
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from baudolo.generation import DUMP_SUFFIX, FILES_DIR, MANIFEST_FILE, SQL_DIR
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
MARKER = "the-application-volume-holds-files"
|
||||
PAYLOAD = "shop-payload"
|
||||
|
||||
|
||||
class TestE2EAppContainerShipsClientTools(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
# uuid4 hex may begin with "db", which the instance regex would split
|
||||
# on and turn the application container into a different instance,
|
||||
# hiding exactly the collision this module is about.
|
||||
cls.prefix = unique("baudolo-e2e-app-tools").replace("-db", "-xb")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.engine = f"{cls.prefix}-shop-database"
|
||||
cls.app = f"{cls.prefix}-shop"
|
||||
cls.engine_volume = f"{cls.prefix}-shop-database-vol"
|
||||
cls.app_volume = f"{cls.prefix}-shop-app-vol"
|
||||
cls.containers = [cls.engine, cls.app]
|
||||
cls.volumes = [cls.engine_volume, cls.app_volume]
|
||||
|
||||
run(["docker", "volume", "create", cls.engine_volume])
|
||||
run(["docker", "volume", "create", cls.app_volume])
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.engine,
|
||||
"-e",
|
||||
"POSTGRES_PASSWORD=shoppw",
|
||||
"-e",
|
||||
"POSTGRES_DB=shopdb",
|
||||
"-e",
|
||||
"POSTGRES_USER=postgres",
|
||||
"-v",
|
||||
f"{cls.engine_volume}:{POSTGRES_DATA_DIR}",
|
||||
POSTGRES_IMAGE,
|
||||
]
|
||||
)
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.app,
|
||||
"--entrypoint",
|
||||
"sh",
|
||||
"-v",
|
||||
f"{cls.app_volume}:/data",
|
||||
POSTGRES_IMAGE,
|
||||
"-c",
|
||||
f"echo '{MARKER}' > /data/marker.txt && sleep 3600",
|
||||
]
|
||||
)
|
||||
|
||||
wait_for_postgres(cls.engine, user="postgres", timeout_s=90)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.engine,
|
||||
"sh",
|
||||
"-lc",
|
||||
(
|
||||
'psql -U postgres -d shopdb -c "CREATE TABLE orders (id int, '
|
||||
f"note text); INSERT INTO orders VALUES (1,'{PAYLOAD}');\""
|
||||
),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(
|
||||
cls.databases_csv,
|
||||
[(cls.app, "shopdb", "postgres", "shoppw")],
|
||||
)
|
||||
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
compose_dir=cls.compose_dir,
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=["dummy-db"],
|
||||
images_no_stop_required=[POSTGRES_IMAGE],
|
||||
)
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
|
||||
def volume_dir(self, volume: str) -> Path:
|
||||
return backup_path(self.backups_dir, self.repo_name, self.version, volume)
|
||||
|
||||
def test_the_engine_volume_was_dumped(self) -> None:
|
||||
dump = self.volume_dir(self.engine_volume) / SQL_DIR / f"shopdb{DUMP_SUFFIX}"
|
||||
self.assertTrue(dump.is_file(), f"expected a dump at {dump}")
|
||||
self.assertIn(PAYLOAD, dump.read_text(encoding="utf-8"))
|
||||
|
||||
def test_the_application_volume_produced_no_dump(self) -> None:
|
||||
"""The collision this module exists for: the application container
|
||||
answers the same instance as the engine and starts a dump of its own."""
|
||||
sql_dir = self.volume_dir(self.app_volume) / SQL_DIR
|
||||
self.assertFalse(
|
||||
sql_dir.exists(),
|
||||
f"the application container was dumped: {sorted(sql_dir.iterdir())}"
|
||||
if sql_dir.exists()
|
||||
else "",
|
||||
)
|
||||
|
||||
def test_the_application_volume_was_backed_up_as_files(self) -> None:
|
||||
"""Refusing the dump must not cost the volume its backup."""
|
||||
marker = self.volume_dir(self.app_volume) / FILES_DIR / "marker.txt"
|
||||
self.assertTrue(marker.is_file(), f"expected a file backup at {marker}")
|
||||
self.assertIn(MARKER, marker.read_text(encoding="utf-8"))
|
||||
|
||||
def test_the_manifest_does_not_call_the_application_volume_a_database(self) -> None:
|
||||
manifest = json.loads(
|
||||
(self.volume_dir(self.app_volume).parent / MANIFEST_FILE).read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
entry = manifest["volumes"][self.app_volume]
|
||||
self.assertFalse(entry["database"], entry)
|
||||
self.assertFalse(entry["dumped"], entry)
|
||||
|
||||
def test_the_manifest_records_the_engine_volume_as_dumped(self) -> None:
|
||||
manifest = json.loads(
|
||||
(self.volume_dir(self.engine_volume).parent / MANIFEST_FILE).read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
entry = manifest["volumes"][self.engine_volume]
|
||||
self.assertTrue(entry["database"], entry)
|
||||
self.assertTrue(entry["dumped"], entry)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,29 +0,0 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import run
|
||||
|
||||
|
||||
class TestE2ECLIContractDumpOnlySql(unittest.TestCase):
|
||||
def test_help_mentions_new_flag(self) -> None:
|
||||
cp = run(["baudolo", "--help"], capture=True, check=True)
|
||||
out = (cp.stdout or "") + "\n" + (cp.stderr or "")
|
||||
self.assertIn(
|
||||
"--dump-only-sql",
|
||||
out,
|
||||
f"Expected '--dump-only-sql' to appear in --help output. Output:\n{out}",
|
||||
)
|
||||
|
||||
def test_old_flag_is_rejected(self) -> None:
|
||||
cp = run(["baudolo", "--dump-only"], capture=True, check=False)
|
||||
self.assertEqual(
|
||||
cp.returncode,
|
||||
2,
|
||||
f"Expected exitcode 2 for unknown args, got {cp.returncode}\n"
|
||||
f"STDOUT={cp.stdout}\nSTDERR={cp.stderr}",
|
||||
)
|
||||
err = (cp.stderr or "") + "\n" + (cp.stdout or "")
|
||||
# Argparse typically prints "unrecognized arguments"
|
||||
self.assertTrue(
|
||||
("unrecognized arguments" in err) or ("usage:" in err.lower()),
|
||||
f"Expected argparse-style error output. Output:\n{err}",
|
||||
)
|
||||
33
tests/e2e/test_e2e_cli_contract_only_sql.py
Normal file
33
tests/e2e/test_e2e_cli_contract_only_sql.py
Normal file
@@ -0,0 +1,33 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import run
|
||||
|
||||
WITHDRAWN_FLAGS = ["--dump-only", "--dump-only-sql", "--everything"]
|
||||
|
||||
|
||||
class TestE2ECLIContractOnlySql(unittest.TestCase):
|
||||
def test_help_mentions_the_flag(self) -> None:
|
||||
cp = run(["baudolo", "--help"], capture=True, check=True)
|
||||
out = (cp.stdout or "") + "\n" + (cp.stderr or "")
|
||||
self.assertIn(
|
||||
"--only-sql",
|
||||
out,
|
||||
f"Expected '--only-sql' to appear in --help output. Output:\n{out}",
|
||||
)
|
||||
|
||||
def test_a_withdrawn_flag_is_rejected(self) -> None:
|
||||
for flag in WITHDRAWN_FLAGS:
|
||||
with self.subTest(flag=flag):
|
||||
cp = run(["baudolo", flag], capture=True, check=False)
|
||||
self.assertEqual(
|
||||
cp.returncode,
|
||||
2,
|
||||
f"Expected exitcode 2 for unknown args, got {cp.returncode}\n"
|
||||
f"STDOUT={cp.stdout}\nSTDERR={cp.stderr}",
|
||||
)
|
||||
err = (cp.stderr or "") + "\n" + (cp.stdout or "")
|
||||
# Argparse typically prints "unrecognized arguments"
|
||||
self.assertTrue(
|
||||
("unrecognized arguments" in err) or ("usage:" in err.lower()),
|
||||
f"Expected argparse-style error output. Output:\n{err}",
|
||||
)
|
||||
177
tests/e2e/test_e2e_engine_detection_by_tool.py
Normal file
177
tests/e2e/test_e2e_engine_detection_by_tool.py
Normal file
@@ -0,0 +1,177 @@
|
||||
"""The engine comes from the tools a container ships, not from its image name.
|
||||
|
||||
Two containers in one backup run, each lying in one direction:
|
||||
|
||||
* a real Postgres tagged `<prefix>-database`, the way a dedicated database is
|
||||
built inside an app's own stack - no engine token anywhere in the name;
|
||||
* an Alpine tagged `postgres:<prefix>`, carrying the token without shipping a
|
||||
single Postgres binary.
|
||||
|
||||
Reading the name gets both wrong, and the second one fatally: pg_dump exits 127
|
||||
inside Alpine and takes the whole run with it.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
IMPOSTOR_BASE_IMAGE = "alpine:3.20"
|
||||
MARKER = "engine-detection-by-tool"
|
||||
|
||||
|
||||
class TestE2EEngineDetectionByTool(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-engine-by-tool")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.engine_image = f"{cls.prefix}-database:17"
|
||||
cls.impostor_image = f"postgres:{cls.prefix}"
|
||||
|
||||
cls.engine_container = f"{cls.prefix}-engine"
|
||||
cls.impostor_container = f"{cls.prefix}-impostor"
|
||||
cls.engine_volume = f"{cls.prefix}-engine-vol"
|
||||
cls.impostor_volume = f"{cls.prefix}-impostor-vol"
|
||||
|
||||
cls.containers = [cls.engine_container, cls.impostor_container]
|
||||
cls.volumes = [cls.engine_volume, cls.impostor_volume]
|
||||
|
||||
run(["docker", "pull", POSTGRES_IMAGE])
|
||||
run(["docker", "pull", IMPOSTOR_BASE_IMAGE])
|
||||
run(["docker", "tag", POSTGRES_IMAGE, cls.engine_image])
|
||||
run(["docker", "tag", IMPOSTOR_BASE_IMAGE, cls.impostor_image])
|
||||
run(["docker", "volume", "create", cls.engine_volume])
|
||||
run(["docker", "volume", "create", cls.impostor_volume])
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.engine_container,
|
||||
"-e",
|
||||
"POSTGRES_PASSWORD=pgpw",
|
||||
"-e",
|
||||
"POSTGRES_DB=appdb",
|
||||
"-e",
|
||||
"POSTGRES_USER=postgres",
|
||||
"-v",
|
||||
f"{cls.engine_volume}:{POSTGRES_DATA_DIR}",
|
||||
cls.engine_image,
|
||||
]
|
||||
)
|
||||
wait_for_postgres(cls.engine_container, user="postgres", timeout_s=90)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.engine_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
(
|
||||
"psql -U postgres -d appdb -c "
|
||||
'"CREATE TABLE t (id int primary key, v text); '
|
||||
"INSERT INTO t VALUES (1,'ok');\""
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.impostor_container,
|
||||
"-v",
|
||||
f"{cls.impostor_volume}:/data",
|
||||
cls.impostor_image,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"echo '{MARKER}' > /data/marker.txt && sleep 3600",
|
||||
]
|
||||
)
|
||||
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(
|
||||
cls.databases_csv,
|
||||
[
|
||||
(cls.engine_container, "appdb", "postgres", "pgpw"),
|
||||
(cls.impostor_container, "appdb", "postgres", "pgpw"),
|
||||
],
|
||||
)
|
||||
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
compose_dir=cls.compose_dir,
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=[cls.engine_container, cls.impostor_container],
|
||||
images_no_stop_required=[cls.engine_image, cls.impostor_image],
|
||||
only_sql=True,
|
||||
)
|
||||
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
run(["docker", "rmi", cls.engine_image], check=False)
|
||||
run(["docker", "rmi", cls.impostor_image], check=False)
|
||||
|
||||
def _volume_dir(self, volume: str):
|
||||
return backup_path(self.backups_dir, self.repo_name, self.version, volume)
|
||||
|
||||
def test_an_engine_without_an_engine_name_is_still_dumped(self) -> None:
|
||||
dump = self._volume_dir(self.engine_volume) / "sql" / "appdb.backup.sql"
|
||||
self.assertTrue(
|
||||
dump.is_file(),
|
||||
f"a Postgres tagged '{self.engine_image}' produced no dump at {dump}",
|
||||
)
|
||||
self.assertIn("Dumped by pg_dump", dump.read_text(encoding="utf-8"))
|
||||
|
||||
def test_an_engine_name_without_an_engine_is_not_dumped(self) -> None:
|
||||
sql_dir = self._volume_dir(self.impostor_volume) / "sql"
|
||||
dumps = list(sql_dir.glob("*.sql")) if sql_dir.exists() else []
|
||||
self.assertEqual(
|
||||
dumps,
|
||||
[],
|
||||
f"'{self.impostor_image}' ships no Postgres yet was dumped: {dumps}",
|
||||
)
|
||||
|
||||
def test_the_recognised_engine_is_dumped_instead_of_copied(self) -> None:
|
||||
files = self._volume_dir(self.engine_volume) / "files"
|
||||
self.assertFalse(
|
||||
files.exists(),
|
||||
f"--only-sql still copied the engine's files to {files}",
|
||||
)
|
||||
|
||||
def test_the_impostor_falls_through_to_a_file_backup(self) -> None:
|
||||
files = self._volume_dir(self.impostor_volume) / "files"
|
||||
self.assertTrue(files.is_dir(), f"expected a file backup at {files}")
|
||||
self.assertEqual(
|
||||
(files / "marker.txt").read_text(encoding="utf-8").strip(), MARKER
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,19 +1,19 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
backup_run,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
REGISTRY_HOST = "svc-db-mariadb-swarm-mgr-01:5000"
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
backup_run,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
)
|
||||
|
||||
|
||||
@@ -30,7 +30,6 @@ class TestE2EFilesFull(unittest.TestCase):
|
||||
cls.containers = []
|
||||
cls.volumes = [cls.volume_src, cls.volume_dst]
|
||||
|
||||
# create source volume with a file
|
||||
run(["docker", "volume", "create", cls.volume_src])
|
||||
run(
|
||||
[
|
||||
@@ -50,7 +49,6 @@ class TestE2EFilesFull(unittest.TestCase):
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(cls.databases_csv, [])
|
||||
|
||||
# Run backup (files should be copied)
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
@@ -97,7 +95,6 @@ class TestE2EFilesFull(unittest.TestCase):
|
||||
]
|
||||
)
|
||||
|
||||
# verify restored file exists in dst volume
|
||||
p = run(
|
||||
[
|
||||
"docker",
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
backup_run,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
)
|
||||
|
||||
|
||||
@@ -29,7 +29,6 @@ class TestE2EFilesNoCopy(unittest.TestCase):
|
||||
cls.containers: list[str] = []
|
||||
cls.volumes = [cls.volume_src]
|
||||
|
||||
# Create source volume and write a marker file
|
||||
run(["docker", "volume", "create", cls.volume_src])
|
||||
run(
|
||||
[
|
||||
@@ -48,7 +47,7 @@ class TestE2EFilesNoCopy(unittest.TestCase):
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(cls.databases_csv, [])
|
||||
|
||||
# dump-only-sql => non-DB volumes are STILL backed up as files
|
||||
# only-sql => non-DB volumes are STILL backed up as files
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
@@ -56,7 +55,7 @@ class TestE2EFilesNoCopy(unittest.TestCase):
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=["dummy-db"],
|
||||
images_no_stop_required=["alpine:3.20"],
|
||||
dump_only_sql=True,
|
||||
only_sql=True,
|
||||
)
|
||||
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
# tests/e2e/test_e2e_images_no_backup_required_early_skip.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
@@ -34,11 +33,9 @@ class TestE2EImagesNoBackupRequiredEarlySkip(unittest.TestCase):
|
||||
cls.containers = [cls.redis_container]
|
||||
cls.volumes = [cls.ignored_volume, cls.normal_volume]
|
||||
|
||||
# Create volumes
|
||||
run(["docker", "volume", "create", cls.ignored_volume])
|
||||
run(["docker", "volume", "create", cls.normal_volume])
|
||||
|
||||
# Start redis container using the ignored volume
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
@@ -71,7 +68,6 @@ class TestE2EImagesNoBackupRequiredEarlySkip(unittest.TestCase):
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(cls.databases_csv, [])
|
||||
|
||||
# Run baudolo with images-no-backup-required redis
|
||||
cmd = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
|
||||
@@ -21,17 +21,18 @@ are verifying is in the DB-dump stage, so testing backup_database() directly
|
||||
keeps the assertion focused and the test runnable both on-host and in DinD.
|
||||
"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import pandas
|
||||
import pandas as pd
|
||||
|
||||
from baudolo.backup import db as db_mod
|
||||
from baudolo.generation import DUMP_SUFFIX, SQL_DIR
|
||||
|
||||
from .helpers import (
|
||||
MARIADB_IMAGE,
|
||||
MARIADB_DATA_DIR,
|
||||
MARIADB_IMAGE,
|
||||
cleanup_docker,
|
||||
require_docker,
|
||||
run,
|
||||
@@ -140,7 +141,7 @@ class TestE2EMariaDBAnonymousPreemption(unittest.TestCase):
|
||||
# paths — just the dump that the negative-control proved is failing
|
||||
# under the same preemption setup.
|
||||
with tempfile.TemporaryDirectory() as volume_dir:
|
||||
df = pandas.DataFrame(
|
||||
df = pd.DataFrame(
|
||||
[(self.db_container, self.db_name, self.db_user, self.db_password)],
|
||||
columns=["instance", "database", "username", "password"],
|
||||
)
|
||||
@@ -148,13 +149,14 @@ class TestE2EMariaDBAnonymousPreemption(unittest.TestCase):
|
||||
container=self.db_container,
|
||||
volume_dir=volume_dir,
|
||||
db_type="mariadb",
|
||||
dump_tool="mariadb-dump",
|
||||
databases_df=df,
|
||||
database_containers=[self.db_container],
|
||||
)
|
||||
self.assertTrue(produced, "backup_database did not produce a dump")
|
||||
dump_path = os.path.join(volume_dir, "sql", f"{self.db_name}.backup.sql")
|
||||
self.assertTrue(os.path.isfile(dump_path), f"expected dump at {dump_path}")
|
||||
with open(dump_path, "r", encoding="utf-8", errors="replace") as f:
|
||||
dump_path = Path(volume_dir) / SQL_DIR / f"{self.db_name}{DUMP_SUFFIX}"
|
||||
self.assertTrue(dump_path.is_file(), f"expected dump at {dump_path}")
|
||||
with dump_path.open(encoding="utf-8", errors="replace") as f:
|
||||
content = f.read()
|
||||
self.assertIn("INSERT INTO", content)
|
||||
self.assertIn("'ok'", content)
|
||||
|
||||
@@ -1,21 +1,20 @@
|
||||
# tests/e2e/test_e2e_mariadb_full.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
MARIADB_IMAGE,
|
||||
MARIADB_DATA_DIR,
|
||||
backup_run,
|
||||
MARIADB_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
unique,
|
||||
wait_for_mariadb,
|
||||
wait_for_mariadb_sql,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
@@ -71,7 +70,6 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
||||
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
||||
)
|
||||
|
||||
# Create table + data via the dedicated user (TCP)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
@@ -79,9 +77,11 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\""
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -112,8 +112,10 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "DROP TABLE {cls.db_name}.t;"',
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "DROP TABLE {cls.db_name}.t;"'
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -161,8 +163,10 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
||||
self.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"',
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"'
|
||||
),
|
||||
]
|
||||
)
|
||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||
|
||||
@@ -1,21 +1,20 @@
|
||||
# tests/e2e/test_e2e_mariadb_no_copy.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
MARIADB_IMAGE,
|
||||
MARIADB_DATA_DIR,
|
||||
backup_run,
|
||||
MARIADB_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
unique,
|
||||
wait_for_mariadb,
|
||||
wait_for_mariadb_sql,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
@@ -69,7 +68,6 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
||||
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
||||
)
|
||||
|
||||
# Create table + data (TCP)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
@@ -77,9 +75,11 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\""
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -89,7 +89,7 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
||||
[(cls.db_container, cls.db_name, cls.db_user, cls.db_password)],
|
||||
)
|
||||
|
||||
# dump-only-sql => no files
|
||||
# only-sql => no files
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
@@ -97,7 +97,7 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=[cls.db_container],
|
||||
images_no_stop_required=[MARIADB_IMAGE],
|
||||
dump_only_sql=True,
|
||||
only_sql=True,
|
||||
)
|
||||
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
@@ -110,8 +110,10 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "DROP TABLE {cls.db_name}.t;"',
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "DROP TABLE {cls.db_name}.t;"'
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -158,8 +160,10 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
||||
self.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"',
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"'
|
||||
),
|
||||
]
|
||||
)
|
||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||
|
||||
116
tests/e2e/test_e2e_only_files.py
Normal file
116
tests/e2e/test_e2e_only_files.py
Normal file
@@ -0,0 +1,116 @@
|
||||
"""--only-files backs a database up as a file tree and asks for no credentials.
|
||||
|
||||
The run deliberately passes no --databases-csv at all: a host that only copies
|
||||
files has no reason to hold database passwords, and requiring the file would
|
||||
make the flag useless there.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
)
|
||||
|
||||
MARKER = "only-files-marker"
|
||||
|
||||
|
||||
class TestE2EOnlyFiles(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-only-files")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.pg_container = f"{cls.prefix}-pg"
|
||||
cls.pg_volume = f"{cls.prefix}-pg-vol"
|
||||
cls.containers = [cls.pg_container]
|
||||
cls.volumes = [cls.pg_volume]
|
||||
|
||||
run(["docker", "volume", "create", cls.pg_volume])
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.pg_container,
|
||||
"-e",
|
||||
"POSTGRES_PASSWORD=pgpw",
|
||||
"-e",
|
||||
"POSTGRES_DB=appdb",
|
||||
"-e",
|
||||
"POSTGRES_USER=postgres",
|
||||
"-v",
|
||||
f"{cls.pg_volume}:{POSTGRES_DATA_DIR}",
|
||||
POSTGRES_IMAGE,
|
||||
]
|
||||
)
|
||||
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.pg_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"echo '{MARKER}' > {POSTGRES_DATA_DIR}/marker.txt",
|
||||
]
|
||||
)
|
||||
|
||||
cp = run(
|
||||
[
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
cls.compose_dir,
|
||||
"--repo-name",
|
||||
cls.repo_name,
|
||||
"--backups-dir",
|
||||
cls.backups_dir,
|
||||
"--images-no-stop-required",
|
||||
POSTGRES_IMAGE,
|
||||
"--only-files",
|
||||
],
|
||||
capture=True,
|
||||
check=True,
|
||||
)
|
||||
cls.stdout = cp.stdout or ""
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
|
||||
def _volume_dir(self):
|
||||
return backup_path(
|
||||
self.backups_dir, self.repo_name, self.version, self.pg_volume
|
||||
)
|
||||
|
||||
def test_the_database_volume_is_backed_up_as_files(self) -> None:
|
||||
marker = self._volume_dir() / "files" / "marker.txt"
|
||||
self.assertTrue(marker.is_file(), f"expected a file backup at {marker}")
|
||||
self.assertEqual(marker.read_text(encoding="utf-8").strip(), MARKER)
|
||||
|
||||
def test_no_dump_is_written(self) -> None:
|
||||
sql_dir = self._volume_dir() / "sql"
|
||||
dumps = list(sql_dir.glob("*.sql")) if sql_dir.exists() else []
|
||||
self.assertEqual(dumps, [], f"did not expect any dump, found: {dumps}")
|
||||
|
||||
def test_the_missing_databases_csv_is_not_reported(self) -> None:
|
||||
self.assertNotIn("databases.csv", self.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,9 +1,11 @@
|
||||
# tests/e2e/test_e2e_dump_only_fallback_to_files.py
|
||||
import json
|
||||
import unittest
|
||||
|
||||
from baudolo.generation import FILES_DIR, MANIFEST_FILE, MANIFEST_SCHEMA, SQL_DIR
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
@@ -12,16 +14,16 @@ from .helpers import (
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
class TestE2EDumpOnlyFallbackToFiles(unittest.TestCase):
|
||||
class TestE2EOnlySqlFallbackToFiles(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-dump-only-sql-fallback")
|
||||
cls.prefix = unique("baudolo-e2e-only-sql-fallback")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
|
||||
@@ -37,7 +39,6 @@ class TestE2EDumpOnlyFallbackToFiles(unittest.TestCase):
|
||||
|
||||
run(["docker", "volume", "create", cls.pg_volume])
|
||||
|
||||
# Start Postgres (creates a real DB volume)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
@@ -59,7 +60,7 @@ class TestE2EDumpOnlyFallbackToFiles(unittest.TestCase):
|
||||
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||
|
||||
# Add a deterministic marker file into the volume
|
||||
cls.marker = "dump-only-sql-fallback-marker"
|
||||
cls.marker = "only-sql-fallback-marker"
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
@@ -75,7 +76,7 @@ class TestE2EDumpOnlyFallbackToFiles(unittest.TestCase):
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(cls.databases_csv, []) # empty except header
|
||||
|
||||
# Run baudolo with --dump-only-sql and a DB container present:
|
||||
# Run baudolo with --only-sql and a DB container present:
|
||||
# Expected: WARNING + FALLBACK to file backup (files/ must exist)
|
||||
cmd = [
|
||||
"baudolo",
|
||||
@@ -93,7 +94,7 @@ class TestE2EDumpOnlyFallbackToFiles(unittest.TestCase):
|
||||
cls.pg_container,
|
||||
"--images-no-stop-required",
|
||||
POSTGRES_IMAGE,
|
||||
"--dump-only-sql",
|
||||
"--only-sql",
|
||||
]
|
||||
cp = run(cmd, capture=True, check=True)
|
||||
|
||||
@@ -124,7 +125,7 @@ class TestE2EDumpOnlyFallbackToFiles(unittest.TestCase):
|
||||
|
||||
def test_warns_about_missing_dump_in_dump_only_mode(self) -> None:
|
||||
self.assertIn(
|
||||
"WARNING: dump-only-sql requested but no DB dump was produced",
|
||||
"WARNING: only-sql requested but no DB dump was produced",
|
||||
self.stdout,
|
||||
f"Expected warning in baudolo output. STDOUT:\n{self.stdout}",
|
||||
)
|
||||
@@ -161,6 +162,31 @@ class TestE2EDumpOnlyFallbackToFiles(unittest.TestCase):
|
||||
f"Did not expect SQL dump files, found: {dumps}",
|
||||
)
|
||||
|
||||
def manifest(self) -> dict:
|
||||
generation = backup_path(
|
||||
self.backups_dir, self.repo_name, self.version, self.pg_volume
|
||||
).parent
|
||||
return json.loads((generation / MANIFEST_FILE).read_text(encoding="utf-8"))
|
||||
|
||||
def test_the_manifest_records_the_volume_as_a_database_left_undumped(self) -> None:
|
||||
"""The fallback is invisible in the tree: files/ looks like any copy."""
|
||||
self.assertEqual(
|
||||
self.manifest()["volumes"][self.pg_volume],
|
||||
{"database": True, "dumped": False, "engine": "postgres"},
|
||||
)
|
||||
|
||||
def test_the_manifest_layout_names_where_the_payload_really_landed(self) -> None:
|
||||
layout = self.manifest()["layout"]
|
||||
volume_dir = backup_path(
|
||||
self.backups_dir, self.repo_name, self.version, self.pg_volume
|
||||
)
|
||||
self.assertTrue((volume_dir / layout["files_dir"]).is_dir())
|
||||
self.assertEqual(layout["files_dir"], FILES_DIR)
|
||||
self.assertEqual(layout["sql_dir"], SQL_DIR)
|
||||
|
||||
def test_the_manifest_states_a_schema_a_reader_can_check(self) -> None:
|
||||
self.assertEqual(self.manifest()["schema"], MANIFEST_SCHEMA)
|
||||
|
||||
def test_restored_files_contain_marker(self) -> None:
|
||||
p = run(
|
||||
[
|
||||
@@ -1,8 +1,11 @@
|
||||
import json
|
||||
import unittest
|
||||
|
||||
from baudolo.generation import MANIFEST_FILE
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
@@ -16,11 +19,11 @@ from .helpers import (
|
||||
)
|
||||
|
||||
|
||||
class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
|
||||
class TestE2EOnlySqlMixedRun(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-dump-only-sql-mixed-run")
|
||||
cls.prefix = unique("baudolo-e2e-only-sql-mixed-run")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
|
||||
@@ -35,7 +38,6 @@ class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
|
||||
cls.containers: list[str] = []
|
||||
cls.volumes = [cls.db_volume, cls.files_volume]
|
||||
|
||||
# Create volumes
|
||||
run(["docker", "volume", "create", cls.db_volume])
|
||||
run(["docker", "volume", "create", cls.files_volume])
|
||||
|
||||
@@ -114,7 +116,6 @@ class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
|
||||
[(cls.pg_container, cls.pg_db, cls.pg_user, cls.pg_password)],
|
||||
)
|
||||
|
||||
# Run baudolo with dump-only-sql
|
||||
cmd = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
@@ -125,7 +126,7 @@ class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
|
||||
cls.pg_container,
|
||||
"--images-no-stop-required",
|
||||
POSTGRES_IMAGE,
|
||||
"--dump-only-sql",
|
||||
"--only-sql",
|
||||
"--backups-dir",
|
||||
cls.backups_dir,
|
||||
"--repo-name",
|
||||
@@ -172,8 +173,8 @@ class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
|
||||
f"Expected files dir for non-DB volume at: {files}",
|
||||
)
|
||||
|
||||
def test_dump_only_sql_does_not_disable_non_db_files_backup(self) -> None:
|
||||
# Regression guard: even with --dump-only-sql, non-DB volumes must still be backed up as files
|
||||
def test_only_sql_does_not_disable_non_db_files_backup(self) -> None:
|
||||
# Regression guard: even with --only-sql, non-DB volumes must still be backed up as files
|
||||
base = backup_path(
|
||||
self.backups_dir, self.repo_name, self.version, self.files_volume
|
||||
)
|
||||
@@ -181,3 +182,21 @@ class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
|
||||
(base / "files").exists(),
|
||||
f"Expected non-DB volume files backup to exist at: {base / 'files'}",
|
||||
)
|
||||
|
||||
def manifest(self) -> dict:
|
||||
generation = backup_path(
|
||||
self.backups_dir, self.repo_name, self.version, self.db_volume
|
||||
).parent
|
||||
return json.loads((generation / MANIFEST_FILE).read_text(encoding="utf-8"))
|
||||
|
||||
def test_the_manifest_records_the_dumped_volume_as_dumped(self) -> None:
|
||||
self.assertEqual(
|
||||
self.manifest()["volumes"][self.db_volume],
|
||||
{"database": True, "dumped": True, "engine": "postgres"},
|
||||
)
|
||||
|
||||
def test_the_manifest_records_the_plain_volume_as_no_database(self) -> None:
|
||||
self.assertEqual(
|
||||
self.manifest()["volumes"][self.files_volume],
|
||||
{"database": False, "dumped": False, "engine": None},
|
||||
)
|
||||
215
tests/e2e/test_e2e_postgres_cluster_restore.py
Normal file
215
tests/e2e/test_e2e_postgres_cluster_restore.py
Normal file
@@ -0,0 +1,215 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
# One statement per entry: psql wraps a multi-statement -c in a transaction,
|
||||
# and CREATE DATABASE is forbidden inside one.
|
||||
SEED_SQL = (
|
||||
"CREATE ROLE app LOGIN PASSWORD 'apppw'",
|
||||
"CREATE DATABASE first OWNER app",
|
||||
"CREATE DATABASE second OWNER app",
|
||||
)
|
||||
SIBLING_SQL = (
|
||||
"CREATE ROLE neighbour LOGIN PASSWORD 'neighbourpw'",
|
||||
"CREATE DATABASE sibling OWNER neighbour",
|
||||
)
|
||||
SIBLING_PAYLOAD = "CREATE TABLE t (v text); INSERT INTO t VALUES ('sibling-payload');"
|
||||
FIRST_SQL = "CREATE TABLE t (v text); INSERT INTO t VALUES ('first-payload');"
|
||||
SECOND_SQL = "CREATE TABLE t (v text); INSERT INTO t VALUES ('second-payload');"
|
||||
|
||||
|
||||
class TestE2EPostgresClusterRestore(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-pg-cluster")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.pg_container = f"{cls.prefix}-pg"
|
||||
cls.pg_volume = f"{cls.prefix}-pg-vol"
|
||||
cls.containers = [cls.pg_container]
|
||||
cls.volumes = [cls.pg_volume]
|
||||
|
||||
run(["docker", "volume", "create", cls.pg_volume])
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.pg_container,
|
||||
"-e",
|
||||
"POSTGRES_PASSWORD=pgpw",
|
||||
"-v",
|
||||
f"{cls.pg_volume}:{POSTGRES_DATA_DIR}",
|
||||
POSTGRES_IMAGE,
|
||||
]
|
||||
)
|
||||
wait_for_postgres(cls.pg_container, user="postgres")
|
||||
|
||||
for statement in SEED_SQL:
|
||||
cls._psql("postgres", statement)
|
||||
cls._psql("first", FIRST_SQL)
|
||||
cls._psql("second", SECOND_SQL)
|
||||
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(
|
||||
cls.databases_csv, [(cls.pg_container, "*", "postgres", "pgpw")]
|
||||
)
|
||||
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
compose_dir=cls.compose_dir,
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=[cls.pg_container],
|
||||
images_no_stop_required=[POSTGRES_IMAGE],
|
||||
)
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
cls.dump = (
|
||||
backup_path(cls.backups_dir, cls.repo_name, cls.version, cls.pg_volume)
|
||||
/ "sql"
|
||||
/ f"{cls.pg_container}.cluster.backup.sql"
|
||||
)
|
||||
|
||||
run(
|
||||
[
|
||||
"baudolo-restore",
|
||||
"cluster",
|
||||
cls.pg_volume,
|
||||
cls.hash,
|
||||
cls.version,
|
||||
"--backups-dir",
|
||||
cls.backups_dir,
|
||||
"--repo-name",
|
||||
cls.repo_name,
|
||||
"--container",
|
||||
cls.pg_container,
|
||||
"--instance",
|
||||
cls.pg_container,
|
||||
"--db-user",
|
||||
"postgres",
|
||||
"--db-password",
|
||||
"pgpw",
|
||||
"--empty",
|
||||
]
|
||||
)
|
||||
|
||||
for statement in SIBLING_SQL:
|
||||
cls._psql("postgres", statement)
|
||||
cls._psql("sibling", SIBLING_PAYLOAD)
|
||||
cls.refused = run(cls._restore_argv(), check=False)
|
||||
|
||||
@classmethod
|
||||
def _restore_argv(cls) -> list:
|
||||
return [
|
||||
"baudolo-restore",
|
||||
"cluster",
|
||||
cls.pg_volume,
|
||||
cls.hash,
|
||||
cls.version,
|
||||
"--backups-dir",
|
||||
cls.backups_dir,
|
||||
"--repo-name",
|
||||
cls.repo_name,
|
||||
"--container",
|
||||
cls.pg_container,
|
||||
"--instance",
|
||||
cls.pg_container,
|
||||
"--db-user",
|
||||
"postgres",
|
||||
"--db-password",
|
||||
"pgpw",
|
||||
"--empty",
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
|
||||
@classmethod
|
||||
def _psql(cls, database: str, sql: str) -> str:
|
||||
p = run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.pg_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f'psql -U postgres -d {database} -t -A -c "{sql}"',
|
||||
]
|
||||
)
|
||||
return (p.stdout or "").strip()
|
||||
|
||||
def test_the_backup_wrote_a_cluster_dump(self) -> None:
|
||||
self.assertTrue(self.dump.is_file(), f"no cluster dump at {self.dump}")
|
||||
|
||||
def test_the_preclean_really_dropped_a_populated_cluster(self) -> None:
|
||||
self.assertEqual(self._psql("first", "SELECT v FROM t"), "first-payload")
|
||||
|
||||
def test_a_second_empty_is_refused_once_a_foreign_database_exists(self) -> None:
|
||||
self.assertNotEqual(self.refused.returncode, 0, self.refused.stdout)
|
||||
self.assertIn("sibling", self.refused.stderr)
|
||||
|
||||
def test_the_refusal_left_the_foreign_database_alone(self) -> None:
|
||||
self.assertEqual(self._psql("sibling", "SELECT v FROM t"), "sibling-payload")
|
||||
|
||||
def test_the_refusal_dropped_nothing_of_its_own(self) -> None:
|
||||
self.assertEqual(self._psql("first", "SELECT v FROM t"), "first-payload")
|
||||
|
||||
def test_both_databases_are_back(self) -> None:
|
||||
listed = self._psql(
|
||||
"postgres",
|
||||
"SELECT datname FROM pg_database WHERE datname IN ('first','second') ORDER BY 1",
|
||||
)
|
||||
self.assertEqual(listed.split(), ["first", "second"])
|
||||
|
||||
def test_each_database_carries_its_own_payload(self) -> None:
|
||||
self.assertEqual(self._psql("first", "SELECT v FROM t"), "first-payload")
|
||||
self.assertEqual(self._psql("second", "SELECT v FROM t"), "second-payload")
|
||||
|
||||
def test_the_superusers_own_create_was_filtered(self) -> None:
|
||||
self.assertEqual(
|
||||
self._psql(
|
||||
"postgres", "SELECT rolsuper FROM pg_roles WHERE rolname = 'postgres'"
|
||||
),
|
||||
"t",
|
||||
)
|
||||
|
||||
def test_the_owning_role_is_back(self) -> None:
|
||||
self.assertEqual(
|
||||
self._psql(
|
||||
"postgres", "SELECT rolname FROM pg_roles WHERE rolname = 'app'"
|
||||
),
|
||||
"app",
|
||||
)
|
||||
|
||||
def test_ownership_survived(self) -> None:
|
||||
self.assertEqual(
|
||||
self._psql(
|
||||
"postgres",
|
||||
"SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = 'first'",
|
||||
),
|
||||
"app",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,9 +1,8 @@
|
||||
# tests/e2e/test_e2e_postgres_empty_drop_hard.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
@@ -25,6 +24,7 @@ from .helpers import (
|
||||
# is not unique") and english_stem_nostop reproduces taiga's text search
|
||||
# dictionary abort (duplicate pg_ts_dict_dictname_index).
|
||||
SCENARIO_SQL = (
|
||||
"CREATE EXTENSION pg_trgm;"
|
||||
"CREATE SCHEMA discourse_functions;"
|
||||
"CREATE TABLE discourse_functions.helper (id int);"
|
||||
"INSERT INTO discourse_functions.helper VALUES (1);"
|
||||
@@ -167,6 +167,13 @@ class TestE2EPostgresEmptyDropHard(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(self._scalar("SELECT public.f(41) + public.f();"), "42")
|
||||
|
||||
def test_the_extension_survived_the_preclean(self) -> None:
|
||||
self.assertEqual(
|
||||
self._scalar("SELECT count(*) FROM pg_extension WHERE extname='pg_trgm';"),
|
||||
"1",
|
||||
)
|
||||
self.assertEqual(self._scalar("SELECT similarity('abc','abc')::int;"), "1")
|
||||
|
||||
def test_text_search_dictionary_restored_once(self) -> None:
|
||||
self.assertEqual(
|
||||
self._scalar(
|
||||
|
||||
@@ -1,20 +1,19 @@
|
||||
# tests/e2e/test_e2e_postgres_full.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
backup_run,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
@@ -55,7 +54,6 @@ class TestE2EPostgresFull(unittest.TestCase):
|
||||
)
|
||||
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||
|
||||
# Create a table + data
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
|
||||
@@ -1,20 +1,19 @@
|
||||
# tests/e2e/test_e2e_postgres_no_copy.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
backup_run,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
@@ -77,7 +76,7 @@ class TestE2EPostgresNoCopy(unittest.TestCase):
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=[cls.pg_container],
|
||||
images_no_stop_required=[POSTGRES_IMAGE],
|
||||
dump_only_sql=True,
|
||||
only_sql=True,
|
||||
)
|
||||
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
159
tests/e2e/test_e2e_postgres_password_required.py
Normal file
159
tests/e2e/test_e2e_postgres_password_required.py
Normal file
@@ -0,0 +1,159 @@
|
||||
"""An engine whose loopback auth really demands a password.
|
||||
|
||||
Every other Postgres scenario runs stock postgres:alpine, whose generated
|
||||
pg_hba grants trust on 127.0.0.1 and ::1 - so `pg_dump -h localhost` never
|
||||
needs the password and a dump succeeds whether or not baudolo hands one to the
|
||||
container. This module makes the password mandatory, which is what a dedicated
|
||||
engine on a real host does.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from baudolo.generation import CLUSTER_SUFFIX, DUMP_SUFFIX, FILES_DIR, SQL_DIR
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
class TestE2EPostgresPasswordRequired(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-pg-password-required")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.pg_container = f"{cls.prefix}-pg"
|
||||
cls.pg_volume = f"{cls.prefix}-pg-vol"
|
||||
cls.containers = [cls.pg_container]
|
||||
cls.volumes = [cls.pg_volume]
|
||||
|
||||
run(["docker", "volume", "create", cls.pg_volume])
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.pg_container,
|
||||
"-e",
|
||||
"POSTGRES_PASSWORD=pgpw",
|
||||
"-e",
|
||||
"POSTGRES_DB=appdb",
|
||||
"-e",
|
||||
"POSTGRES_USER=postgres",
|
||||
# The entrypoint evals this into its initdb call, so the host
|
||||
# lines of pg_hba demand scram while the local socket stays
|
||||
# trust - the entrypoint's own init and the seeding below keep
|
||||
# working, and only a TCP connection needs the password.
|
||||
"-e",
|
||||
"POSTGRES_INITDB_ARGS=--auth-host=scram-sha-256",
|
||||
"-v",
|
||||
f"{cls.pg_volume}:{POSTGRES_DATA_DIR}",
|
||||
POSTGRES_IMAGE,
|
||||
]
|
||||
)
|
||||
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.pg_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
(
|
||||
'psql -U postgres -d appdb -c "CREATE TABLE t (id int primary '
|
||||
"key, v text); INSERT INTO t VALUES (1,'ok');\""
|
||||
),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
cls.unauthenticated = run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.pg_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
"pg_dump -U postgres -d appdb -h localhost",
|
||||
],
|
||||
capture=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(
|
||||
cls.databases_csv,
|
||||
[
|
||||
(cls.pg_container, "appdb", "postgres", "pgpw"),
|
||||
(cls.pg_container, "*", "postgres", "pgpw"),
|
||||
],
|
||||
)
|
||||
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
compose_dir=cls.compose_dir,
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=[cls.pg_container],
|
||||
images_no_stop_required=[POSTGRES_IMAGE],
|
||||
only_sql=True,
|
||||
)
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
|
||||
def volume_dir(self) -> Path:
|
||||
return backup_path(
|
||||
self.backups_dir, self.repo_name, self.version, self.pg_volume
|
||||
)
|
||||
|
||||
def test_a_dump_without_the_password_is_refused_by_the_server(self) -> None:
|
||||
"""Without this the module is vacuous: a pg_hba still saying trust would
|
||||
let a baudolo that forwards nothing pass just as well."""
|
||||
self.assertNotEqual(self.unauthenticated.returncode, 0)
|
||||
self.assertIn("no password supplied", self.unauthenticated.stderr or "")
|
||||
|
||||
def test_the_configured_database_was_dumped(self) -> None:
|
||||
dump = self.volume_dir() / SQL_DIR / f"appdb{DUMP_SUFFIX}"
|
||||
self.assertTrue(dump.is_file(), f"expected a dump at {dump}")
|
||||
self.assertIn("Dumped by pg_dump", dump.read_text(encoding="utf-8"))
|
||||
|
||||
def test_the_dump_carries_the_payload(self) -> None:
|
||||
"""pg_dump emits table data as COPY ... FROM stdin, so the row reads as
|
||||
tab-separated values rather than as an INSERT literal."""
|
||||
dump = self.volume_dir() / SQL_DIR / f"appdb{DUMP_SUFFIX}"
|
||||
self.assertIn("COPY public.t (id, v) FROM stdin;", dump.read_text("utf-8"))
|
||||
self.assertIn("1\tok", dump.read_text(encoding="utf-8"))
|
||||
|
||||
def test_the_cluster_row_was_dumped_too(self) -> None:
|
||||
cluster = self.volume_dir() / SQL_DIR / f"{self.pg_container}{CLUSTER_SUFFIX}"
|
||||
self.assertTrue(cluster.is_file(), f"expected a cluster dump at {cluster}")
|
||||
self.assertIn("CREATE DATABASE", cluster.read_text(encoding="utf-8"))
|
||||
|
||||
def test_only_sql_left_no_file_copy_behind(self) -> None:
|
||||
self.assertFalse((self.volume_dir() / FILES_DIR).exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,9 +1,8 @@
|
||||
# tests/e2e/test_e2e_postgres_single_transaction_live_writer.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
|
||||
120
tests/e2e/test_e2e_restore_files_backing_store.py
Normal file
120
tests/e2e/test_e2e_restore_files_backing_store.py
Normal file
@@ -0,0 +1,120 @@
|
||||
"""Restoring files into a volume that has a backing store of its own.
|
||||
|
||||
Docker keeps the same ``/var/lib/docker/volumes/<name>/_data`` path for such a
|
||||
volume and mounts the real storage over it only while a container holds it.
|
||||
Writing there unmounted lands in the empty directory underneath, is hidden by
|
||||
the next mount, and rsync reports success - so the restore has to refuse.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from .helpers import (
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
ensure_empty_dir,
|
||||
machine_hash,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
)
|
||||
|
||||
MARKER = "restored-payload"
|
||||
VERSION = "20260817000000"
|
||||
|
||||
|
||||
def mountpoint_of(volume: str) -> Path:
|
||||
return Path("/var/lib/docker/volumes") / volume / "_data"
|
||||
|
||||
|
||||
def contents(directory: Path) -> list[str]:
|
||||
return sorted(p.name for p in directory.iterdir()) if directory.is_dir() else []
|
||||
|
||||
|
||||
class TestE2ERestoreFilesBackingStore(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-backing")
|
||||
cls.repo_name = cls.prefix
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
cls.backing = Path(f"/tmp/{cls.prefix}/backing")
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
ensure_empty_dir(str(cls.backing))
|
||||
|
||||
cls.bound_volume = f"{cls.prefix}-bound"
|
||||
cls.plain_volume = f"{cls.prefix}-plain"
|
||||
cls.volumes = [cls.bound_volume, cls.plain_volume]
|
||||
|
||||
for volume in cls.volumes:
|
||||
files = (
|
||||
backup_path(cls.backups_dir, cls.repo_name, VERSION, volume) / "files"
|
||||
)
|
||||
files.mkdir(parents=True, exist_ok=True)
|
||||
(files / "marker.txt").write_text(MARKER, encoding="utf-8")
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"volume",
|
||||
"create",
|
||||
"--driver",
|
||||
"local",
|
||||
"--opt",
|
||||
"type=none",
|
||||
"--opt",
|
||||
"o=bind",
|
||||
"--opt",
|
||||
f"device={cls.backing}",
|
||||
cls.bound_volume,
|
||||
]
|
||||
)
|
||||
run(["docker", "volume", "create", cls.plain_volume])
|
||||
|
||||
cls.refused = cls.restore(cls.bound_volume)
|
||||
cls.accepted = cls.restore(cls.plain_volume)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=[], volumes=cls.volumes)
|
||||
|
||||
@classmethod
|
||||
def restore(cls, volume: str):
|
||||
return run(
|
||||
[
|
||||
"baudolo-restore",
|
||||
"files",
|
||||
volume,
|
||||
machine_hash(),
|
||||
VERSION,
|
||||
"--backups-dir",
|
||||
cls.backups_dir,
|
||||
"--repo-name",
|
||||
cls.repo_name,
|
||||
],
|
||||
check=False,
|
||||
)
|
||||
|
||||
def test_a_volume_with_its_own_backing_store_is_refused(self) -> None:
|
||||
self.assertEqual(self.refused.returncode, 2, self.refused.stdout)
|
||||
self.assertIn("backing store of its own", self.refused.stderr)
|
||||
|
||||
def test_nothing_was_written_into_the_backing_store(self) -> None:
|
||||
self.assertEqual(contents(self.backing), [])
|
||||
|
||||
def test_nothing_was_written_under_the_mount_either(self) -> None:
|
||||
self.assertEqual(
|
||||
contents(mountpoint_of(self.bound_volume)),
|
||||
[],
|
||||
"the copy landed in the directory the next mount hides",
|
||||
)
|
||||
|
||||
def test_a_plain_volume_is_still_restored(self) -> None:
|
||||
self.assertEqual(self.accepted.returncode, 0, self.accepted.stderr)
|
||||
restored = mountpoint_of(self.plain_volume) / "marker.txt"
|
||||
self.assertTrue(restored.is_file(), f"{restored} missing")
|
||||
self.assertEqual(restored.read_text(encoding="utf-8"), MARKER)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,8 +1,8 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
@@ -157,7 +157,6 @@ class TestE2ESeedStarAndDbEntriesBackupPostgres(unittest.TestCase):
|
||||
]
|
||||
)
|
||||
|
||||
# --- Run baudolo with dump-only-sql ---
|
||||
cmd = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
@@ -168,7 +167,7 @@ class TestE2ESeedStarAndDbEntriesBackupPostgres(unittest.TestCase):
|
||||
cls.pg_container,
|
||||
"--images-no-stop-required",
|
||||
POSTGRES_IMAGE,
|
||||
"--dump-only-sql",
|
||||
"--only-sql",
|
||||
"--backups-dir",
|
||||
cls.backups_dir,
|
||||
"--repo-name",
|
||||
@@ -194,7 +193,7 @@ class TestE2ESeedStarAndDbEntriesBackupPostgres(unittest.TestCase):
|
||||
self.assertTrue(sql_dir.exists(), f"Expected sql dir at: {sql_dir}")
|
||||
self.assertFalse(
|
||||
files_dir.exists(),
|
||||
f"Did not expect files dir for DB volume when dump-only-sql succeeded: {files_dir}",
|
||||
f"Did not expect files dir for DB volume when only-sql succeeded: {files_dir}",
|
||||
)
|
||||
|
||||
# Cluster dump file produced by '*' entry
|
||||
|
||||
@@ -17,6 +17,7 @@ from .helpers import require_docker, run, unique
|
||||
|
||||
REPO_SRC = Path(__file__).resolve().parents[2] / "src"
|
||||
DRIVER = Path(__file__).resolve().parent / "snapshot_driver.py"
|
||||
FAITHFUL_DRIVER = Path(__file__).resolve().parent / "faithful_driver.py"
|
||||
IMAGE = "alpine:3.20"
|
||||
PACKAGES = "apk add -q btrfs-progs e2fsprogs zfs python3 util-linux"
|
||||
ATTACH = (
|
||||
@@ -76,20 +77,17 @@ def required(fstype: str) -> bool:
|
||||
return fstype in demanded.replace(",", " ").split()
|
||||
|
||||
|
||||
def stage() -> Path:
|
||||
def stage(driver: Path) -> Path:
|
||||
"""Copy source and driver under /tmp, the only path the DinD daemon shares."""
|
||||
staged = Path("/tmp") / unique("baudolo-e2e-snapshot")
|
||||
shutil.copytree(REPO_SRC, staged / "src")
|
||||
shutil.copy(DRIVER, staged / "driver.py")
|
||||
shutil.copy(driver, staged / "driver.py")
|
||||
return staged
|
||||
|
||||
|
||||
def drive(fstype: str, kind: str, expect: str) -> str:
|
||||
staged = stage()
|
||||
script = (
|
||||
f"set -e; {mount_script(fstype)}; "
|
||||
f"python3 /driver.py {kind} /subject/docker {expect}"
|
||||
)
|
||||
def drive(fstype: str, arguments: str, *, driver: Path = DRIVER) -> str:
|
||||
staged = stage(driver)
|
||||
script = f"set -e; {mount_script(fstype)}; python3 /driver.py {arguments}"
|
||||
try:
|
||||
proc = run(
|
||||
[
|
||||
@@ -115,7 +113,7 @@ def drive(fstype: str, kind: str, expect: str) -> str:
|
||||
shutil.rmtree(staged, ignore_errors=True)
|
||||
if proc.returncode != 0:
|
||||
raise AssertionError(
|
||||
f"{fstype}/{kind} driver failed:\n{proc.stdout}\n{proc.stderr}"
|
||||
f"{fstype} driver failed on {arguments}:\n{proc.stdout}\n{proc.stderr}"
|
||||
)
|
||||
return proc.stdout
|
||||
|
||||
@@ -126,7 +124,7 @@ class TestE2ESnapshot(unittest.TestCase):
|
||||
require_docker()
|
||||
|
||||
def assert_freezes(self, fstype: str) -> None:
|
||||
output = drive(fstype, fstype, "supported")
|
||||
output = drive(fstype, f"{fstype} /subject/docker supported")
|
||||
self.assertIn("PASS the snapshot exposes the volume", output)
|
||||
self.assertIn("PASS a later write does not reach the snapshot", output)
|
||||
self.assertIn("PASS the snapshot is removed afterwards", output)
|
||||
@@ -148,13 +146,24 @@ class TestE2ESnapshot(unittest.TestCase):
|
||||
self.assert_freezes("zfs")
|
||||
|
||||
def test_ext4_has_no_snapshot_and_says_so(self) -> None:
|
||||
output = drive("ext4", "btrfs", "unsupported")
|
||||
output = drive("ext4", "btrfs /subject/docker unsupported")
|
||||
self.assertIn("PASS refused loudly", output)
|
||||
|
||||
def test_an_unknown_kind_is_refused_before_touching_the_filesystem(self) -> None:
|
||||
output = drive("ext4", "lvm", "unsupported")
|
||||
output = drive("ext4", "lvm /subject/docker unsupported")
|
||||
self.assertIn("PASS refused loudly", output)
|
||||
|
||||
def test_a_volume_with_its_own_storage_is_copied_live_not_from_the_snapshot(
|
||||
self,
|
||||
) -> None:
|
||||
output = drive("btrfs", "/subject/docker", driver=FAITHFUL_DRIVER)
|
||||
self.assertIn(
|
||||
"PASS the snapshot shows the other volume as an empty directory", output
|
||||
)
|
||||
self.assertIn("PASS the other volume degrades to live", output)
|
||||
self.assertIn("PASS the plain volume is read from the snapshot", output)
|
||||
self.assertIn("ALL OK", output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
# tests/e2e/test_e2e_swarm_task_skip.py
|
||||
#
|
||||
# Reproduces the swarm flake fixed on this branch: baudolo used to stop a
|
||||
# swarm task container around the volume file backup because its image was
|
||||
# not whitelisted; the orchestrator immediately replaced the stopped task and
|
||||
|
||||
283
tests/e2e/test_e2e_version_gate.py
Normal file
283
tests/e2e/test_e2e_version_gate.py
Normal file
@@ -0,0 +1,283 @@
|
||||
"""A dump from a newer engine must be refused before --empty destroys anything.
|
||||
|
||||
The pre-clean and the replay are two separate sessions with no rollback across
|
||||
them, so a dump the engine cannot parse leaves an emptied database behind. The
|
||||
decisive assertion here is not the non-zero exit - it is that the payload is
|
||||
still readable afterwards.
|
||||
"""
|
||||
|
||||
import re
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from .helpers import (
|
||||
MARIADB_DATA_DIR,
|
||||
MARIADB_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
wait_for_mariadb,
|
||||
wait_for_mariadb_sql,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
PAYLOAD = "gate-payload"
|
||||
FUTURE = "99.0"
|
||||
|
||||
|
||||
def rewrite_version(dump: Path, pattern: str, version: str) -> str:
|
||||
"""Make the dump claim ``version``; return what it claimed before."""
|
||||
text = dump.read_text(encoding="utf-8", errors="replace")
|
||||
found = re.search(pattern, text)
|
||||
if not found:
|
||||
raise AssertionError(f"{dump} carries no version header matching {pattern}")
|
||||
claimed = found.group(1)
|
||||
dump.write_text(
|
||||
text.replace(found.group(0), found.group(0).replace(claimed, version), 1),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return claimed
|
||||
|
||||
|
||||
class GateCase:
|
||||
"""Drive one engine through refusal, escape hatch and truthful replay."""
|
||||
|
||||
engine = ""
|
||||
pattern = ""
|
||||
|
||||
@classmethod
|
||||
def restore(cls, *extra: str):
|
||||
return run(
|
||||
[
|
||||
"baudolo-restore",
|
||||
cls.engine,
|
||||
cls.volume,
|
||||
cls.hash,
|
||||
cls.version,
|
||||
"--backups-dir",
|
||||
cls.backups_dir,
|
||||
"--repo-name",
|
||||
cls.repo_name,
|
||||
"--container",
|
||||
cls.container,
|
||||
"--db-name",
|
||||
cls.db_name,
|
||||
"--db-user",
|
||||
cls.db_user,
|
||||
"--db-password",
|
||||
cls.db_password,
|
||||
"--empty",
|
||||
*extra,
|
||||
],
|
||||
check=False,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def prepare(cls) -> None:
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(
|
||||
cls.databases_csv,
|
||||
[(cls.container, cls.db_name, cls.db_user, cls.db_password)],
|
||||
)
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
compose_dir=cls.compose_dir,
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=[cls.container],
|
||||
images_no_stop_required=[cls.image],
|
||||
)
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
cls.dump = (
|
||||
backup_path(cls.backups_dir, cls.repo_name, cls.version, cls.volume)
|
||||
/ "sql"
|
||||
/ f"{cls.db_name}.backup.sql"
|
||||
)
|
||||
|
||||
cls.truthful_version = rewrite_version(cls.dump, cls.pattern, FUTURE)
|
||||
cls.refused = cls.restore()
|
||||
cls.payload_after_refusal = cls.read_payload()
|
||||
|
||||
cls.forced = cls.restore("--no-version-check")
|
||||
cls.payload_after_force = cls.read_payload()
|
||||
|
||||
rewrite_version(cls.dump, cls.pattern, cls.truthful_version)
|
||||
cls.replayed = cls.restore()
|
||||
cls.payload_after_replay = cls.read_payload()
|
||||
|
||||
def test_the_dump_states_the_engine_it_came_from(self) -> None:
|
||||
self.assertRegex(self.truthful_version, r"^\d+")
|
||||
|
||||
def test_a_newer_dump_is_refused(self) -> None:
|
||||
self.assertNotEqual(self.refused.returncode, 0, self.refused.stdout)
|
||||
|
||||
def test_the_refusal_names_the_version_it_refused(self) -> None:
|
||||
self.assertIn(FUTURE, self.refused.stderr)
|
||||
self.assertIn("older engine", self.refused.stderr)
|
||||
|
||||
def test_the_refusal_left_the_data_untouched(self) -> None:
|
||||
self.assertEqual(
|
||||
self.payload_after_refusal,
|
||||
PAYLOAD,
|
||||
"--empty pre-cleaned before the version was checked",
|
||||
)
|
||||
|
||||
def test_the_escape_hatch_replays_anyway(self) -> None:
|
||||
self.assertEqual(self.forced.returncode, 0, self.forced.stderr)
|
||||
self.assertEqual(self.payload_after_force, PAYLOAD)
|
||||
|
||||
def test_a_truthful_dump_replays(self) -> None:
|
||||
self.assertEqual(self.replayed.returncode, 0, self.replayed.stderr)
|
||||
self.assertEqual(self.payload_after_replay, PAYLOAD)
|
||||
|
||||
|
||||
class TestE2EPostgresVersionGate(GateCase, unittest.TestCase):
|
||||
engine = "postgres"
|
||||
pattern = r"-- Dumped from database version (\S+)"
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-pg-gate")
|
||||
cls.container = f"{cls.prefix}-pg"
|
||||
cls.volume = f"{cls.prefix}-pg-vol"
|
||||
cls.image = POSTGRES_IMAGE
|
||||
cls.db_name = "appdb"
|
||||
cls.db_user = "postgres"
|
||||
cls.db_password = "pgpw"
|
||||
|
||||
run(["docker", "volume", "create", cls.volume])
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.container,
|
||||
"-e",
|
||||
f"POSTGRES_PASSWORD={cls.db_password}",
|
||||
"-v",
|
||||
f"{cls.volume}:{POSTGRES_DATA_DIR}",
|
||||
POSTGRES_IMAGE,
|
||||
]
|
||||
)
|
||||
wait_for_postgres(cls.container, user=cls.db_user)
|
||||
cls.sql("postgres", f"CREATE DATABASE {cls.db_name}")
|
||||
cls.sql(
|
||||
cls.db_name,
|
||||
f"CREATE TABLE t (v text); INSERT INTO t VALUES ('{PAYLOAD}');",
|
||||
)
|
||||
cls.prepare()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=[cls.container], volumes=[cls.volume])
|
||||
|
||||
@classmethod
|
||||
def sql(cls, database: str, statement: str) -> str:
|
||||
p = run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f'psql -U {cls.db_user} -d {database} -t -A -c "{statement}"',
|
||||
],
|
||||
check=False,
|
||||
)
|
||||
return (p.stdout or "").strip()
|
||||
|
||||
@classmethod
|
||||
def read_payload(cls) -> str:
|
||||
return cls.sql(cls.db_name, "SELECT v FROM t")
|
||||
|
||||
|
||||
class TestE2EMariadbVersionGate(GateCase, unittest.TestCase):
|
||||
engine = "mariadb"
|
||||
pattern = r"-- Server version\s+(\S+)"
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-mdb-gate")
|
||||
cls.container = f"{cls.prefix}-mdb"
|
||||
cls.volume = f"{cls.prefix}-mdb-vol"
|
||||
cls.image = MARIADB_IMAGE
|
||||
cls.db_name = "appdb"
|
||||
cls.db_user = "test"
|
||||
cls.db_password = "testpw"
|
||||
|
||||
run(["docker", "volume", "create", cls.volume])
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.container,
|
||||
"-e",
|
||||
"MARIADB_ROOT_PASSWORD=rootpw",
|
||||
"-e",
|
||||
f"MARIADB_DATABASE={cls.db_name}",
|
||||
"-e",
|
||||
f"MARIADB_USER={cls.db_user}",
|
||||
"-e",
|
||||
f"MARIADB_PASSWORD={cls.db_password}",
|
||||
"-v",
|
||||
f"{cls.volume}:{MARIADB_DATA_DIR}",
|
||||
MARIADB_IMAGE,
|
||||
]
|
||||
)
|
||||
wait_for_mariadb(cls.container, root_password="rootpw", timeout_s=90)
|
||||
wait_for_mariadb_sql(
|
||||
cls.container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
||||
)
|
||||
cls.sql(
|
||||
f"CREATE TABLE {cls.db_name}.t (v VARCHAR(50)); "
|
||||
f"INSERT INTO {cls.db_name}.t VALUES ('{PAYLOAD}');"
|
||||
)
|
||||
cls.prepare()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=[cls.container], volumes=[cls.volume])
|
||||
|
||||
@classmethod
|
||||
def sql(cls, statement: str) -> str:
|
||||
p = run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.container,
|
||||
"sh",
|
||||
"-lc",
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-N -B -e "{statement}"'
|
||||
),
|
||||
],
|
||||
check=False,
|
||||
)
|
||||
return (p.stdout or "").strip()
|
||||
|
||||
@classmethod
|
||||
def read_payload(cls) -> str:
|
||||
return cls.sql(f"SELECT v FROM {cls.db_name}.t")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,10 @@
|
||||
"""The smallest argv the backup CLI accepts, shared by every test that drives it."""
|
||||
|
||||
REQUIRED_PAIRS = [
|
||||
("--compose-dir", "/compose"),
|
||||
("--backups-dir", "/backups"),
|
||||
("--repo-name", "stack"),
|
||||
("--databases-csv", "/etc/baudolo/databases.csv"),
|
||||
]
|
||||
REQUIRED = [arg for pair in REQUIRED_PAIRS for arg in pair]
|
||||
BASE_ARGV = ["baudolo", *REQUIRED]
|
||||
|
||||
@@ -3,7 +3,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def touch(p: Path) -> None:
|
||||
|
||||
@@ -1,22 +1,21 @@
|
||||
import io
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import redirect_stderr
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
# Adjust if your package name/import path differs.
|
||||
from baudolo.backup.dumps import load_databases_df
|
||||
|
||||
|
||||
EXPECTED_COLUMNS = ["instance", "database", "username", "password"]
|
||||
|
||||
|
||||
class TestLoadDatabasesDf(unittest.TestCase):
|
||||
def test_missing_csv_is_handled_with_warning_and_empty_df(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
missing_path = os.path.join(td, "does-not-exist.csv")
|
||||
missing_path = str(Path(td) / "does-not-exist.csv")
|
||||
|
||||
buf = io.StringIO()
|
||||
with redirect_stderr(buf):
|
||||
@@ -32,9 +31,8 @@ class TestLoadDatabasesDf(unittest.TestCase):
|
||||
|
||||
def test_empty_csv_is_handled_with_warning_and_empty_df(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
empty_path = os.path.join(td, "databases.csv")
|
||||
# Create an empty file (0 bytes)
|
||||
with open(empty_path, "w", encoding="utf-8") as f:
|
||||
empty_path = Path(td) / "databases.csv"
|
||||
with empty_path.open("w", encoding="utf-8") as f:
|
||||
f.write("")
|
||||
|
||||
buf = io.StringIO()
|
||||
@@ -51,10 +49,10 @@ class TestLoadDatabasesDf(unittest.TestCase):
|
||||
|
||||
def test_valid_csv_loads_without_warning(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
csv_path = os.path.join(td, "databases.csv")
|
||||
csv_path = Path(td) / "databases.csv"
|
||||
|
||||
content = "instance;database;username;password\nmyapp;*;dbuser;secret\n"
|
||||
with open(csv_path, "w", encoding="utf-8") as f:
|
||||
with csv_path.open("w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
|
||||
buf = io.StringIO()
|
||||
|
||||
84
tests/unit/backup/test_app_manifest.py
Normal file
84
tests/unit/backup/test_app_manifest.py
Normal file
@@ -0,0 +1,84 @@
|
||||
"""What main() records in the manifest for each volume it touched."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from baudolo.backup import app
|
||||
from baudolo.backup.dumps import VolumeOutcome
|
||||
from baudolo.backup.volume import Backing
|
||||
|
||||
from . import REQUIRED_PAIRS
|
||||
|
||||
ARGV = ["baudolo", *[arg for pair in REQUIRED_PAIRS for arg in pair]]
|
||||
|
||||
|
||||
def drive(argv: list[str], dump_result: VolumeOutcome) -> dict:
|
||||
"""Run main() over one volume and return the manifest's volume section.
|
||||
|
||||
Args:
|
||||
argv: the command line under test.
|
||||
dump_result: what backup_dumps_for_volume reports.
|
||||
"""
|
||||
with (
|
||||
mock.patch("sys.argv", argv),
|
||||
mock.patch.object(app, "get_machine_id", return_value="machine"),
|
||||
mock.patch.object(app, "create_version_directory", return_value="/gen"),
|
||||
mock.patch.object(app, "create_volume_directory", return_value="/gen/vol"),
|
||||
mock.patch.object(app, "load_databases_df", return_value=None),
|
||||
mock.patch.object(app, "docker_volume_names", return_value=["pgdata"]),
|
||||
mock.patch.object(app, "containers_using_volume", return_value=["db"]),
|
||||
mock.patch.object(app, "volume_is_fully_ignored", return_value=False),
|
||||
mock.patch.object(app, "backup_dumps_for_volume", return_value=dump_result),
|
||||
mock.patch.object(app, "inspect_backing", return_value=Backing("/data")),
|
||||
mock.patch.object(app, "write_manifest") as manifest,
|
||||
mock.patch.object(app, "stamp_directory"),
|
||||
mock.patch.object(app, "handle_docker_compose_services"),
|
||||
mock.patch("os.path.isdir", return_value=True),
|
||||
mock.patch.object(app, "backup_volume"),
|
||||
mock.patch.object(app, "filter_stoppable", return_value=[]),
|
||||
mock.patch.object(app, "requires_stop", return_value=False),
|
||||
mock.patch.object(app, "change_containers_status"),
|
||||
):
|
||||
app.main()
|
||||
return manifest.call_args.args[1]
|
||||
|
||||
|
||||
class TestManifest(unittest.TestCase):
|
||||
def test_a_database_volume_without_a_dump_is_recorded_as_undumped(self) -> None:
|
||||
volumes = drive(
|
||||
[*ARGV, "--only-sql"],
|
||||
VolumeOutcome(database=True, dumped=False, engine="postgres"),
|
||||
)
|
||||
self.assertEqual(
|
||||
volumes["pgdata"],
|
||||
VolumeOutcome(database=True, dumped=False, engine="postgres"),
|
||||
)
|
||||
|
||||
def test_a_dumped_database_volume_is_recorded_as_dumped(self) -> None:
|
||||
volumes = drive(
|
||||
[*ARGV, "--only-sql"],
|
||||
VolumeOutcome(database=True, dumped=True, engine="mariadb"),
|
||||
)
|
||||
self.assertEqual(volumes["pgdata"].dumped, True)
|
||||
self.assertEqual(volumes["pgdata"].engine, "mariadb")
|
||||
|
||||
def test_a_plain_volume_is_recorded_as_no_database(self) -> None:
|
||||
volumes = drive(ARGV, VolumeOutcome(database=False, dumped=False))
|
||||
self.assertEqual(volumes["pgdata"].database, False)
|
||||
self.assertIsNone(volumes["pgdata"].engine)
|
||||
|
||||
def test_the_dumped_volume_is_recorded_even_though_the_copy_is_skipped(
|
||||
self,
|
||||
) -> None:
|
||||
"""--only-sql returns to the loop head on success, before the copy."""
|
||||
volumes = drive(
|
||||
[*ARGV, "--only-sql"],
|
||||
VolumeOutcome(database=True, dumped=True, engine="postgres"),
|
||||
)
|
||||
self.assertIn("pgdata", volumes)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
66
tests/unit/backup/test_app_only_files.py
Normal file
66
tests/unit/backup/test_app_only_files.py
Normal file
@@ -0,0 +1,66 @@
|
||||
"""Contract of --only-files: no dump is attempted, every volume is copied."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from baudolo.backup import app
|
||||
from baudolo.backup.volume import Backing
|
||||
|
||||
from . import REQUIRED_PAIRS
|
||||
|
||||
ARGV_WITHOUT_CSV = [
|
||||
"baudolo",
|
||||
*[arg for pair in REQUIRED_PAIRS if pair[0] != "--databases-csv" for arg in pair],
|
||||
"--only-files",
|
||||
]
|
||||
|
||||
|
||||
def drive(argv: list[str]) -> tuple[list[str], list, list]:
|
||||
backed_up: list[str] = []
|
||||
|
||||
def record_backup(versions_dir, volume_name, volume_dir, *, authoritative, source):
|
||||
backed_up.append(volume_name)
|
||||
|
||||
with (
|
||||
mock.patch("sys.argv", argv),
|
||||
mock.patch.object(app, "get_machine_id", return_value="machine"),
|
||||
mock.patch.object(app, "create_version_directory", return_value="/gen"),
|
||||
mock.patch.object(app, "create_volume_directory", return_value="/gen/vol"),
|
||||
mock.patch.object(app, "load_databases_df") as load_csv,
|
||||
mock.patch.object(app, "docker_volume_names", return_value=["pgdata"]),
|
||||
mock.patch.object(app, "containers_using_volume", return_value=["db"]),
|
||||
mock.patch.object(app, "volume_is_fully_ignored", return_value=False),
|
||||
mock.patch.object(app, "backup_dumps_for_volume") as dumps,
|
||||
mock.patch.object(app, "inspect_backing", return_value=Backing("/data")),
|
||||
mock.patch.object(app, "write_manifest"),
|
||||
mock.patch.object(app, "stamp_directory"),
|
||||
mock.patch.object(app, "handle_docker_compose_services"),
|
||||
mock.patch("os.path.isdir", return_value=True),
|
||||
mock.patch.object(app, "backup_volume", side_effect=record_backup),
|
||||
mock.patch.object(app, "filter_stoppable", return_value=[]),
|
||||
mock.patch.object(app, "requires_stop", return_value=False),
|
||||
mock.patch.object(app, "change_containers_status"),
|
||||
):
|
||||
app.main()
|
||||
return backed_up, dumps.mock_calls, load_csv.mock_calls
|
||||
|
||||
|
||||
class TestOnlyFiles(unittest.TestCase):
|
||||
def test_no_dump_is_attempted(self) -> None:
|
||||
_backed_up, dumps, _load_csv = drive(ARGV_WITHOUT_CSV)
|
||||
self.assertEqual(dumps, [])
|
||||
|
||||
def test_the_databases_csv_is_never_read(self) -> None:
|
||||
"""It may legitimately be absent, so reading it would abort the run."""
|
||||
_backed_up, _dumps, load_csv = drive(ARGV_WITHOUT_CSV)
|
||||
self.assertEqual(load_csv, [])
|
||||
|
||||
def test_the_volume_is_still_copied(self) -> None:
|
||||
backed_up, _dumps, _load_csv = drive(ARGV_WITHOUT_CSV)
|
||||
self.assertEqual(backed_up, ["pgdata"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -6,7 +6,11 @@ import unittest
|
||||
from unittest import mock
|
||||
|
||||
from baudolo.backup import app
|
||||
from baudolo.backup import snapshot as snapshot_mod
|
||||
from baudolo.backup.snapshot import volume_snapshot
|
||||
from baudolo.backup.volume import Backing
|
||||
|
||||
from . import BASE_ARGV
|
||||
|
||||
|
||||
def stubbed_snapshot(kind: str, subject: str, tag: str):
|
||||
@@ -14,11 +18,7 @@ def stubbed_snapshot(kind: str, subject: str, tag: str):
|
||||
|
||||
|
||||
ARGV = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
"/compose",
|
||||
"--backups-dir",
|
||||
"/backups",
|
||||
*BASE_ARGV,
|
||||
"--snapshot",
|
||||
"btrfs",
|
||||
"--snapshot-subject",
|
||||
@@ -26,7 +26,7 @@ ARGV = [
|
||||
]
|
||||
|
||||
|
||||
def drive(*, present: bool) -> list[dict]:
|
||||
def drive(*, present: bool = True, reason: str | None = None) -> list[dict]:
|
||||
calls: list[dict] = []
|
||||
|
||||
def record(versions_dir, volume_name, volume_dir, *, authoritative, source):
|
||||
@@ -45,11 +45,15 @@ def drive(*, present: bool) -> list[dict]:
|
||||
mock.patch.object(app, "volume_is_fully_ignored", return_value=False),
|
||||
mock.patch.object(app, "backup_dumps_for_volume", return_value=(False, False)),
|
||||
mock.patch.object(
|
||||
app, "get_storage_path", return_value="/var/lib/docker/volumes/vol/_data/"
|
||||
app,
|
||||
"inspect_backing",
|
||||
return_value=Backing("/var/lib/docker/volumes/vol/_data"),
|
||||
),
|
||||
mock.patch.object(snapshot_mod, "unsnapshotted", return_value=reason),
|
||||
mock.patch.object(app, "write_manifest"),
|
||||
mock.patch.object(app, "stamp_directory"),
|
||||
mock.patch.object(app, "handle_docker_compose_services"),
|
||||
mock.patch.object(app.os.path, "isdir", return_value=present),
|
||||
mock.patch("os.path.isdir", return_value=present),
|
||||
mock.patch.object(app, "backup_volume", side_effect=record),
|
||||
mock.patch.object(app, "volume_snapshot", stubbed_snapshot),
|
||||
):
|
||||
@@ -75,6 +79,14 @@ class TestSnapshotBranch(unittest.TestCase):
|
||||
self.assertEqual(call["source"], "/var/lib/docker/volumes/vol/_data/")
|
||||
self.assertFalse(call["authoritative"])
|
||||
|
||||
def test_a_volume_with_its_own_backing_store_is_copied_live(self) -> None:
|
||||
call = drive(reason="it declares its own backing store")[0]
|
||||
self.assertEqual(call["source"], "/var/lib/docker/volumes/vol/_data/")
|
||||
self.assertFalse(
|
||||
call["authoritative"],
|
||||
"the snapshot holds an empty directory for it, not its data",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -7,13 +7,12 @@ import unittest
|
||||
from unittest import mock
|
||||
|
||||
from baudolo.backup import app
|
||||
from baudolo.backup.volume import Backing
|
||||
|
||||
from . import BASE_ARGV
|
||||
|
||||
ARGV = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
"/compose",
|
||||
"--backups-dir",
|
||||
"/backups",
|
||||
*BASE_ARGV,
|
||||
"--volumes-no-backup-required",
|
||||
"derived",
|
||||
]
|
||||
@@ -47,10 +46,11 @@ def drive() -> tuple[list[str], list[str], list[str]]:
|
||||
),
|
||||
mock.patch.object(app, "volume_is_fully_ignored", return_value=False),
|
||||
mock.patch.object(app, "backup_dumps_for_volume", return_value=(False, False)),
|
||||
mock.patch.object(app, "get_storage_path", return_value="/data/"),
|
||||
mock.patch.object(app, "inspect_backing", return_value=Backing("/data")),
|
||||
mock.patch.object(app, "write_manifest"),
|
||||
mock.patch.object(app, "stamp_directory"),
|
||||
mock.patch.object(app, "handle_docker_compose_services"),
|
||||
mock.patch.object(app.os.path, "isdir", return_value=True),
|
||||
mock.patch("os.path.isdir", return_value=True),
|
||||
mock.patch.object(app, "backup_volume", side_effect=record_backup),
|
||||
mock.patch.object(app, "filter_stoppable", return_value=[]),
|
||||
mock.patch.object(app, "requires_stop", return_value=False),
|
||||
|
||||
@@ -7,7 +7,7 @@ from unittest import mock
|
||||
|
||||
from baudolo.backup.cli import parse_args
|
||||
|
||||
REQUIRED = ["--compose-dir", "/compose", "--backups-dir", "/backups"]
|
||||
from . import REQUIRED, REQUIRED_PAIRS
|
||||
|
||||
|
||||
def parse(*extra: str):
|
||||
@@ -68,15 +68,42 @@ class TestSnapshotFlags(unittest.TestCase):
|
||||
|
||||
|
||||
class TestRequiredFlags(unittest.TestCase):
|
||||
def test_backups_dir_is_required(self) -> None:
|
||||
with mock.patch("sys.argv", ["baudolo", "--compose-dir", "/compose"]):
|
||||
with self.assertRaises(SystemExit):
|
||||
def test_no_flag_falls_back_to_a_default(self) -> None:
|
||||
for omitted, _ in REQUIRED_PAIRS:
|
||||
argv = [a for pair in REQUIRED_PAIRS if pair[0] != omitted for a in pair]
|
||||
with (
|
||||
self.subTest(omitted=omitted),
|
||||
mock.patch("sys.argv", ["baudolo", *argv]),
|
||||
self.assertRaises(SystemExit),
|
||||
):
|
||||
parse_args()
|
||||
|
||||
def test_compose_dir_is_required(self) -> None:
|
||||
with mock.patch("sys.argv", ["baudolo", "--backups-dir", "/backups"]):
|
||||
with self.assertRaises(SystemExit):
|
||||
parse_args()
|
||||
|
||||
class TestBackupScope(unittest.TestCase):
|
||||
"""--only-sql and --only-files name the two halves a generation can hold."""
|
||||
|
||||
def test_both_halves_by_default(self) -> None:
|
||||
args = parse()
|
||||
self.assertFalse(args.only_sql)
|
||||
self.assertFalse(args.only_files)
|
||||
|
||||
def test_either_half_alone_is_accepted(self) -> None:
|
||||
self.assertTrue(parse("--only-sql").only_sql)
|
||||
self.assertTrue(parse("--only-files").only_files)
|
||||
|
||||
def test_asking_for_both_halves_alone_is_rejected(self) -> None:
|
||||
with self.assertRaises(SystemExit):
|
||||
parse("--only-sql", "--only-files")
|
||||
|
||||
def test_only_files_needs_no_databases_csv(self) -> None:
|
||||
argv = [
|
||||
arg
|
||||
for pair in REQUIRED_PAIRS
|
||||
if pair[0] != "--databases-csv"
|
||||
for arg in pair
|
||||
]
|
||||
with mock.patch("sys.argv", ["baudolo", *argv, "--only-files"]):
|
||||
self.assertIsNone(parse_args().databases_csv)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -3,7 +3,6 @@ from __future__ import annotations
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
from unittest.mock import patch
|
||||
|
||||
from .compose_fixture import setup_compose_dir as _setup_compose_dir
|
||||
@@ -99,7 +98,7 @@ class TestCompose(unittest.TestCase):
|
||||
str(d), ["up", "-d", "--force-recreate"]
|
||||
)
|
||||
|
||||
expected: List[str] = [
|
||||
expected: list[str] = [
|
||||
"/usr/bin/docker",
|
||||
"compose",
|
||||
"--chdir",
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from typing import List
|
||||
from unittest.mock import patch
|
||||
|
||||
from . import BASE_ARGV
|
||||
|
||||
|
||||
class HardRestartArgTests(unittest.TestCase):
|
||||
"""The hard-restart list defaults to empty (no compose down/up); callers
|
||||
@@ -11,17 +12,13 @@ class HardRestartArgTests(unittest.TestCase):
|
||||
the dir is a stack whose overlay network collides with compose up, pass
|
||||
nothing."""
|
||||
|
||||
def _parse(self, extra: List[str]):
|
||||
def _parse(self, extra: list[str]):
|
||||
import sys
|
||||
|
||||
from baudolo.backup import cli
|
||||
|
||||
argv = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
"/tmp",
|
||||
"--backups-dir",
|
||||
"/tmp/backup",
|
||||
*BASE_ARGV,
|
||||
"--database-containers",
|
||||
"postgres",
|
||||
"--images-no-stop-required",
|
||||
@@ -43,23 +40,6 @@ class HardRestartArgTests(unittest.TestCase):
|
||||
args = self._parse(["--hard-restart-projects", "mailu", "foo"])
|
||||
self.assertEqual(args.hard_restart_projects, ["mailu", "foo"])
|
||||
|
||||
def test_backups_dir_is_required(self) -> None:
|
||||
import sys
|
||||
|
||||
from baudolo.backup import cli
|
||||
|
||||
argv = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
"/tmp",
|
||||
"--database-containers",
|
||||
"postgres",
|
||||
"--images-no-stop-required",
|
||||
"redis",
|
||||
]
|
||||
with patch.object(sys, "argv", argv), self.assertRaises(SystemExit):
|
||||
cli.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
|
||||
@@ -2,33 +2,34 @@ import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import pandas
|
||||
import pandas as pd
|
||||
|
||||
from baudolo.backup import db as db_mod
|
||||
|
||||
|
||||
def _df(rows):
|
||||
return pandas.DataFrame(
|
||||
rows, columns=["instance", "database", "username", "password"]
|
||||
)
|
||||
return pd.DataFrame(rows, columns=["instance", "database", "username", "password"])
|
||||
|
||||
|
||||
def _capture_commands(*, db_type, rows, container):
|
||||
def _capture_dumps(*, db_type, rows, container, dump_tool="mariadb-dump"):
|
||||
"""Every (argv, env) the dump path would have run."""
|
||||
captured = []
|
||||
|
||||
def _capture(cmd):
|
||||
captured.append(cmd)
|
||||
return []
|
||||
def _capture(command, out_file, *, env=None):
|
||||
captured.append((list(command), env))
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
with patch.object(db_mod, "execute_shell_command", side_effect=_capture):
|
||||
db_mod.backup_database(
|
||||
container=container,
|
||||
volume_dir=td,
|
||||
db_type=db_type,
|
||||
databases_df=_df(rows),
|
||||
database_containers=[container],
|
||||
)
|
||||
with (
|
||||
tempfile.TemporaryDirectory() as td,
|
||||
patch.object(db_mod, "execute_to_file", side_effect=_capture),
|
||||
):
|
||||
db_mod.backup_database(
|
||||
container=container,
|
||||
volume_dir=td,
|
||||
db_type=db_type,
|
||||
dump_tool=dump_tool,
|
||||
databases_df=_df(rows),
|
||||
database_containers=[container],
|
||||
)
|
||||
return captured
|
||||
|
||||
|
||||
@@ -38,32 +39,64 @@ class TestMariaDBDumpUsesTCP(unittest.TestCase):
|
||||
# the connection is auth-matched against '%' instead of socket->localhost.
|
||||
|
||||
def test_mariadb_dump_forces_tcp_loopback(self):
|
||||
captured = _capture_commands(
|
||||
captured = _capture_dumps(
|
||||
db_type="mariadb",
|
||||
rows=[("mariadb", "appdb", "appuser", "s3cret")],
|
||||
container="mariadb",
|
||||
)
|
||||
dump_cmds = [c for c in captured if "mariadb-dump" in c]
|
||||
self.assertEqual(
|
||||
len(dump_cmds), 1, f"expected one dump command, got: {captured}"
|
||||
)
|
||||
self.assertEqual(len(captured), 1, f"expected one dump, got: {captured}")
|
||||
|
||||
cmd = dump_cmds[0]
|
||||
self.assertIn("-h 127.0.0.1", cmd)
|
||||
self.assertIn("--protocol=tcp", cmd)
|
||||
self.assertIn("-u appuser", cmd)
|
||||
self.assertIn("-ps3cret", cmd)
|
||||
self.assertIn(" appdb", cmd)
|
||||
argv, env = captured[0]
|
||||
self.assertEqual(argv[:3], ["docker", "exec", "mariadb"])
|
||||
self.assertIn("--protocol=tcp", argv)
|
||||
self.assertEqual(argv[argv.index("-h") + 1], "127.0.0.1")
|
||||
self.assertEqual(argv[argv.index("-u") + 1], "appuser")
|
||||
self.assertIn("-ps3cret", argv)
|
||||
self.assertEqual(argv[-1], "appdb")
|
||||
self.assertIsNone(env)
|
||||
|
||||
def test_the_probed_client_is_the_one_invoked(self):
|
||||
captured = _capture_dumps(
|
||||
db_type="mariadb",
|
||||
rows=[("mariadb", "appdb", "appuser", "s3cret")],
|
||||
container="mariadb",
|
||||
dump_tool="mysqldump",
|
||||
)
|
||||
argv, _env = captured[0]
|
||||
self.assertIn("mysqldump", argv)
|
||||
self.assertNotIn("mariadb-dump", argv)
|
||||
|
||||
def test_postgres_dump_unaffected(self):
|
||||
captured = _capture_commands(
|
||||
captured = _capture_dumps(
|
||||
db_type="postgres",
|
||||
rows=[("pg", "appdb", "appuser", "s3cret")],
|
||||
container="pg",
|
||||
)
|
||||
dump_cmds = [c for c in captured if "pg_dump" in c and "pg_dumpall" not in c]
|
||||
self.assertEqual(len(dump_cmds), 1)
|
||||
self.assertNotIn("--protocol=tcp", dump_cmds[0])
|
||||
argv, _env = captured[0]
|
||||
self.assertIn("pg_dump", argv)
|
||||
self.assertNotIn("--protocol=tcp", argv)
|
||||
|
||||
def test_the_password_travels_in_the_environment_not_the_argv(self):
|
||||
"""A process listing shows argv; PGPASSWORD must not be in it."""
|
||||
captured = _capture_dumps(
|
||||
db_type="postgres",
|
||||
rows=[("pg", "appdb", "appuser", "s3cret")],
|
||||
container="pg",
|
||||
)
|
||||
argv, env = captured[0]
|
||||
self.assertEqual(env, {"PGPASSWORD": "s3cret"})
|
||||
self.assertNotIn("s3cret", argv)
|
||||
|
||||
|
||||
class TestNoShellReachesTheDump(unittest.TestCase):
|
||||
def test_a_hostile_database_name_never_reaches_a_command(self):
|
||||
"""validate_database refuses it, so no argv is built at all."""
|
||||
with self.assertRaises(ValueError):
|
||||
_capture_dumps(
|
||||
db_type="postgres",
|
||||
rows=[("pg", "app;rm -rf /", "appuser", "s3cret")],
|
||||
container="pg",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
61
tests/unit/backup/test_docker_exec_env.py
Normal file
61
tests/unit/backup/test_docker_exec_env.py
Normal file
@@ -0,0 +1,61 @@
|
||||
"""How a secret reaches the command running inside the container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from baudolo.backup.db import fallback_pg_dumpall
|
||||
from baudolo.backup.docker import docker_exec_argv
|
||||
|
||||
|
||||
class TestForwardEnv(unittest.TestCase):
|
||||
def test_nothing_is_added_when_no_variable_is_named(self) -> None:
|
||||
self.assertEqual(
|
||||
docker_exec_argv("c1", ["true"]),
|
||||
["docker", "exec", "c1", "true"],
|
||||
)
|
||||
|
||||
def test_a_named_variable_is_forwarded_without_its_value(self) -> None:
|
||||
"""-e NAME=value would publish the secret in the host's process list."""
|
||||
argv = docker_exec_argv("c1", ["true"], forward_env=["PGPASSWORD"])
|
||||
self.assertEqual(argv, ["docker", "exec", "-e", "PGPASSWORD", "c1", "true"])
|
||||
|
||||
def test_the_flag_precedes_the_container(self) -> None:
|
||||
"""docker reads options before the container name, arguments after it."""
|
||||
argv = docker_exec_argv(
|
||||
"c1", ["pg_dump", "-U", "u"], interactive=True, forward_env=["PGPASSWORD"]
|
||||
)
|
||||
self.assertLess(argv.index("-e"), argv.index("c1"))
|
||||
self.assertLess(argv.index("-i"), argv.index("c1"))
|
||||
self.assertGreater(argv.index("pg_dump"), argv.index("c1"))
|
||||
|
||||
def test_several_variables_each_get_their_own_flag(self) -> None:
|
||||
argv = docker_exec_argv("c1", ["true"], forward_env=["A", "B"])
|
||||
self.assertEqual(argv[:6], ["docker", "exec", "-e", "A", "-e", "B"])
|
||||
|
||||
|
||||
class TestPostgresDumpCarriesThePassword(unittest.TestCase):
|
||||
def test_the_cluster_dump_forwards_pgpassword(self) -> None:
|
||||
seen: dict = {}
|
||||
|
||||
def fake(command, out_file, *, env=None):
|
||||
seen["command"] = command
|
||||
seen["env"] = env
|
||||
|
||||
import baudolo.backup.db as db
|
||||
|
||||
original = db.execute_to_file
|
||||
db.execute_to_file = fake
|
||||
try:
|
||||
fallback_pg_dumpall("pg", "user", "secret", "/tmp/out.sql")
|
||||
finally:
|
||||
db.execute_to_file = original
|
||||
|
||||
self.assertIn("-e", seen["command"])
|
||||
self.assertEqual(seen["command"][seen["command"].index("-e") + 1], "PGPASSWORD")
|
||||
self.assertEqual(seen["env"], {"PGPASSWORD": "secret"})
|
||||
self.assertNotIn("secret", seen["command"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,50 +0,0 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from baudolo.backup import docker as docker_mod
|
||||
|
||||
|
||||
def _with_image(reference: str):
|
||||
return patch.object(docker_mod, "execute_shell_command", return_value=[reference])
|
||||
|
||||
|
||||
class TestImageName(unittest.TestCase):
|
||||
def test_plain_reference(self) -> None:
|
||||
with _with_image("postgres:16"):
|
||||
self.assertEqual(docker_mod.image_name("c1"), "postgres")
|
||||
|
||||
def test_registry_host_is_dropped(self) -> None:
|
||||
with _with_image("svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5"):
|
||||
self.assertEqual(docker_mod.image_name("c1"), "postgres_custom")
|
||||
|
||||
def test_pull_through_path_is_kept(self) -> None:
|
||||
with _with_image(
|
||||
"svc-db-mariadb-swarm-mgr-01:5000/ghcr.io/x/mirror/docker.io/postgres:16"
|
||||
):
|
||||
self.assertEqual(
|
||||
docker_mod.image_name("c1"), "ghcr.io/x/mirror/docker.io/postgres"
|
||||
)
|
||||
|
||||
def test_digest_is_dropped(self) -> None:
|
||||
with _with_image("registry:5000/postgres@sha256:" + "0" * 64):
|
||||
self.assertEqual(docker_mod.image_name("c1"), "postgres")
|
||||
|
||||
|
||||
class TestHasImage(unittest.TestCase):
|
||||
def test_registry_hostname_does_not_decide_the_engine(self) -> None:
|
||||
with _with_image("svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5"):
|
||||
self.assertFalse(docker_mod.has_image("c1", "mariadb"))
|
||||
with _with_image("svc-db-mariadb-swarm-mgr-01:5000/postgres_custom:17-3.5"):
|
||||
self.assertTrue(docker_mod.has_image("c1", "postgres"))
|
||||
|
||||
def test_tag_does_not_decide_the_engine(self) -> None:
|
||||
with _with_image("registry:5000/xwiki_custom:lts-postgres-tomcat"):
|
||||
self.assertFalse(docker_mod.has_image("c1", "postgres"))
|
||||
|
||||
def test_mirrored_mariadb_still_matches(self) -> None:
|
||||
with _with_image("registry:5000/ghcr.io/x/mirror/docker.io/mariadb:11"):
|
||||
self.assertTrue(docker_mod.has_image("c1", "mariadb"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -2,7 +2,7 @@ import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from baudolo.backup import docker as docker_mod
|
||||
from baudolo.backup.shell import BackupException
|
||||
from baudolo.backup.shell import BackupError
|
||||
|
||||
|
||||
class TestIsSwarmTask(unittest.TestCase):
|
||||
@@ -21,7 +21,7 @@ class TestIsSwarmTask(unittest.TestCase):
|
||||
@patch.object(
|
||||
docker_mod,
|
||||
"execute_shell_command",
|
||||
side_effect=[BackupException("gone"), []],
|
||||
side_effect=[BackupError("gone"), []],
|
||||
)
|
||||
def test_vanished_container_counts_as_not_stoppable(self, _mock) -> None:
|
||||
# A container removed between listing and inspect must not abort the
|
||||
@@ -32,13 +32,13 @@ class TestIsSwarmTask(unittest.TestCase):
|
||||
@patch.object(
|
||||
docker_mod,
|
||||
"execute_shell_command",
|
||||
side_effect=[BackupException("daemon hiccup"), ["still-here"]],
|
||||
side_effect=[BackupError("daemon hiccup"), ["still-here"]],
|
||||
)
|
||||
def test_inspect_failure_on_existing_container_still_fails(self, _mock) -> None:
|
||||
# If the container still exists, an inspect failure must keep failing
|
||||
# the run: silently skipping the stop would back up a hot volume and
|
||||
# report green without the stop guarantee.
|
||||
with self.assertRaises(BackupException):
|
||||
with self.assertRaises(BackupError):
|
||||
docker_mod.is_swarm_task("still-here")
|
||||
|
||||
|
||||
|
||||
44
tests/unit/backup/test_docker_tool_probe.py
Normal file
44
tests/unit/backup/test_docker_tool_probe.py
Normal file
@@ -0,0 +1,44 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from baudolo.backup import docker as docker_mod
|
||||
from baudolo.backup.shell import BackupError
|
||||
|
||||
|
||||
class TestImageId(unittest.TestCase):
|
||||
def test_the_id_is_returned_without_surrounding_whitespace(self) -> None:
|
||||
with patch.object(
|
||||
docker_mod, "execute_shell_command", return_value=["sha256:abc \n"]
|
||||
):
|
||||
self.assertEqual(docker_mod.image_id("c1"), "sha256:abc")
|
||||
|
||||
|
||||
class TestHasTool(unittest.TestCase):
|
||||
def test_a_tool_that_runs_is_present(self) -> None:
|
||||
with patch.object(docker_mod, "execute_shell_command", return_value=[]):
|
||||
self.assertTrue(docker_mod.has_tool("c1", "pg_dumpall"))
|
||||
|
||||
def test_a_tool_that_exits_non_zero_is_absent(self) -> None:
|
||||
with patch.object(
|
||||
docker_mod, "execute_shell_command", side_effect=BackupError("127")
|
||||
):
|
||||
self.assertFalse(docker_mod.has_tool("c1", "mariadb-dump"))
|
||||
|
||||
def test_the_probe_needs_no_shell_in_the_image(self) -> None:
|
||||
"""A distroless database ships no shell; `sh -c` would deny every tool."""
|
||||
captured = []
|
||||
|
||||
def _capture(cmd):
|
||||
captured.append(cmd)
|
||||
return []
|
||||
|
||||
with patch.object(docker_mod, "execute_shell_command", side_effect=_capture):
|
||||
docker_mod.has_tool("c1", "pg_dumpall")
|
||||
|
||||
self.assertEqual(
|
||||
captured, [["docker", "exec", "c1", "pg_dumpall", "--version"]]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
137
tests/unit/backup/test_dumps_engine_detection.py
Normal file
137
tests/unit/backup/test_dumps_engine_detection.py
Normal file
@@ -0,0 +1,137 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from baudolo.backup import dumps as dumps_mod
|
||||
|
||||
|
||||
def _df(rows):
|
||||
return pd.DataFrame(rows, columns=["instance", "database", "username", "password"])
|
||||
|
||||
|
||||
class _Probe:
|
||||
def __init__(self, available, image="sha256:aaa"):
|
||||
self.available = set(available)
|
||||
self.image = image
|
||||
self.calls = []
|
||||
|
||||
def has_tool(self, container, tool):
|
||||
self.calls.append((container, tool))
|
||||
return tool in self.available
|
||||
|
||||
def image_id(self, container):
|
||||
return self.image if isinstance(self.image, str) else self.image[container]
|
||||
|
||||
|
||||
def _detect(probe, container="c1"):
|
||||
dumps_mod._ENGINE_BY_IMAGE.clear()
|
||||
with (
|
||||
patch.object(dumps_mod, "has_tool", probe.has_tool),
|
||||
patch.object(dumps_mod, "image_id", probe.image_id),
|
||||
):
|
||||
return dumps_mod.container_engine(container)
|
||||
|
||||
|
||||
class TestContainerEngine(unittest.TestCase):
|
||||
def test_a_postgres_is_found_by_its_dump_tool(self):
|
||||
self.assertEqual(_detect(_Probe(["pg_dumpall"])), ("postgres", "pg_dumpall"))
|
||||
|
||||
def test_a_mariadb_is_found_by_its_dump_tool(self):
|
||||
self.assertEqual(_detect(_Probe(["mariadb-dump"])), ("mariadb", "mariadb-dump"))
|
||||
|
||||
def test_an_image_with_only_mysqldump_is_dumped_with_mysqldump(self):
|
||||
self.assertEqual(_detect(_Probe(["mysqldump"])), ("mariadb", "mysqldump"))
|
||||
|
||||
def test_a_container_without_either_tool_is_no_database(self):
|
||||
self.assertIsNone(_detect(_Probe([])))
|
||||
|
||||
def test_the_probe_stops_at_the_first_tool_it_finds(self):
|
||||
probe = _Probe(["pg_dumpall", "mariadb-dump"])
|
||||
_detect(probe)
|
||||
self.assertEqual(probe.calls, [("c1", "pg_dumpall")])
|
||||
|
||||
def test_the_image_name_does_not_decide_the_engine(self):
|
||||
"""The trap the old substring test fell into, from both directions."""
|
||||
probe = _Probe(["pg_dumpall"], image="svc-db-mariadb-mgr-01:5000/pg_custom")
|
||||
self.assertEqual(_detect(probe), ("postgres", "pg_dumpall"))
|
||||
|
||||
probe = _Probe(["mariadb-dump"], image="discourse-database:17")
|
||||
self.assertEqual(_detect(probe), ("mariadb", "mariadb-dump"))
|
||||
|
||||
|
||||
class TestProbeCache(unittest.TestCase):
|
||||
def test_replicas_of_one_image_are_probed_once(self):
|
||||
probe = _Probe(["pg_dumpall"])
|
||||
dumps_mod._ENGINE_BY_IMAGE.clear()
|
||||
with (
|
||||
patch.object(dumps_mod, "has_tool", probe.has_tool),
|
||||
patch.object(dumps_mod, "image_id", probe.image_id),
|
||||
):
|
||||
first = dumps_mod.container_engine("replica-1")
|
||||
second = dumps_mod.container_engine("replica-2")
|
||||
self.assertEqual(first, second)
|
||||
self.assertEqual(len(probe.calls), 1)
|
||||
|
||||
def test_a_second_image_is_probed_separately(self):
|
||||
probe = _Probe(["pg_dumpall"], image={"pg": "sha256:aaa", "app": "sha256:bbb"})
|
||||
dumps_mod._ENGINE_BY_IMAGE.clear()
|
||||
with (
|
||||
patch.object(dumps_mod, "has_tool", probe.has_tool),
|
||||
patch.object(dumps_mod, "image_id", probe.image_id),
|
||||
):
|
||||
self.assertEqual(
|
||||
dumps_mod.container_engine("pg"), ("postgres", "pg_dumpall")
|
||||
)
|
||||
probe.available = set()
|
||||
self.assertIsNone(dumps_mod.container_engine("app"))
|
||||
|
||||
|
||||
class TestBackupDispatch(unittest.TestCase):
|
||||
def test_the_probed_tool_reaches_the_dump(self):
|
||||
probe = _Probe(["mysqldump"])
|
||||
seen = {}
|
||||
|
||||
def _fake_backup_database(**kwargs):
|
||||
seen.update(kwargs)
|
||||
return True
|
||||
|
||||
dumps_mod._ENGINE_BY_IMAGE.clear()
|
||||
with (
|
||||
patch.object(dumps_mod, "has_tool", probe.has_tool),
|
||||
patch.object(dumps_mod, "image_id", probe.image_id),
|
||||
patch.object(dumps_mod, "backup_database", _fake_backup_database),
|
||||
):
|
||||
outcome = dumps_mod.backup_mariadb_or_postgres(
|
||||
container="c1",
|
||||
volume_dir="/tmp",
|
||||
databases_df=_df([("c1", "appdb", "u", "p")]),
|
||||
database_containers=["c1"],
|
||||
)
|
||||
|
||||
self.assertTrue(outcome.database)
|
||||
self.assertTrue(outcome.dumped)
|
||||
self.assertEqual(outcome.engine, "mariadb")
|
||||
self.assertEqual(seen["db_type"], "mariadb")
|
||||
self.assertEqual(seen["dump_tool"], "mysqldump")
|
||||
|
||||
def test_a_non_database_container_is_left_to_the_file_backup(self):
|
||||
probe = _Probe([])
|
||||
dumps_mod._ENGINE_BY_IMAGE.clear()
|
||||
with (
|
||||
patch.object(dumps_mod, "has_tool", probe.has_tool),
|
||||
patch.object(dumps_mod, "image_id", probe.image_id),
|
||||
):
|
||||
self.assertEqual(
|
||||
dumps_mod.backup_mariadb_or_postgres(
|
||||
container="c1",
|
||||
volume_dir="/tmp",
|
||||
databases_df=_df([]),
|
||||
database_containers=[],
|
||||
),
|
||||
dumps_mod.VolumeOutcome(database=False, dumped=False, engine=None),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
80
tests/unit/backup/test_get_instance.py
Normal file
80
tests/unit/backup/test_get_instance.py
Normal file
@@ -0,0 +1,80 @@
|
||||
"""Which databases.csv instance a container name resolves to.
|
||||
|
||||
The cases are the container names real deployments produce, in both compose
|
||||
and swarm, so a change to the regex has to state which shape it gives up.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from baudolo.backup.db import get_instance
|
||||
|
||||
|
||||
class TestDeclaredContainers(unittest.TestCase):
|
||||
def test_a_declared_container_is_its_own_instance(self) -> None:
|
||||
self.assertEqual(
|
||||
get_instance("postgres-central", ["postgres-central"]), "postgres-central"
|
||||
)
|
||||
|
||||
def test_a_declaration_beats_the_regex(self) -> None:
|
||||
"""A declared name is taken whole even when it carries a token the
|
||||
fallback would otherwise strip."""
|
||||
self.assertEqual(
|
||||
get_instance("shop-database", ["shop-database"]), "shop-database"
|
||||
)
|
||||
|
||||
def test_a_qualified_central_name_still_has_to_be_declared(self) -> None:
|
||||
self.assertIsNone(get_instance("postgres-central", []))
|
||||
|
||||
|
||||
class TestContainersNamedAfterTheirEngine(unittest.TestCase):
|
||||
def test_a_bare_engine_name_is_its_own_instance(self) -> None:
|
||||
for name in ("postgres", "mariadb", "mysql", "db", "database"):
|
||||
with self.subTest(container=name):
|
||||
self.assertEqual(get_instance(name, []), name)
|
||||
|
||||
def test_a_swarm_task_of_such_a_container_keeps_the_instance(self) -> None:
|
||||
self.assertEqual(get_instance("postgres_postgres.1.k3f9x2", []), "postgres")
|
||||
self.assertEqual(get_instance("mariadb_mariadb.1.k3f9x2", []), "mariadb")
|
||||
|
||||
|
||||
class TestDedicatedEngines(unittest.TestCase):
|
||||
def test_compose_names_the_container_with_a_hyphen(self) -> None:
|
||||
self.assertEqual(get_instance("discourse-database", []), "discourse")
|
||||
|
||||
def test_swarm_names_the_task_with_an_underscore_and_a_slot(self) -> None:
|
||||
"""Swarm suppresses container_name and names the task
|
||||
<stack>_<service>.<slot>.<id>, which must land on the same instance as
|
||||
the compose name so one databases.csv serves both modes."""
|
||||
self.assertEqual(get_instance("discourse_database.1.k3f9x2", []), "discourse")
|
||||
|
||||
def test_an_explicitly_named_engine_keeps_its_entity(self) -> None:
|
||||
self.assertEqual(get_instance("bigbluebutton-postgres-1", []), "bigbluebutton")
|
||||
|
||||
def test_the_short_token_is_stripped_too(self) -> None:
|
||||
self.assertEqual(get_instance("matomo-db", []), "matomo")
|
||||
|
||||
def test_mariadb_uses_the_same_suffix(self) -> None:
|
||||
self.assertEqual(get_instance("matomo-database", []), "matomo")
|
||||
|
||||
def test_an_engine_named_suffix_is_stripped_too(self) -> None:
|
||||
self.assertEqual(get_instance("shop-mariadb", []), "shop")
|
||||
self.assertEqual(get_instance("shop-mysql", []), "shop")
|
||||
|
||||
|
||||
class TestApplicationContainers(unittest.TestCase):
|
||||
def test_a_bare_application_name_is_not_a_database(self) -> None:
|
||||
"""Returning the name unchanged here would offer the application as a
|
||||
second engine for its own dedicated database's instance."""
|
||||
self.assertIsNone(get_instance("discourse", []))
|
||||
|
||||
def test_a_swarm_application_task_is_not_a_database(self) -> None:
|
||||
self.assertIsNone(get_instance("discourse_discourse.1.k3f9x2", []))
|
||||
|
||||
def test_an_application_that_merely_starts_with_a_token_is_not_split(self) -> None:
|
||||
self.assertIsNone(get_instance("dbeaver", []))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -8,6 +8,7 @@ from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from baudolo.backup import layout as mod
|
||||
from baudolo.backup.shell import BackupError
|
||||
|
||||
|
||||
class TestVersionDirectory(unittest.TestCase):
|
||||
@@ -17,11 +18,12 @@ class TestVersionDirectory(unittest.TestCase):
|
||||
self.assertTrue(Path(created).is_dir())
|
||||
self.assertEqual(Path(created).name, "20260731020304")
|
||||
|
||||
def test_it_is_idempotent(self) -> None:
|
||||
def test_it_refuses_a_generation_another_run_already_claimed(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
first = mod.create_version_directory(tmp, "20260731")
|
||||
second = mod.create_version_directory(tmp, "20260731")
|
||||
self.assertEqual(first, second)
|
||||
mod.create_version_directory(tmp, "20260731")
|
||||
with self.assertRaises(BackupError) as caught:
|
||||
mod.create_version_directory(tmp, "20260731")
|
||||
self.assertIn("20260731", str(caught.exception))
|
||||
|
||||
def test_it_creates_missing_parents(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
|
||||
@@ -4,19 +4,19 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from baudolo.backup.shell import BackupException
|
||||
from baudolo.backup.shell import BackupError
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
||||
|
||||
|
||||
class Runner:
|
||||
def __init__(self, replies: dict[str, list[str]] | None = None) -> None:
|
||||
self.calls: list[str] = []
|
||||
self.calls: list[list[str]] = []
|
||||
self.replies = replies or {}
|
||||
|
||||
def __call__(self, command: str) -> list[str]:
|
||||
self.calls.append(command)
|
||||
def __call__(self, command: list[str]) -> list[str]:
|
||||
self.calls.append(list(command))
|
||||
for prefix, reply in self.replies.items():
|
||||
if command.startswith(prefix):
|
||||
if " ".join(command).startswith(prefix):
|
||||
return reply
|
||||
return []
|
||||
|
||||
@@ -28,7 +28,14 @@ class TestBtrfs(unittest.TestCase):
|
||||
pass
|
||||
self.assertEqual(
|
||||
run.calls[0],
|
||||
"btrfs subvolume snapshot -r /var/lib/docker /var/lib/docker/.baudolo-20260731",
|
||||
[
|
||||
"btrfs",
|
||||
"subvolume",
|
||||
"snapshot",
|
||||
"-r",
|
||||
"/var/lib/docker",
|
||||
"/var/lib/docker/.baudolo-20260731",
|
||||
],
|
||||
)
|
||||
|
||||
def test_it_removes_the_snapshot_afterwards(self) -> None:
|
||||
@@ -36,7 +43,8 @@ class TestBtrfs(unittest.TestCase):
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
||||
pass
|
||||
self.assertEqual(
|
||||
run.calls[-1], "btrfs subvolume delete /var/lib/docker/.baudolo-20260731"
|
||||
run.calls[-1],
|
||||
["btrfs", "subvolume", "delete", "/var/lib/docker/.baudolo-20260731"],
|
||||
)
|
||||
|
||||
def test_it_maps_a_volume_path_into_the_snapshot(self) -> None:
|
||||
@@ -61,10 +69,12 @@ class TestBtrfs(unittest.TestCase):
|
||||
|
||||
def test_it_removes_the_snapshot_even_when_the_body_raises(self) -> None:
|
||||
run = Runner()
|
||||
with self.assertRaises(ZeroDivisionError):
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
||||
raise ZeroDivisionError
|
||||
self.assertTrue(run.calls[-1].startswith("btrfs subvolume delete"))
|
||||
with (
|
||||
self.assertRaises(ZeroDivisionError),
|
||||
volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run),
|
||||
):
|
||||
raise ZeroDivisionError
|
||||
self.assertEqual(run.calls[-1][:3], ["btrfs", "subvolume", "delete"])
|
||||
|
||||
|
||||
class TestZfs(unittest.TestCase):
|
||||
@@ -75,13 +85,15 @@ class TestZfs(unittest.TestCase):
|
||||
run = self._run()
|
||||
with volume_snapshot("zfs", "/var/lib/docker", "20260731", run=run):
|
||||
pass
|
||||
self.assertIn("zfs snapshot tank/docker@baudolo-20260731", run.calls)
|
||||
self.assertIn(["zfs", "snapshot", "tank/docker@baudolo-20260731"], run.calls)
|
||||
|
||||
def test_it_destroys_the_snapshot_afterwards(self) -> None:
|
||||
run = self._run()
|
||||
with volume_snapshot("zfs", "/var/lib/docker", "20260731", run=run):
|
||||
pass
|
||||
self.assertEqual(run.calls[-1], "zfs destroy tank/docker@baudolo-20260731")
|
||||
self.assertEqual(
|
||||
run.calls[-1], ["zfs", "destroy", "tank/docker@baudolo-20260731"]
|
||||
)
|
||||
|
||||
def test_it_maps_a_volume_path_through_the_dot_zfs_directory(self) -> None:
|
||||
run = self._run()
|
||||
@@ -93,26 +105,30 @@ class TestZfs(unittest.TestCase):
|
||||
|
||||
def test_an_unmounted_dataset_is_an_error(self) -> None:
|
||||
run = Runner({"zfs list": [""]})
|
||||
with self.assertRaises(SnapshotError):
|
||||
with volume_snapshot("zfs", "/var/lib/docker", "20260731", run=run):
|
||||
pass
|
||||
with (
|
||||
self.assertRaises(SnapshotError),
|
||||
volume_snapshot("zfs", "/var/lib/docker", "20260731", run=run),
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
class TestRejections(unittest.TestCase):
|
||||
def test_an_unknown_kind_is_rejected(self) -> None:
|
||||
run = Runner()
|
||||
with self.assertRaises(SnapshotError):
|
||||
with volume_snapshot("ext4", "/var/lib/docker", "20260731", run=run):
|
||||
pass
|
||||
with (
|
||||
self.assertRaises(SnapshotError),
|
||||
volume_snapshot("ext4", "/var/lib/docker", "20260731", run=run),
|
||||
):
|
||||
pass
|
||||
self.assertEqual(run.calls, [])
|
||||
|
||||
def test_a_path_outside_the_subject_is_rejected(self) -> None:
|
||||
run = Runner()
|
||||
with volume_snapshot(
|
||||
"btrfs", "/var/lib/docker", "20260731", run=run
|
||||
) as resolve:
|
||||
with self.assertRaises(SnapshotError):
|
||||
resolve("/etc/passwd")
|
||||
with (
|
||||
volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve,
|
||||
self.assertRaises(SnapshotError),
|
||||
):
|
||||
resolve("/etc/passwd")
|
||||
|
||||
def test_the_subject_itself_resolves_to_the_snapshot_root(self) -> None:
|
||||
run = Runner()
|
||||
@@ -125,9 +141,9 @@ class TestRejections(unittest.TestCase):
|
||||
|
||||
|
||||
class Busy(Runner):
|
||||
def __call__(self, command: str) -> list[str]:
|
||||
if command.startswith("btrfs subvolume delete"):
|
||||
raise BackupException("target is busy")
|
||||
def __call__(self, command: list[str]) -> list[str]:
|
||||
if command[:3] == ["btrfs", "subvolume", "delete"]:
|
||||
raise BackupError("target is busy")
|
||||
return super().__call__(command)
|
||||
|
||||
|
||||
@@ -137,9 +153,11 @@ class TestRemovalFailure(unittest.TestCase):
|
||||
pass
|
||||
|
||||
def test_a_failed_removal_does_not_mask_the_body(self) -> None:
|
||||
with self.assertRaises(ZeroDivisionError):
|
||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=Busy()):
|
||||
raise ZeroDivisionError
|
||||
with (
|
||||
self.assertRaises(ZeroDivisionError),
|
||||
volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=Busy()),
|
||||
):
|
||||
raise ZeroDivisionError
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
127
tests/unit/backup/test_snapshot_faithfulness.py
Normal file
127
tests/unit/backup/test_snapshot_faithfulness.py
Normal file
@@ -0,0 +1,127 @@
|
||||
"""Which volumes a snapshot of the subject actually contains.
|
||||
|
||||
The failure this guards against is silent: a volume with a backing store of
|
||||
its own is present inside the snapshot as an empty directory, so rsync
|
||||
succeeds, the generation is stamped complete, and the volume is empty in it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from baudolo.backup.snapshot import SnapshotError, snapshot_source, unsnapshotted
|
||||
from baudolo.backup.volume import Backing
|
||||
|
||||
|
||||
class TestUnsnapshotted(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.subject = tempfile.mkdtemp()
|
||||
self.mountpoint = str(Path(self.subject) / "volumes" / "app" / "_data")
|
||||
Path(self.mountpoint).mkdir(parents=True)
|
||||
|
||||
def backing(self, **kwargs) -> Backing:
|
||||
return Backing(kwargs.pop("mountpoint", self.mountpoint), **kwargs)
|
||||
|
||||
def test_a_plain_local_volume_is_captured(self) -> None:
|
||||
self.assertIsNone(unsnapshotted(self.backing(), self.subject))
|
||||
|
||||
def test_a_foreign_driver_is_not(self) -> None:
|
||||
reason = unsnapshotted(self.backing(driver="rexray"), self.subject)
|
||||
self.assertIn("rexray", reason)
|
||||
|
||||
def test_declared_driver_options_are_not(self) -> None:
|
||||
reason = unsnapshotted(
|
||||
self.backing(options={"type": "nfs", "device": ":/exports/app"}),
|
||||
self.subject,
|
||||
)
|
||||
self.assertIn("backing store", reason)
|
||||
|
||||
def test_the_declaration_decides_not_the_mount_table(self) -> None:
|
||||
"""Docker unmounts an NFS volume when its last container stops."""
|
||||
with mock.patch.object(os.path, "ismount", return_value=False):
|
||||
reason = unsnapshotted(self.backing(options={"type": "nfs"}), self.subject)
|
||||
self.assertIsNotNone(reason)
|
||||
|
||||
def test_a_volume_without_a_mountpoint_is_not(self) -> None:
|
||||
reason = unsnapshotted(Backing(""), self.subject)
|
||||
self.assertIn("no mountpoint", reason)
|
||||
|
||||
def test_an_own_mount_is_not(self) -> None:
|
||||
with mock.patch.object(os.path, "ismount", return_value=True):
|
||||
reason = unsnapshotted(self.backing(), self.subject)
|
||||
self.assertIn("own mount", reason)
|
||||
|
||||
def test_a_filesystem_boundary_is_not(self) -> None:
|
||||
real = os.stat
|
||||
|
||||
def crossing(path, *args, **kwargs):
|
||||
info = real(path, *args, **kwargs)
|
||||
if os.path.realpath(path) == os.path.realpath(self.mountpoint):
|
||||
return os.stat_result(
|
||||
(info.st_mode, info.st_ino, info.st_dev + 1, *tuple(info)[3:])
|
||||
)
|
||||
return info
|
||||
|
||||
with mock.patch.object(os, "stat", side_effect=crossing):
|
||||
reason = unsnapshotted(self.backing(), self.subject)
|
||||
self.assertIn("filesystem boundary", reason)
|
||||
|
||||
def test_an_unreadable_mountpoint_is_not(self) -> None:
|
||||
reason = unsnapshotted(
|
||||
self.backing(mountpoint=str(Path(self.subject) / "gone")), self.subject
|
||||
)
|
||||
self.assertIn("could not be read", reason)
|
||||
|
||||
|
||||
class TestSnapshotSource(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.subject = tempfile.mkdtemp()
|
||||
self.mountpoint = str(Path(self.subject) / "volumes" / "app" / "_data")
|
||||
Path(self.mountpoint).mkdir(parents=True)
|
||||
self.snapshot = str(
|
||||
Path(self.subject) / ".baudolo-tag" / "volumes" / "app" / "_data"
|
||||
)
|
||||
Path(self.snapshot).mkdir(parents=True)
|
||||
self.backing = Backing(self.mountpoint)
|
||||
|
||||
def test_a_captured_volume_reads_from_the_snapshot(self) -> None:
|
||||
source, reason = snapshot_source(
|
||||
lambda path: self.snapshot + "/", self.backing, self.subject
|
||||
)
|
||||
self.assertEqual(source, self.snapshot + "/")
|
||||
self.assertEqual(reason, "")
|
||||
|
||||
def test_an_uncaptured_volume_is_refused_before_the_resolver_runs(self) -> None:
|
||||
def resolve(path):
|
||||
raise AssertionError("must not resolve a volume the snapshot misses")
|
||||
|
||||
source, reason = snapshot_source(
|
||||
resolve, Backing(self.mountpoint, options={"type": "nfs"}), self.subject
|
||||
)
|
||||
self.assertIsNone(source)
|
||||
self.assertIn("backing store", reason)
|
||||
|
||||
def test_a_volume_outside_the_subject_degrades_instead_of_raising(self) -> None:
|
||||
def resolve(path):
|
||||
raise SnapshotError(f"{path} lies outside the snapshot subject")
|
||||
|
||||
source, reason = snapshot_source(resolve, self.backing, self.subject)
|
||||
self.assertIsNone(source)
|
||||
self.assertIn("lies outside", reason)
|
||||
|
||||
def test_a_volume_created_after_the_snapshot_degrades(self) -> None:
|
||||
source, reason = snapshot_source(
|
||||
lambda path: str(Path(self.subject) / "absent") + "/",
|
||||
self.backing,
|
||||
self.subject,
|
||||
)
|
||||
self.assertIsNone(source)
|
||||
self.assertIn("created after", reason)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -48,7 +48,7 @@ class TestBackupVolume(unittest.TestCase):
|
||||
|
||||
def test_it_keeps_no_twin_of_what_the_second_pass_replaces(self) -> None:
|
||||
command = self.copy(authoritative=True)
|
||||
self.assertIn("rsync -aP ", command)
|
||||
self.assertEqual(command[:2], ["rsync", "-aP"])
|
||||
self.assertNotIn("--backup", command)
|
||||
|
||||
def test_it_creates_the_destination(self) -> None:
|
||||
|
||||
54
tests/unit/restore/test_cli_version_flag.py
Normal file
54
tests/unit/restore/test_cli_version_flag.py
Normal file
@@ -0,0 +1,54 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from baudolo.restore import __main__ as cli
|
||||
|
||||
ENGINES = {
|
||||
"postgres": ("restore_postgres_sql", ["--db-name", "app"]),
|
||||
"mariadb": ("restore_mariadb_sql", ["--db-name", "app"]),
|
||||
"cluster": ("restore_cluster_sql", ["--instance", "central", "--db-user", "root"]),
|
||||
}
|
||||
|
||||
|
||||
class TestVersionFlagReachesEveryEngine(unittest.TestCase):
|
||||
def call(self, engine: str, extra: list) -> dict:
|
||||
target, required = ENGINES[engine]
|
||||
argv = [
|
||||
engine,
|
||||
"app_vol",
|
||||
"hash",
|
||||
"20260817000000",
|
||||
"--repo-name",
|
||||
"repo",
|
||||
"--container",
|
||||
"db",
|
||||
"--db-password",
|
||||
"pw",
|
||||
*required,
|
||||
*extra,
|
||||
]
|
||||
with patch.object(cli, target) as restore:
|
||||
self.assertEqual(cli.main(argv), 0)
|
||||
return restore.call_args.kwargs
|
||||
|
||||
def test_the_gate_is_on_by_default(self) -> None:
|
||||
for engine in ENGINES:
|
||||
with self.subTest(engine=engine):
|
||||
self.assertTrue(self.call(engine, [])["check_version"])
|
||||
|
||||
def test_the_flag_turns_it_off(self) -> None:
|
||||
for engine in ENGINES:
|
||||
with self.subTest(engine=engine):
|
||||
kwargs = self.call(engine, ["--no-version-check"])
|
||||
self.assertFalse(kwargs["check_version"])
|
||||
|
||||
def test_empty_stays_independent_of_the_gate(self) -> None:
|
||||
for engine in ENGINES:
|
||||
with self.subTest(engine=engine):
|
||||
kwargs = self.call(engine, ["--empty"])
|
||||
self.assertTrue(kwargs["empty"])
|
||||
self.assertTrue(kwargs["check_version"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
126
tests/unit/restore/test_cluster_preclean_scope.py
Normal file
126
tests/unit/restore/test_cluster_preclean_scope.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""The pre-clean may only drop what the dump can bring back.
|
||||
|
||||
A shared instance carries databases and roles from other applications, and a
|
||||
database created after the backup is in no dump at all. Dropping those would
|
||||
destroy data this restore cannot restore.
|
||||
"""
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from baudolo.restore.db import cluster as cluster_mod
|
||||
|
||||
DUMP = """--
|
||||
-- PostgreSQL database cluster dump
|
||||
--
|
||||
|
||||
CREATE ROLE app;
|
||||
ALTER ROLE app WITH LOGIN;
|
||||
CREATE ROLE reporting;
|
||||
|
||||
CREATE DATABASE appdb OWNER app;
|
||||
|
||||
\\connect appdb
|
||||
|
||||
CREATE TABLE t (v text);
|
||||
|
||||
\\connect template1
|
||||
"""
|
||||
|
||||
|
||||
def dump_file(text: str) -> str:
|
||||
path = Path(tempfile.mkdtemp()) / "central.cluster.backup.sql"
|
||||
path.write_text(text, encoding="utf-8")
|
||||
return str(path)
|
||||
|
||||
|
||||
class TestDumpInventory(unittest.TestCase):
|
||||
def test_it_reads_databases_and_roles_the_dump_recreates(self) -> None:
|
||||
databases, roles = cluster_mod.dump_inventory(dump_file(DUMP))
|
||||
self.assertEqual(databases, ["appdb", "template1"])
|
||||
self.assertEqual(roles, ["app", "reporting"])
|
||||
|
||||
def test_a_quoted_name_keeps_its_spaces(self) -> None:
|
||||
databases, roles = cluster_mod.dump_inventory(
|
||||
dump_file('\\connect "odd name"\nCREATE ROLE "odd role";\n')
|
||||
)
|
||||
self.assertEqual(databases, ["odd name"])
|
||||
self.assertEqual(roles, ["odd role"])
|
||||
|
||||
def test_psql_options_are_not_mistaken_for_the_database(self) -> None:
|
||||
databases, _roles = cluster_mod.dump_inventory(
|
||||
dump_file("\\connect -reuse-previous=on dbname=appdb\n")
|
||||
)
|
||||
self.assertEqual(databases, ["appdb"])
|
||||
|
||||
def test_create_database_options_are_not_part_of_the_name(self) -> None:
|
||||
databases, _roles = cluster_mod.dump_inventory(
|
||||
dump_file("CREATE DATABASE appdb WITH TEMPLATE = template0 OWNER = app;\n")
|
||||
)
|
||||
self.assertEqual(databases, ["appdb"])
|
||||
|
||||
def test_a_name_is_listed_once(self) -> None:
|
||||
databases, _roles = cluster_mod.dump_inventory(
|
||||
dump_file("\\connect a\n\\connect a\n")
|
||||
)
|
||||
self.assertEqual(databases, ["a"])
|
||||
|
||||
|
||||
class TestInstanceRefusal(unittest.TestCase):
|
||||
"""--empty wipes the whole instance, so it may only run on one this dump
|
||||
can rebuild. Scoping the sweep instead wedges the restore: a surviving
|
||||
database that grants to a dumped role pins it, DROP ROLE fails, and the
|
||||
pre-clean aborts after the dump's own databases are already gone."""
|
||||
|
||||
def check(self, present: str, dump: str = DUMP):
|
||||
from unittest import mock
|
||||
|
||||
with mock.patch.object(
|
||||
cluster_mod, "instance_databases", return_value=present.split()
|
||||
):
|
||||
cluster_mod.assert_instance_matches_dump(
|
||||
"db", "postgres", dump_file(dump), {}
|
||||
)
|
||||
|
||||
def test_an_instance_the_dump_covers_passes(self) -> None:
|
||||
self.check("appdb")
|
||||
|
||||
def test_an_empty_instance_passes(self) -> None:
|
||||
self.check("")
|
||||
|
||||
def test_a_database_the_dump_lacks_is_refused(self) -> None:
|
||||
with self.assertRaises(RuntimeError) as raised:
|
||||
self.check("appdb sibling")
|
||||
self.assertIn("sibling", str(raised.exception))
|
||||
|
||||
def test_the_refusal_names_every_foreign_database(self) -> None:
|
||||
with self.assertRaises(RuntimeError) as raised:
|
||||
self.check("one two")
|
||||
message = str(raised.exception)
|
||||
self.assertIn("one", message)
|
||||
self.assertIn("two", message)
|
||||
|
||||
def test_the_refusal_happens_before_anything_is_dropped(self) -> None:
|
||||
from unittest import mock
|
||||
|
||||
with (
|
||||
mock.patch.object(
|
||||
cluster_mod, "instance_databases", return_value=["foreign"]
|
||||
),
|
||||
mock.patch.object(cluster_mod, "docker_exec") as touched,
|
||||
self.assertRaises(RuntimeError),
|
||||
):
|
||||
cluster_mod.restore_cluster_sql(
|
||||
container="db",
|
||||
user="postgres",
|
||||
password="pw",
|
||||
sql_path=dump_file(DUMP),
|
||||
empty=True,
|
||||
check_version=False,
|
||||
)
|
||||
touched.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
125
tests/unit/restore/test_cluster_replay.py
Normal file
125
tests/unit/restore/test_cluster_replay.py
Normal file
@@ -0,0 +1,125 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from baudolo.restore.db import cluster as cluster_mod
|
||||
from baudolo.restore.paths import BackupPaths
|
||||
|
||||
|
||||
class TestClusterReplay(unittest.TestCase):
|
||||
def _replay(self, *, empty: bool):
|
||||
calls = []
|
||||
|
||||
def _capture(container, argv, **kwargs):
|
||||
if "-tAc" in argv:
|
||||
return MagicMock(stdout=b"")
|
||||
calls.append((argv, kwargs.get("stdin")))
|
||||
return MagicMock()
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".sql") as sql:
|
||||
sql.write(b"CREATE ROLE app;\nCREATE DATABASE app OWNER app;\n")
|
||||
sql.flush()
|
||||
with patch.object(cluster_mod, "docker_exec", side_effect=_capture):
|
||||
cluster_mod.restore_cluster_sql(
|
||||
container="db",
|
||||
user="postgres",
|
||||
password="pw",
|
||||
sql_path=sql.name,
|
||||
empty=empty,
|
||||
check_version=False,
|
||||
)
|
||||
return calls
|
||||
|
||||
def test_the_replay_is_not_wrapped_in_a_transaction(self) -> None:
|
||||
argv, _ = self._replay(empty=False)[0]
|
||||
self.assertNotIn(
|
||||
"--single-transaction",
|
||||
argv,
|
||||
"CREATE DATABASE cannot run inside a transaction block, so unlike the "
|
||||
"single-database replay this stream must not be wrapped in one",
|
||||
)
|
||||
self.assertIn("ON_ERROR_STOP=1", argv)
|
||||
|
||||
def test_the_replay_targets_the_control_database(self) -> None:
|
||||
argv, _ = self._replay(empty=False)[0]
|
||||
self.assertEqual(argv[argv.index("-d") + 1], cluster_mod.CONTROL_DB)
|
||||
self.assertEqual(argv[argv.index("-U") + 1], "postgres")
|
||||
|
||||
def test_without_empty_nothing_is_dropped_first(self) -> None:
|
||||
self.assertEqual(len(self._replay(empty=False)), 1)
|
||||
|
||||
def test_empty_drops_databases_before_their_owners(self) -> None:
|
||||
calls = self._replay(empty=True)
|
||||
self.assertEqual(len(calls), 2, f"expected pre-clean + replay: {calls}")
|
||||
preclean = calls[0][1].decode()
|
||||
self.assertLess(
|
||||
preclean.index("DROP DATABASE"),
|
||||
preclean.index("DROP ROLE"),
|
||||
"a role cannot be dropped while it still owns a database",
|
||||
)
|
||||
self.assertIn("DROP OWNED BY", preclean)
|
||||
self.assertIn("ORDER BY phase", preclean)
|
||||
|
||||
def test_the_preclean_spares_what_no_dump_recreates(self) -> None:
|
||||
preclean = self._replay(empty=True)[0][1].decode()
|
||||
self.assertIn("NOT datistemplate", preclean)
|
||||
self.assertIn("datname <> current_database()", preclean)
|
||||
self.assertIn("starts_with(rolname, 'pg_')", preclean)
|
||||
self.assertIn("rolname <> current_user", preclean)
|
||||
|
||||
def test_only_the_connecting_role_loses_its_create(self) -> None:
|
||||
# Captured from pg_dumpall 17.
|
||||
dump = [
|
||||
b"CREATE ROLE app;\n",
|
||||
b"ALTER ROLE app WITH NOSUPERUSER INHERIT LOGIN PASSWORD 'SCRAM-SHA-256$...';\n",
|
||||
b"CREATE ROLE postgres;\n",
|
||||
b"ALTER ROLE postgres WITH SUPERUSER INHERIT LOGIN PASSWORD 'SCRAM-SHA-256$...';\n",
|
||||
b'CREATE ROLE "odd-name";\n',
|
||||
]
|
||||
kept = list(cluster_mod.filter_own_role_creation(dump, "postgres"))
|
||||
self.assertNotIn(b"CREATE ROLE postgres;\n", kept)
|
||||
self.assertIn(b"CREATE ROLE app;\n", kept)
|
||||
self.assertIn(b'CREATE ROLE "odd-name";\n', kept)
|
||||
self.assertEqual(
|
||||
sum(1 for line in kept if line.startswith(b"ALTER ROLE postgres")),
|
||||
1,
|
||||
"the ALTER re-applies the superuser's attributes and password",
|
||||
)
|
||||
|
||||
def test_a_quoted_connecting_role_is_matched_too(self) -> None:
|
||||
kept = list(
|
||||
cluster_mod.filter_own_role_creation(
|
||||
[b'CREATE ROLE "odd-name";\n'], "odd-name"
|
||||
)
|
||||
)
|
||||
self.assertEqual(kept, [])
|
||||
|
||||
def test_a_role_whose_name_merely_starts_the_same_is_kept(self) -> None:
|
||||
kept = list(
|
||||
cluster_mod.filter_own_role_creation(
|
||||
[b"CREATE ROLE postgresql;\n"], "postgres"
|
||||
)
|
||||
)
|
||||
self.assertEqual(kept, [b"CREATE ROLE postgresql;\n"])
|
||||
|
||||
def test_a_missing_dump_is_reported_as_such(self) -> None:
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
cluster_mod.restore_cluster_sql(
|
||||
container="db",
|
||||
user="postgres",
|
||||
password="pw",
|
||||
sql_path="/nonexistent/x.cluster.backup.sql",
|
||||
empty=False,
|
||||
check_version=False,
|
||||
)
|
||||
|
||||
def test_the_path_helper_names_the_dumpall_file(self) -> None:
|
||||
paths = BackupPaths("vol", "hash", "v1", repo_name="repo", backups_dir="/B")
|
||||
self.assertEqual(
|
||||
paths.cluster_file("bigbluebutton"),
|
||||
"/B/hash/repo/v1/vol/sql/bigbluebutton.cluster.backup.sql",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
62
tests/unit/restore/test_files_backing_store.py
Normal file
62
tests/unit/restore/test_files_backing_store.py
Normal file
@@ -0,0 +1,62 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from baudolo.restore import files as files_mod
|
||||
|
||||
|
||||
class TestBackingStoreGuard(unittest.TestCase):
|
||||
def restore(self, inspect: str, mounted: bool) -> tuple[int, list]:
|
||||
calls = []
|
||||
|
||||
def _run(cmd, **kwargs):
|
||||
calls.append(cmd)
|
||||
return MagicMock(stdout=inspect.encode())
|
||||
|
||||
with (
|
||||
patch.object(files_mod, "docker_volume_exists", return_value=True),
|
||||
patch.object(files_mod.os.path, "ismount", return_value=mounted),
|
||||
patch.object(files_mod, "run", side_effect=_run),
|
||||
):
|
||||
code = files_mod.restore_volume_files("app_data", tempfile.mkdtemp())
|
||||
return code, calls
|
||||
|
||||
def rsynced(self, calls: list) -> bool:
|
||||
return any(cmd[0] == "rsync" for cmd in calls)
|
||||
|
||||
def test_plain_local_volume_is_restored_unmounted(self) -> None:
|
||||
code, calls = self.restore("/var/lib/docker/volumes/a/_data|local|plain", False)
|
||||
self.assertEqual(code, 0)
|
||||
self.assertTrue(self.rsynced(calls))
|
||||
|
||||
def test_volume_with_driver_options_is_refused_while_unmounted(self) -> None:
|
||||
code, calls = self.restore("/var/lib/docker/volumes/a/_data|local|opts", False)
|
||||
self.assertEqual(code, 2)
|
||||
self.assertFalse(
|
||||
self.rsynced(calls),
|
||||
"an NFS or bind volume writes under the mount and reports success",
|
||||
)
|
||||
|
||||
def test_volume_with_driver_options_is_restored_once_mounted(self) -> None:
|
||||
code, calls = self.restore("/var/lib/docker/volumes/a/_data|local|opts", True)
|
||||
self.assertEqual(code, 0)
|
||||
self.assertTrue(self.rsynced(calls))
|
||||
|
||||
def test_foreign_driver_is_refused_while_unmounted(self) -> None:
|
||||
code, calls = self.restore("/mnt/gluster/a|glusterfs|plain", False)
|
||||
self.assertEqual(code, 2)
|
||||
self.assertFalse(self.rsynced(calls))
|
||||
|
||||
def test_an_unresolvable_mountpoint_still_fails_first(self) -> None:
|
||||
code, calls = self.restore("|local|plain", False)
|
||||
self.assertEqual(code, 2)
|
||||
self.assertFalse(self.rsynced(calls))
|
||||
|
||||
def test_a_format_without_the_new_fields_is_treated_as_plain(self) -> None:
|
||||
code, calls = self.restore("/var/lib/docker/volumes/a/_data", False)
|
||||
self.assertEqual(code, 0)
|
||||
self.assertTrue(self.rsynced(calls))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -29,6 +29,7 @@ class TestMariadbEmptyDrop(unittest.TestCase):
|
||||
password="pw",
|
||||
sql_path=sql.name,
|
||||
empty=True,
|
||||
check_version=False,
|
||||
)
|
||||
|
||||
drop_calls = [argv for argv in calls if any("DROP TABLE" in a for a in argv)]
|
||||
|
||||
@@ -24,6 +24,7 @@ class TestPostgresSingleTransaction(unittest.TestCase):
|
||||
password="pw",
|
||||
sql_path=sql.name,
|
||||
empty=True,
|
||||
check_version=False,
|
||||
)
|
||||
|
||||
self.assertEqual(len(calls), 2, f"expected pre-clean + replay: {calls}")
|
||||
|
||||
244
tests/unit/restore/test_version_gate.py
Normal file
244
tests/unit/restore/test_version_gate.py
Normal file
@@ -0,0 +1,244 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from baudolo.restore.db import cluster as cluster_mod
|
||||
from baudolo.restore.db import mariadb as mdb_mod
|
||||
from baudolo.restore.db import postgres as pg_mod
|
||||
from baudolo.restore.db import version as ver
|
||||
|
||||
POSTGRES_HEADER = """--
|
||||
-- PostgreSQL database dump
|
||||
--
|
||||
|
||||
\\restrict BbyzwODc1rWKL3rDyLhEjgCF0Kf2TU5ma7gcTs8eQI7copLtydXkc61zdULsPav
|
||||
|
||||
-- Dumped from database version 17.11
|
||||
-- Dumped by pg_dump version 17.11
|
||||
|
||||
SET statement_timeout = 0;
|
||||
"""
|
||||
|
||||
MARIADB_HEADER = """/*M!999999\\- enable the sandbox mode */
|
||||
-- MariaDB dump 10.19-11.8.8-MariaDB, for debian-linux-gnu (x86_64)
|
||||
--
|
||||
-- Host: 127.0.0.1 Database: mysql
|
||||
-- ------------------------------------------------------
|
||||
-- Server version\t11.8.8-MariaDB-ubu2404
|
||||
|
||||
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
|
||||
"""
|
||||
|
||||
|
||||
def cluster_header(roles: int) -> str:
|
||||
"""A pg_dumpall stream: banner, N roles, then the first database's dump."""
|
||||
head = [
|
||||
"--",
|
||||
"-- PostgreSQL database cluster dump",
|
||||
"--",
|
||||
"",
|
||||
"SET default_transaction_read_only = off;",
|
||||
"",
|
||||
"--",
|
||||
"-- Roles",
|
||||
"--",
|
||||
]
|
||||
for i in range(roles):
|
||||
head.append(f'CREATE ROLE "app{i}";')
|
||||
head.append(f'ALTER ROLE "app{i}" WITH NOSUPERUSER INHERIT LOGIN;')
|
||||
head.append("\\connect app")
|
||||
head.append("")
|
||||
return "\n".join(head) + "\n" + POSTGRES_HEADER
|
||||
|
||||
|
||||
def dump_file(text: str) -> str:
|
||||
path = Path(tempfile.mkdtemp()) / "app.backup.sql"
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
handle.write(text)
|
||||
return str(path)
|
||||
|
||||
|
||||
class TestDumpVersion(unittest.TestCase):
|
||||
"""Headers captured from postgres:17-alpine and mariadb:11 themselves."""
|
||||
|
||||
def test_postgres_reads_the_source_server(self) -> None:
|
||||
self.assertEqual(
|
||||
ver.dump_version(dump_file(POSTGRES_HEADER), "postgres"), "17.11"
|
||||
)
|
||||
|
||||
def test_mariadb_reads_the_server_not_the_dump_tool(self) -> None:
|
||||
found = ver.dump_version(dump_file(MARIADB_HEADER), "mariadb")
|
||||
self.assertEqual(found, "11.8.8-MariaDB-ubu2404")
|
||||
self.assertNotEqual(
|
||||
ver.major_of(found),
|
||||
10,
|
||||
"10.19 is mariadb-dump's own version, not the server's",
|
||||
)
|
||||
|
||||
def test_cluster_dump_states_its_version_far_below_the_header(self) -> None:
|
||||
path = dump_file(cluster_header(roles=200))
|
||||
with Path(path).open(encoding="utf-8") as handle:
|
||||
offset = next(i for i, line in enumerate(handle) if "Dumped from" in line)
|
||||
self.assertGreater(offset, 100, "fixture must exercise the deep scan")
|
||||
self.assertEqual(ver.dump_version(path, "postgres"), "17.11")
|
||||
|
||||
def test_a_version_beyond_the_scan_limit_is_refused_not_ignored(self) -> None:
|
||||
path = dump_file(cluster_header(roles=ver.SCAN_LINES))
|
||||
with self.assertRaises(ver.VersionMismatchError):
|
||||
ver.dump_version(path, "postgres")
|
||||
|
||||
def test_a_dump_without_a_version_header_is_refused(self) -> None:
|
||||
path = dump_file("CREATE TABLE t (id int);\n")
|
||||
with self.assertRaises(ver.VersionMismatchError):
|
||||
ver.dump_version(path, "postgres")
|
||||
|
||||
|
||||
class TestMajorOf(unittest.TestCase):
|
||||
def test_reads_the_leading_number(self) -> None:
|
||||
self.assertEqual(ver.major_of("17.11"), 17)
|
||||
self.assertEqual(ver.major_of("11.8.8-MariaDB-ubu2404"), 11)
|
||||
self.assertEqual(ver.major_of("9.6.24"), 9)
|
||||
self.assertEqual(ver.major_of("18beta1"), 18)
|
||||
|
||||
def test_refuses_an_unreadable_version(self) -> None:
|
||||
with self.assertRaises(ver.VersionMismatchError):
|
||||
ver.major_of("unknown")
|
||||
|
||||
|
||||
class TestAssertReplayable(unittest.TestCase):
|
||||
def test_newer_dump_into_older_engine_is_refused(self) -> None:
|
||||
with self.assertRaises(ver.VersionMismatchError) as caught:
|
||||
ver.assert_replayable("/b/app.sql", "postgres", "17.11", "15.6")
|
||||
self.assertIn("17.11", str(caught.exception))
|
||||
self.assertIn("15.6", str(caught.exception))
|
||||
|
||||
def test_same_major_passes(self) -> None:
|
||||
ver.assert_replayable("/b/app.sql", "postgres", "17.4", "17.11")
|
||||
|
||||
def test_older_dump_into_newer_engine_passes(self) -> None:
|
||||
ver.assert_replayable("/b/app.sql", "postgres", "15.6", "17.11")
|
||||
|
||||
|
||||
class TestServerVersion(unittest.TestCase):
|
||||
def test_postgres_asks_over_pgpassword(self) -> None:
|
||||
with patch.object(ver, "docker_exec") as exec_:
|
||||
exec_.return_value = MagicMock(stdout=b" 17.11 \n")
|
||||
found = ver.server_version("db", "postgres", "app", "pw")
|
||||
self.assertEqual(found, "17.11")
|
||||
argv = exec_.call_args.args[1]
|
||||
self.assertIn("SHOW server_version", argv)
|
||||
self.assertEqual(exec_.call_args.kwargs["docker_env"], {"PGPASSWORD": "pw"})
|
||||
|
||||
def test_mariadb_asks_through_the_detected_client(self) -> None:
|
||||
with patch.object(ver, "docker_exec") as exec_:
|
||||
exec_.return_value = MagicMock(stdout=b"11.8.8-MariaDB-ubu2404\n")
|
||||
found = ver.server_version("db", "mariadb", "app", "pw", client="mysql")
|
||||
self.assertEqual(found, "11.8.8-MariaDB-ubu2404")
|
||||
self.assertEqual(exec_.call_args.args[1][0], "mysql")
|
||||
|
||||
|
||||
class TestGateStopsBeforeDestroying(unittest.TestCase):
|
||||
"""--empty drops in one session and replays in the next, with no rollback
|
||||
between them, so the refusal has to land before the first session."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self.serving = patch.object(ver, "docker_exec").start()
|
||||
self.addCleanup(patch.stopall)
|
||||
|
||||
def serve(self, version: str) -> None:
|
||||
self.serving.return_value = MagicMock(stdout=version.encode())
|
||||
|
||||
def test_postgres_refuses_without_running_the_preclean(self) -> None:
|
||||
self.serve("15.6")
|
||||
path = dump_file(POSTGRES_HEADER)
|
||||
with (
|
||||
patch.object(pg_mod, "docker_exec") as replay,
|
||||
self.assertRaises(ver.VersionMismatchError),
|
||||
):
|
||||
pg_mod.restore_postgres_sql(
|
||||
container="db",
|
||||
db_name="app",
|
||||
user="app",
|
||||
password="pw",
|
||||
sql_path=path,
|
||||
empty=True,
|
||||
)
|
||||
replay.assert_not_called()
|
||||
|
||||
def test_cluster_refuses_without_running_the_preclean(self) -> None:
|
||||
self.serve("15.6")
|
||||
path = dump_file(cluster_header(roles=3))
|
||||
with (
|
||||
patch.object(cluster_mod, "docker_exec") as replay,
|
||||
self.assertRaises(ver.VersionMismatchError),
|
||||
):
|
||||
cluster_mod.restore_cluster_sql(
|
||||
container="db",
|
||||
user="postgres",
|
||||
password="pw",
|
||||
sql_path=path,
|
||||
empty=True,
|
||||
)
|
||||
replay.assert_not_called()
|
||||
|
||||
def test_mariadb_refuses_without_dropping_tables(self) -> None:
|
||||
self.serve("10.11.6-MariaDB")
|
||||
path = dump_file(MARIADB_HEADER)
|
||||
with (
|
||||
patch.object(mdb_mod, "_pick_client", return_value="mariadb"),
|
||||
patch.object(mdb_mod, "docker_exec") as replay,
|
||||
self.assertRaises(ver.VersionMismatchError),
|
||||
):
|
||||
mdb_mod.restore_mariadb_sql(
|
||||
container="db",
|
||||
db_name="app",
|
||||
user="app",
|
||||
password="pw",
|
||||
sql_path=path,
|
||||
empty=True,
|
||||
)
|
||||
replay.assert_not_called()
|
||||
|
||||
def test_matching_versions_let_the_replay_through(self) -> None:
|
||||
self.serve("17.11")
|
||||
path = dump_file(POSTGRES_HEADER)
|
||||
with patch.object(pg_mod, "docker_exec") as replay:
|
||||
pg_mod.restore_postgres_sql(
|
||||
container="db",
|
||||
db_name="app",
|
||||
user="app",
|
||||
password="pw",
|
||||
sql_path=path,
|
||||
empty=False,
|
||||
)
|
||||
replay.assert_called_once()
|
||||
|
||||
def test_the_escape_hatch_asks_the_engine_nothing(self) -> None:
|
||||
path = dump_file("CREATE TABLE t (id int);\n")
|
||||
with patch.object(pg_mod, "docker_exec"):
|
||||
pg_mod.restore_postgres_sql(
|
||||
container="db",
|
||||
db_name="app",
|
||||
user="app",
|
||||
password="pw",
|
||||
sql_path=path,
|
||||
empty=False,
|
||||
check_version=False,
|
||||
)
|
||||
self.serving.assert_not_called()
|
||||
|
||||
def test_a_missing_dump_is_reported_as_missing_not_as_a_mismatch(self) -> None:
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
pg_mod.restore_postgres_sql(
|
||||
container="db",
|
||||
db_name="app",
|
||||
user="app",
|
||||
password="pw",
|
||||
sql_path=str(Path(tempfile.mkdtemp()) / "absent.sql"),
|
||||
empty=True,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -17,20 +17,15 @@ class TestSeedMain(unittest.TestCase):
|
||||
columns=["instance", "database", "username", "password"]
|
||||
)
|
||||
|
||||
def test_validate_database_value_rejects_empty(self) -> None:
|
||||
def test_a_rejected_database_never_reaches_the_file(self) -> None:
|
||||
with self.assertRaises(ValueError):
|
||||
seed_main._validate_database_value("", instance="x")
|
||||
|
||||
def test_validate_database_value_accepts_star(self) -> None:
|
||||
self.assertEqual(seed_main._validate_database_value("*", instance="x"), "*")
|
||||
|
||||
def test_validate_database_value_rejects_nan(self) -> None:
|
||||
with self.assertRaises(ValueError):
|
||||
seed_main._validate_database_value("nan", instance="x")
|
||||
|
||||
def test_validate_database_value_rejects_invalid_name(self) -> None:
|
||||
with self.assertRaises(ValueError):
|
||||
seed_main._validate_database_value("bad name", instance="x")
|
||||
seed_main.check_and_add_entry(
|
||||
file_path="/nonexistent/databases.csv",
|
||||
instance="x",
|
||||
database="bad name",
|
||||
username="u",
|
||||
password="p",
|
||||
)
|
||||
|
||||
def _mock_df_mask_any(self, *, any_value: bool) -> MagicMock:
|
||||
"""
|
||||
@@ -56,7 +51,7 @@ class TestSeedMain(unittest.TestCase):
|
||||
|
||||
return df
|
||||
|
||||
@patch("baudolo.seed.__main__.os.path.exists", return_value=False)
|
||||
@patch("baudolo.seed.__main__.Path.exists", return_value=False)
|
||||
@patch("baudolo.seed.__main__.pd.read_csv")
|
||||
@patch("baudolo.seed.__main__._empty_df")
|
||||
@patch("baudolo.seed.__main__.pd.concat")
|
||||
@@ -88,7 +83,7 @@ class TestSeedMain(unittest.TestCase):
|
||||
"/tmp/databases.csv", sep=";", index=False
|
||||
)
|
||||
|
||||
@patch("baudolo.seed.__main__.os.path.exists", return_value=True)
|
||||
@patch("baudolo.seed.__main__.Path.exists", return_value=True)
|
||||
@patch("baudolo.seed.__main__.pd.read_csv", side_effect=EmptyDataError("empty"))
|
||||
@patch("baudolo.seed.__main__._empty_df")
|
||||
@patch("baudolo.seed.__main__.pd.concat")
|
||||
@@ -115,8 +110,8 @@ class TestSeedMain(unittest.TestCase):
|
||||
password="pass",
|
||||
)
|
||||
|
||||
exists.assert_called_once_with("/tmp/databases.csv")
|
||||
read_csv.assert_called_once()
|
||||
exists.assert_called_once_with()
|
||||
self.assertEqual(read_csv.call_args.args, ("/tmp/databases.csv",))
|
||||
empty_df.assert_called_once()
|
||||
concat.assert_called_once()
|
||||
|
||||
@@ -131,7 +126,6 @@ class TestSeedMain(unittest.TestCase):
|
||||
warning_calls,
|
||||
"Expected a WARNING print when databases.csv is empty, but none was found.",
|
||||
)
|
||||
# Ensure the warning goes to stderr
|
||||
_, warn_kwargs = warning_calls[0]
|
||||
self.assertEqual(warn_kwargs.get("file"), seed_main.sys.stderr)
|
||||
|
||||
@@ -139,7 +133,7 @@ class TestSeedMain(unittest.TestCase):
|
||||
"/tmp/databases.csv", sep=";", index=False
|
||||
)
|
||||
|
||||
@patch("baudolo.seed.__main__.os.path.exists", return_value=True)
|
||||
@patch("baudolo.seed.__main__.Path.exists", return_value=True)
|
||||
@patch("baudolo.seed.__main__.pd.read_csv")
|
||||
def test_check_and_add_entry_updates_existing_row(
|
||||
self,
|
||||
|
||||
84
tests/unit/test_databases.py
Normal file
84
tests/unit/test_databases.py
Normal file
@@ -0,0 +1,84 @@
|
||||
"""Contract of databases.csv: the seed writes it, the backup and a restore read it."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from baudolo.databases import (
|
||||
CLUSTER_ROW,
|
||||
COLUMNS,
|
||||
DELIMITER,
|
||||
DatabasesCsvError,
|
||||
Row,
|
||||
read_rows,
|
||||
validate_database,
|
||||
)
|
||||
|
||||
HEADER = DELIMITER.join(COLUMNS)
|
||||
|
||||
|
||||
def _csv(*lines: str) -> str:
|
||||
path = Path(tempfile.mkdtemp()) / "databases.csv"
|
||||
path.write_text("\n".join((HEADER, *lines)) + "\n", encoding="utf-8")
|
||||
return str(path)
|
||||
|
||||
|
||||
class TestValidateDatabase(unittest.TestCase):
|
||||
def test_a_concrete_name_passes(self) -> None:
|
||||
self.assertEqual(validate_database("app_db-1", instance="x"), "app_db-1")
|
||||
|
||||
def test_the_cluster_marker_passes(self) -> None:
|
||||
self.assertEqual(validate_database(CLUSTER_ROW, instance="x"), CLUSTER_ROW)
|
||||
|
||||
def test_an_empty_column_is_rejected(self) -> None:
|
||||
with self.assertRaises(DatabasesCsvError):
|
||||
validate_database("", instance="x")
|
||||
|
||||
def test_the_string_nan_is_rejected(self) -> None:
|
||||
"""pandas used to hand back NaN, which wrote a nan.backup.sql."""
|
||||
with self.assertRaises(DatabasesCsvError):
|
||||
validate_database("nan", instance="x")
|
||||
|
||||
def test_a_name_that_could_reach_a_shell_is_rejected(self) -> None:
|
||||
for hostile in ("bad name", "a;rm -rf /", "$(id)", "a`id`", "a/b"):
|
||||
with self.subTest(name=hostile), self.assertRaises(DatabasesCsvError):
|
||||
validate_database(hostile, instance="x")
|
||||
|
||||
def test_the_error_is_a_value_error(self) -> None:
|
||||
"""Callers predating the shared module catch ValueError."""
|
||||
with self.assertRaises(ValueError):
|
||||
validate_database("", instance="x")
|
||||
|
||||
|
||||
class TestReadRows(unittest.TestCase):
|
||||
def test_the_header_is_skipped(self) -> None:
|
||||
rows = read_rows(_csv(f"pg{DELIMITER}app{DELIMITER}u{DELIMITER}p"))
|
||||
self.assertEqual(rows, [Row("pg", "app", "u", "p")])
|
||||
|
||||
def test_a_blank_row_is_dropped(self) -> None:
|
||||
rows = read_rows(_csv("", f"pg{DELIMITER}app{DELIMITER}u{DELIMITER}p", ""))
|
||||
self.assertEqual(len(rows), 1)
|
||||
|
||||
def test_a_short_row_is_refused(self) -> None:
|
||||
with self.assertRaises(DatabasesCsvError):
|
||||
read_rows(_csv(f"pg{DELIMITER}app{DELIMITER}u"))
|
||||
|
||||
def test_values_arrive_verbatim(self) -> None:
|
||||
"""A password may legitimately begin or end with a space."""
|
||||
rows = read_rows(_csv(f"pg{DELIMITER}app{DELIMITER}u{DELIMITER} pw "))
|
||||
self.assertEqual(rows[0].password, " pw ")
|
||||
|
||||
def test_a_cluster_row_knows_itself(self) -> None:
|
||||
rows = read_rows(
|
||||
_csv(
|
||||
f"pg{DELIMITER}{CLUSTER_ROW}{DELIMITER}postgres{DELIMITER}p",
|
||||
f"pg{DELIMITER}app{DELIMITER}u{DELIMITER}p",
|
||||
)
|
||||
)
|
||||
self.assertEqual([row.is_cluster for row in rows], [True, False])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
65
tests/unit/test_generation.py
Normal file
65
tests/unit/test_generation.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""Contract of the generation manifest document and the file it lands in."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from baudolo.backup.dumps import VolumeOutcome
|
||||
from baudolo.backup.layout import write_manifest
|
||||
from baudolo.generation import (
|
||||
CLUSTER_SUFFIX,
|
||||
DUMP_SUFFIX,
|
||||
FILES_DIR,
|
||||
MANIFEST_FILE,
|
||||
MANIFEST_SCHEMA,
|
||||
SQL_DIR,
|
||||
manifest_document,
|
||||
)
|
||||
|
||||
|
||||
class TestManifestDocument(unittest.TestCase):
|
||||
def test_it_states_the_layout_a_reader_needs(self) -> None:
|
||||
document = manifest_document({})
|
||||
self.assertEqual(
|
||||
document["layout"],
|
||||
{
|
||||
"files_dir": FILES_DIR,
|
||||
"sql_dir": SQL_DIR,
|
||||
"dump_suffix": DUMP_SUFFIX,
|
||||
"cluster_suffix": CLUSTER_SUFFIX,
|
||||
},
|
||||
)
|
||||
|
||||
def test_it_carries_a_schema_so_a_reader_can_refuse_a_newer_one(self) -> None:
|
||||
self.assertEqual(manifest_document({})["schema"], MANIFEST_SCHEMA)
|
||||
|
||||
def test_it_sorts_volumes_so_two_runs_produce_the_same_bytes(self) -> None:
|
||||
state = VolumeOutcome(database=False, dumped=False)
|
||||
document = manifest_document({"b": state, "a": state})
|
||||
self.assertEqual(list(document["volumes"]), ["a", "b"])
|
||||
|
||||
|
||||
class TestWriteManifest(unittest.TestCase):
|
||||
def test_it_writes_readable_json_next_to_the_volumes(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as version_dir:
|
||||
path = write_manifest(
|
||||
version_dir,
|
||||
{
|
||||
"pgdata": VolumeOutcome(
|
||||
database=True, dumped=False, engine="postgres"
|
||||
)
|
||||
},
|
||||
)
|
||||
self.assertEqual(Path(path).name, MANIFEST_FILE)
|
||||
document = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
self.assertEqual(
|
||||
document["volumes"]["pgdata"],
|
||||
{"database": True, "dumped": False, "engine": "postgres"},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user