mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 02:54:26 +00:00
A databases.csv row with database = '*' makes the backup dump a whole instance with pg_dumpall. The recover chain had no path for that file: it sorted cluster dumps out and printed SKIP, so a host whose engine is dumped that way could not be restored at all, and the drill refused to certify it. Cluster dumps now go back through 'baudolo-restore cluster' with the instance's superuser. Three details decide the shape. A cluster row states '*' where a single-database row names a database, so credentials_of would file every instance under one key and keep the first, hence a lookup keyed by instance. Which applications must be down is stated by the dump's own connect lines and nowhere else, and reading them needs a real identifier parser: a quoted name may hold spaces and psql options precede the target. And the consumers of a cluster sit in the compose project of its engine, not in projects named after each database, so both are checked before anything is dropped. The version gate moves out. baudolo compares the dump's header against the running server before its own pre-clean, so the copy here was a second implementation of the same rule, free to drift from the one that actually gates the replay. The test asserts the delegation negatively: the replay must not query the engine version itself. The pin follows, and baudolo-restore joins the command assertion. The docker probing splits into utils/recovery/docker.py, which keeps the replay module under the size limit and gives the two halves one job each. Calls go through the module rather than through imported names, or a test patching the new module would not reach them. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
136 lines
4.4 KiB
Python
136 lines
4.4 KiB
Python
"""Ask the docker daemon what a replay is allowed to touch.
|
|
|
|
Split out of :mod:`utils.recovery.databases` so the replay module carries the
|
|
generation's layout and the credentials, and this one the probing: which
|
|
container serves a volume, which compose project it belongs to, and who is
|
|
still running in it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import subprocess
|
|
|
|
DOCKER_BIN = "docker"
|
|
|
|
|
|
class RecoveryError(Exception):
|
|
"""A condition that makes the replay unprovable."""
|
|
|
|
|
|
def _docker(argv: list[str], docker_host: str | None) -> list[str]:
|
|
return [DOCKER_BIN, *(["-H", docker_host] if docker_host else []), *argv]
|
|
|
|
|
|
def _run(argv: list[str], secret: str = "") -> str:
|
|
"""Run a command, aborting the replay when it fails.
|
|
|
|
Args:
|
|
argv: the command.
|
|
secret: a value to redact from the error message.
|
|
"""
|
|
try:
|
|
result = subprocess.run(argv, capture_output=True, text=True, check=False)
|
|
except FileNotFoundError as missing:
|
|
raise RecoveryError(f"{argv[0]} is not on PATH") from missing
|
|
if result.returncode != 0:
|
|
shown = " ".join(argv)
|
|
if secret:
|
|
shown = shown.replace(secret, "***")
|
|
raise RecoveryError(
|
|
f"command failed ({result.returncode}): {shown}\n{result.stderr.strip()}"
|
|
)
|
|
return result.stdout
|
|
|
|
|
|
def container_of_volume(volume: str, docker_host: str | None = None) -> str:
|
|
"""Name the running container that mounts a volume.
|
|
|
|
Raises:
|
|
RecoveryError: nothing is running to replay into. A dump reaches its
|
|
database through ``docker exec``, so unlike a file tree it cannot
|
|
be restored onto a bare host - the engine has to be up. The two
|
|
ways to get there differ, so they are reported apart.
|
|
"""
|
|
listed = _docker(
|
|
["ps", "--filter", f"volume={volume}", "--format", "{{.Names}}"], docker_host
|
|
)
|
|
running = _run(listed).split()
|
|
if running:
|
|
return running[0]
|
|
known = _run(
|
|
_docker(
|
|
["ps", "-a", "--filter", f"volume={volume}", "--format", "{{.Names}}"],
|
|
docker_host,
|
|
)
|
|
).split()
|
|
if known:
|
|
raise RecoveryError(
|
|
f"{known[0]} mounts volume {volume} but is not running; the dump is "
|
|
"replayed through docker exec, so start that container first"
|
|
)
|
|
raise RecoveryError(
|
|
f"no container mounts volume {volume} on this host; a dump can only be "
|
|
"replayed into a running database service, so deploy the stack first "
|
|
"and recover with the consumers stopped"
|
|
)
|
|
|
|
|
|
def consumers_running(project: str, docker_host: str | None = None) -> list[str]:
|
|
"""List the running containers of one compose project."""
|
|
return _run(
|
|
_docker(
|
|
[
|
|
"ps",
|
|
"--filter",
|
|
f"label=com.docker.compose.project={project}",
|
|
"--format",
|
|
"{{.Names}}",
|
|
],
|
|
docker_host,
|
|
)
|
|
).split()
|
|
|
|
|
|
def project_of(container: str, docker_host: str | None = None) -> str:
|
|
"""The compose project a container belongs to, empty when it has none."""
|
|
return _run(
|
|
_docker(
|
|
[
|
|
"inspect",
|
|
"--type",
|
|
"container",
|
|
"-f",
|
|
'{{index .Config.Labels "com.docker.compose.project"}}',
|
|
container,
|
|
],
|
|
docker_host,
|
|
)
|
|
).strip()
|
|
|
|
|
|
def assert_no_consumers(
|
|
project: str, docker_host: str | None = None, ignore: tuple[str, ...] = ()
|
|
) -> None:
|
|
"""Refuse to replay into a database whose application is still up.
|
|
|
|
Args:
|
|
project: the compose project to check.
|
|
docker_host: remote docker endpoint, or None for this host.
|
|
ignore: containers that are not consumers - the engine itself, when
|
|
the project checked is the one the engine lives in.
|
|
|
|
Raises:
|
|
RecoveryError: the compose project still runs containers. A booting
|
|
consumer recreates the pre-cleaned schema under the replay.
|
|
"""
|
|
if not project:
|
|
return
|
|
still_up = [
|
|
name for name in consumers_running(project, docker_host) if name not in ignore
|
|
]
|
|
if still_up:
|
|
raise RecoveryError(
|
|
f"project '{project}' still runs {', '.join(still_up)}; a booting "
|
|
"consumer recreates the pre-cleaned schema under the replay, so stop it first"
|
|
)
|