mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2026-08-20 13:12:48 +00:00
fix(backup)!: dump the container that holds the database, with its password
Two defects kept dedicated Postgres databases out of the backup. docker exec never forwarded PGPASSWORD. execute_to_file set it on baudolo's own process, but nothing carried it across the container boundary, so an engine whose pg_hba demands a password on TCP loopback refused the dump. forward_env passes a bare `-e NAME`, letting docker copy the value out of this process's environment instead of spelling it into argv, where the host's process list would publish it. get_instance returned the container name unchanged when that name carried no database token, claiming an instance it had never derived. An application container therefore answered the same databases.csv row as its own dedicated engine, and application images often ship the engine's client tools, so the dump command started and wrote a file that looked like a backup and held none of the data. Discourse is the live case: its launcher names the container `discourse`, and the image ships pg_dumpall. The regex stays a normaliser - `<app>-database` from compose and `<app>_database.1.<task>` from swarm still resolve to the same instance. Only the fallthrough changes. BREAKING CHANGE: a database container whose name carries no `database`, `db` or `postgres` token must now be named in --database-containers. Without that declaration its rows no longer match and no dump is written. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
163
tests/e2e/test_e2e_app_container_ships_client_tools.py
Normal file
163
tests/e2e/test_e2e_app_container_ships_client_tools.py
Normal file
@@ -0,0 +1,163 @@
|
||||
"""An application container that ships the engine's client tools.
|
||||
|
||||
This is the shape a dedicated database deploys in: the engine runs as
|
||||
`<app>-database` while the application itself runs as `<app>`, and neither is
|
||||
declared through --database-containers, so both names go through the instance
|
||||
regex. `<app>-database` loses its suffix and lands on the instance `<app>` -
|
||||
and `<app>` carries no database token at all, so a fallback that returns the
|
||||
name unchanged lands on that same instance and offers the application container
|
||||
as a second engine for the same row.
|
||||
|
||||
Discourse is the live example: its application container is named `discourse`
|
||||
by its own launcher and ships pg_dumpall, so a dump command starts there and
|
||||
writes a file that looks like a backup and holds none of the data.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from baudolo.generation import DUMP_SUFFIX, FILES_DIR, SQL_DIR
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
MARKER = "the-application-volume-holds-files"
|
||||
PAYLOAD = "shop-payload"
|
||||
|
||||
|
||||
class TestE2EAppContainerShipsClientTools(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
# uuid4 hex may begin with "db", which the instance regex would split
|
||||
# on and turn the application container into a different instance,
|
||||
# hiding exactly the collision this module is about.
|
||||
cls.prefix = unique("baudolo-e2e-app-tools").replace("-db", "-xb")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.engine = f"{cls.prefix}-shop-database"
|
||||
cls.app = f"{cls.prefix}-shop"
|
||||
cls.engine_volume = f"{cls.prefix}-shop-database-vol"
|
||||
cls.app_volume = f"{cls.prefix}-shop-app-vol"
|
||||
cls.containers = [cls.engine, cls.app]
|
||||
cls.volumes = [cls.engine_volume, cls.app_volume]
|
||||
|
||||
run(["docker", "volume", "create", cls.engine_volume])
|
||||
run(["docker", "volume", "create", cls.app_volume])
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.engine,
|
||||
"-e",
|
||||
"POSTGRES_PASSWORD=shoppw",
|
||||
"-e",
|
||||
"POSTGRES_DB=shopdb",
|
||||
"-e",
|
||||
"POSTGRES_USER=postgres",
|
||||
"-v",
|
||||
f"{cls.engine_volume}:{POSTGRES_DATA_DIR}",
|
||||
POSTGRES_IMAGE,
|
||||
]
|
||||
)
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.app,
|
||||
"--entrypoint",
|
||||
"sh",
|
||||
"-v",
|
||||
f"{cls.app_volume}:/data",
|
||||
POSTGRES_IMAGE,
|
||||
"-c",
|
||||
f"echo '{MARKER}' > /data/marker.txt && sleep 3600",
|
||||
]
|
||||
)
|
||||
|
||||
wait_for_postgres(cls.engine, user="postgres", timeout_s=90)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.engine,
|
||||
"sh",
|
||||
"-lc",
|
||||
(
|
||||
'psql -U postgres -d shopdb -c "CREATE TABLE orders (id int, '
|
||||
f"note text); INSERT INTO orders VALUES (1,'{PAYLOAD}');\""
|
||||
),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(
|
||||
cls.databases_csv,
|
||||
[(cls.app, "shopdb", "postgres", "shoppw")],
|
||||
)
|
||||
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
compose_dir=cls.compose_dir,
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=["dummy-db"],
|
||||
images_no_stop_required=[POSTGRES_IMAGE],
|
||||
)
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
|
||||
def volume_dir(self, volume: str) -> Path:
|
||||
return backup_path(self.backups_dir, self.repo_name, self.version, volume)
|
||||
|
||||
def test_the_engine_volume_was_dumped(self) -> None:
|
||||
dump = self.volume_dir(self.engine_volume) / SQL_DIR / f"shopdb{DUMP_SUFFIX}"
|
||||
self.assertTrue(dump.is_file(), f"expected a dump at {dump}")
|
||||
self.assertIn(PAYLOAD, dump.read_text(encoding="utf-8"))
|
||||
|
||||
def test_the_application_volume_produced_no_dump(self) -> None:
|
||||
"""The collision this module exists for: the application container
|
||||
answers the same instance as the engine and starts a dump of its own."""
|
||||
sql_dir = self.volume_dir(self.app_volume) / SQL_DIR
|
||||
self.assertFalse(
|
||||
sql_dir.exists(),
|
||||
f"the application container was dumped: {sorted(sql_dir.iterdir())}"
|
||||
if sql_dir.exists()
|
||||
else "",
|
||||
)
|
||||
|
||||
def test_the_application_volume_was_backed_up_as_files(self) -> None:
|
||||
"""Refusing the dump must not cost the volume its backup."""
|
||||
marker = self.volume_dir(self.app_volume) / FILES_DIR / "marker.txt"
|
||||
self.assertTrue(marker.is_file(), f"expected a file backup at {marker}")
|
||||
self.assertIn(MARKER, marker.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
159
tests/e2e/test_e2e_postgres_password_required.py
Normal file
159
tests/e2e/test_e2e_postgres_password_required.py
Normal file
@@ -0,0 +1,159 @@
|
||||
"""An engine whose loopback auth really demands a password.
|
||||
|
||||
Every other Postgres scenario runs stock postgres:alpine, whose generated
|
||||
pg_hba grants trust on 127.0.0.1 and ::1 - so `pg_dump -h localhost` never
|
||||
needs the password and a dump succeeds whether or not baudolo hands one to the
|
||||
container. This module makes the password mandatory, which is what a dedicated
|
||||
engine on a real host does.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from baudolo.generation import CLUSTER_SUFFIX, DUMP_SUFFIX, FILES_DIR, SQL_DIR
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
class TestE2EPostgresPasswordRequired(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-pg-password-required")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.pg_container = f"{cls.prefix}-pg"
|
||||
cls.pg_volume = f"{cls.prefix}-pg-vol"
|
||||
cls.containers = [cls.pg_container]
|
||||
cls.volumes = [cls.pg_volume]
|
||||
|
||||
run(["docker", "volume", "create", cls.pg_volume])
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.pg_container,
|
||||
"-e",
|
||||
"POSTGRES_PASSWORD=pgpw",
|
||||
"-e",
|
||||
"POSTGRES_DB=appdb",
|
||||
"-e",
|
||||
"POSTGRES_USER=postgres",
|
||||
# The entrypoint evals this into its initdb call, so the host
|
||||
# lines of pg_hba demand scram while the local socket stays
|
||||
# trust - the entrypoint's own init and the seeding below keep
|
||||
# working, and only a TCP connection needs the password.
|
||||
"-e",
|
||||
"POSTGRES_INITDB_ARGS=--auth-host=scram-sha-256",
|
||||
"-v",
|
||||
f"{cls.pg_volume}:{POSTGRES_DATA_DIR}",
|
||||
POSTGRES_IMAGE,
|
||||
]
|
||||
)
|
||||
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.pg_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
(
|
||||
'psql -U postgres -d appdb -c "CREATE TABLE t (id int primary '
|
||||
"key, v text); INSERT INTO t VALUES (1,'ok');\""
|
||||
),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
cls.unauthenticated = run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.pg_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
"pg_dump -U postgres -d appdb -h localhost",
|
||||
],
|
||||
capture=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(
|
||||
cls.databases_csv,
|
||||
[
|
||||
(cls.pg_container, "appdb", "postgres", "pgpw"),
|
||||
(cls.pg_container, "*", "postgres", "pgpw"),
|
||||
],
|
||||
)
|
||||
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
compose_dir=cls.compose_dir,
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=[cls.pg_container],
|
||||
images_no_stop_required=[POSTGRES_IMAGE],
|
||||
only_sql=True,
|
||||
)
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
|
||||
def volume_dir(self) -> Path:
|
||||
return backup_path(
|
||||
self.backups_dir, self.repo_name, self.version, self.pg_volume
|
||||
)
|
||||
|
||||
def test_a_dump_without_the_password_is_refused_by_the_server(self) -> None:
|
||||
"""Without this the module is vacuous: a pg_hba still saying trust would
|
||||
let a baudolo that forwards nothing pass just as well."""
|
||||
self.assertNotEqual(self.unauthenticated.returncode, 0)
|
||||
self.assertIn("no password supplied", self.unauthenticated.stderr or "")
|
||||
|
||||
def test_the_configured_database_was_dumped(self) -> None:
|
||||
dump = self.volume_dir() / SQL_DIR / f"appdb{DUMP_SUFFIX}"
|
||||
self.assertTrue(dump.is_file(), f"expected a dump at {dump}")
|
||||
self.assertIn("Dumped by pg_dump", dump.read_text(encoding="utf-8"))
|
||||
|
||||
def test_the_dump_carries_the_payload(self) -> None:
|
||||
"""pg_dump emits table data as COPY ... FROM stdin, so the row reads as
|
||||
tab-separated values rather than as an INSERT literal."""
|
||||
dump = self.volume_dir() / SQL_DIR / f"appdb{DUMP_SUFFIX}"
|
||||
self.assertIn("COPY public.t (id, v) FROM stdin;", dump.read_text("utf-8"))
|
||||
self.assertIn("1\tok", dump.read_text(encoding="utf-8"))
|
||||
|
||||
def test_the_cluster_row_was_dumped_too(self) -> None:
|
||||
cluster = self.volume_dir() / SQL_DIR / f"{self.pg_container}{CLUSTER_SUFFIX}"
|
||||
self.assertTrue(cluster.is_file(), f"expected a cluster dump at {cluster}")
|
||||
self.assertIn("CREATE DATABASE", cluster.read_text(encoding="utf-8"))
|
||||
|
||||
def test_only_sql_left_no_file_copy_behind(self) -> None:
|
||||
self.assertFalse((self.volume_dir() / FILES_DIR).exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user