feat(backup): write database dumps gzip-compressed

Dumps stream through gzip into <db>.backup.sql.gz; no uncompressed copy touches the disk. Restore reads gzip and plain dumps by their magic bytes, verifies a gzip dump to its end before anything is dropped, and still finds the uncompressed names of older generations. The manifest names the compression.

🎯 Fix: 92 %
💥 Break: 20 %
💸 Cost: –
⏱️ Time: –
🛡️ Security: ~100 %
🏆 Score: 74
🚦 Gate: ⛔ break 20 % > 5

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/e2e/helpers/fixtures.py

@@ -2,10 +2,14 @@
 
 from __future__ import annotations
 
+import gzip
 import shutil
 import subprocess
 from pathlib import Path
 
+from baudolo.generation import DUMP_SUFFIX, LEGACY_DUMP_SUFFIX
+from baudolo.restore.dumpfile import open_dump
+
 from .process import machine_hash, run
 
 # postgres 18+ mounts at /var/lib/postgresql, not /var/lib/postgresql/data.
@@ -112,3 +116,54 @@ def ensure_empty_dir(path: str) -> None:
     if p.exists():
         shutil.rmtree(p)
     p.mkdir(parents=True, exist_ok=True)
+
+
+def dump_text(path: Path | str) -> str:
+    """
+    Args:
+        path: a dump, gzip-compressed or plain.
+
+    Returns:
+        Its SQL text.
+    """
+    with open_dump(str(path)) as handle:
+        return handle.read().decode("utf-8", errors="replace")
+
+
+def write_dump_text(path: Path | str, text: str) -> None:
+    """
+    Args:
+        path: a gzip dump to overwrite.
+        text: the SQL text it carries afterwards.
+    """
+    with gzip.open(str(path), "wb") as handle:
+        handle.write(text.encode("utf-8"))
+
+
+def dumps_in(sql_dir: Path) -> list[Path]:
+    """
+    Args:
+        sql_dir: a volume's sql directory, which may be absent.
+
+    Returns:
+        Every dump in it, compressed or plain, single-database or cluster.
+    """
+    if not sql_dir.is_dir():
+        return []
+    return sorted(
+        {*sql_dir.glob(f"*{DUMP_SUFFIX}"), *sql_dir.glob(f"*{LEGACY_DUMP_SUFFIX}")}
+    )
+
+
+def assert_dumps_in_sees_both_suffixes(case, scratch: str) -> None:
+    """
+    Args:
+        case: the running ``unittest.TestCase``.
+        scratch: an empty directory to place the probe dumps in.
+    """
+    probe = Path(scratch) / "dumps-in-probe"
+    probe.mkdir(parents=True, exist_ok=True)
+    expected = [probe / f"a{DUMP_SUFFIX}", probe / f"b{LEGACY_DUMP_SUFFIX}"]
+    for path in expected:
+        path.write_bytes(b"")
+    case.assertEqual(dumps_in(probe), expected)

tests/e2e/test_e2e_app_container_ships_client_tools.py

@@ -26,6 +26,7 @@ from .helpers import (
     backup_run,
     cleanup_docker,
     create_minimal_compose_dir,
+    dump_text,
     ensure_empty_dir,
     latest_version_dir,
     require_docker,
@@ -140,7 +141,7 @@ class TestE2EAppContainerShipsClientTools(unittest.TestCase):
     def test_the_engine_volume_was_dumped(self) -> None:
         dump = self.volume_dir(self.engine_volume) / SQL_DIR / f"shopdb{DUMP_SUFFIX}"
         self.assertTrue(dump.is_file(), f"expected a dump at {dump}")
-        self.assertIn(PAYLOAD, dump.read_text(encoding="utf-8"))
+        self.assertIn(PAYLOAD, dump_text(dump))
 
     def test_the_application_volume_produced_no_dump(self) -> None:
         """The collision this module exists for: the application container

tests/e2e/test_e2e_engine_detection_by_tool.py

@@ -13,6 +13,8 @@ inside Alpine and takes the whole run with it.
 
 import unittest
 
+from baudolo.generation import DUMP_SUFFIX, SQL_DIR
+
 from .helpers import (
     POSTGRES_DATA_DIR,
     POSTGRES_IMAGE,
@@ -20,6 +22,8 @@ from .helpers import (
     backup_run,
     cleanup_docker,
     create_minimal_compose_dir,
+    dump_text,
+    dumps_in,
     ensure_empty_dir,
     latest_version_dir,
     require_docker,
@@ -142,16 +146,20 @@ class TestE2EEngineDetectionByTool(unittest.TestCase):
         return backup_path(self.backups_dir, self.repo_name, self.version, volume)
 
     def test_an_engine_without_an_engine_name_is_still_dumped(self) -> None:
-        dump = self._volume_dir(self.engine_volume) / "sql" / "appdb.backup.sql"
+        dump = self._volume_dir(self.engine_volume) / SQL_DIR / f"appdb{DUMP_SUFFIX}"
         self.assertTrue(
             dump.is_file(),
             f"a Postgres tagged '{self.engine_image}' produced no dump at {dump}",
         )
-        self.assertIn("Dumped by pg_dump", dump.read_text(encoding="utf-8"))
+        self.assertIn("Dumped by pg_dump", dump_text(dump))
 
     def test_an_engine_name_without_an_engine_is_not_dumped(self) -> None:
-        sql_dir = self._volume_dir(self.impostor_volume) / "sql"
-        dumps = list(sql_dir.glob("*.sql")) if sql_dir.exists() else []
+        self.assertEqual(
+            dumps_in(self._volume_dir(self.engine_volume) / SQL_DIR),
+            [self._volume_dir(self.engine_volume) / SQL_DIR / f"appdb{DUMP_SUFFIX}"],
+            "the glob must see the real engine's dump, or the check below is vacuous",
+        )
+        dumps = dumps_in(self._volume_dir(self.impostor_volume) / SQL_DIR)
         self.assertEqual(
             dumps,
             [],

tests/e2e/test_e2e_engine_detection_registry_prefix.py

@@ -1,5 +1,7 @@
 import unittest
 
+from baudolo.generation import DUMP_SUFFIX, SQL_DIR
+
 from .helpers import (
     POSTGRES_DATA_DIR,
     POSTGRES_IMAGE,
@@ -7,6 +9,7 @@ from .helpers import (
     backup_run,
     cleanup_docker,
     create_minimal_compose_dir,
+    dump_text,
     ensure_empty_dir,
     latest_version_dir,
     require_docker,
@@ -94,11 +97,11 @@ class TestE2EEngineDetectionRegistryPrefix(unittest.TestCase):
     def test_the_registry_host_does_not_pick_the_engine(self) -> None:
         p = (
             backup_path(self.backups_dir, self.repo_name, self.version, self.pg_volume)
-            / "sql"
-            / "appdb.backup.sql"
+            / SQL_DIR
+            / f"appdb{DUMP_SUFFIX}"
         )
         self.assertTrue(p.is_file(), f"Expected a pg_dump at: {p}")
-        self.assertIn("Dumped by pg_dump", p.read_text(encoding="utf-8"))
+        self.assertIn("Dumped by pg_dump", dump_text(p))
 
 
 if __name__ == "__main__":

tests/e2e/test_e2e_mariadb_anonymous_preemption.py

@@ -34,6 +34,7 @@ from .helpers import (
     MARIADB_DATA_DIR,
     MARIADB_IMAGE,
     cleanup_docker,
+    dump_text,
     require_docker,
     run,
     unique,
@@ -159,8 +160,7 @@ class TestE2EMariaDBAnonymousPreemption(unittest.TestCase):
             self.assertTrue(produced, "backup_database did not produce a dump")
             dump_path = Path(volume_dir) / SQL_DIR / f"{self.db_name}{DUMP_SUFFIX}"
             self.assertTrue(dump_path.is_file(), f"expected dump at {dump_path}")
-            with dump_path.open(encoding="utf-8", errors="replace") as f:
-                content = f.read()
+            content = dump_text(dump_path)
             self.assertIn("INSERT INTO", content)
             self.assertIn("'ok'", content)
 

tests/e2e/test_e2e_mariadb_full.py

@@ -1,5 +1,7 @@
 import unittest
 
+from baudolo.generation import DUMP_SUFFIX, SQL_DIR
+
 from .helpers import (
     MARIADB_DATA_DIR,
     MARIADB_IMAGE,
@@ -7,6 +9,7 @@ from .helpers import (
     backup_run,
     cleanup_docker,
     create_minimal_compose_dir,
+    dump_text,
     ensure_empty_dir,
     latest_version_dir,
     require_docker,
@@ -150,10 +153,11 @@ class TestE2EMariaDBFull(unittest.TestCase):
     def test_dump_file_exists(self) -> None:
         p = (
             backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume)
-            / "sql"
-            / f"{self.db_name}.backup.sql"
+            / SQL_DIR
+            / f"{self.db_name}{DUMP_SUFFIX}"
         )
         self.assertTrue(p.is_file(), f"Expected dump file at: {p}")
+        self.assertIn("INSERT INTO", dump_text(p))
 
     def test_data_restored(self) -> None:
         p = run(

tests/e2e/test_e2e_only_files.py

@@ -7,12 +7,16 @@ make the flag useless there.
 
 import unittest
 
+from baudolo.generation import SQL_DIR
+
 from .helpers import (
     POSTGRES_DATA_DIR,
     POSTGRES_IMAGE,
+    assert_dumps_in_sees_both_suffixes,
     backup_path,
     cleanup_docker,
     create_minimal_compose_dir,
+    dumps_in,
     ensure_empty_dir,
     latest_version_dir,
     require_docker,
@@ -104,8 +108,8 @@ class TestE2EOnlyFiles(unittest.TestCase):
         self.assertEqual(marker.read_text(encoding="utf-8").strip(), MARKER)
 
     def test_no_dump_is_written(self) -> None:
-        sql_dir = self._volume_dir() / "sql"
-        dumps = list(sql_dir.glob("*.sql")) if sql_dir.exists() else []
+        assert_dumps_in_sees_both_suffixes(self, f"/tmp/{self.prefix}")
+        dumps = dumps_in(self._volume_dir() / SQL_DIR)
         self.assertEqual(dumps, [], f"did not expect any dump, found: {dumps}")
 
     def test_the_missing_databases_csv_is_not_reported(self) -> None:

tests/e2e/test_e2e_only_sql_fallback_to_files.py

@@ -1,14 +1,24 @@
 import json
 import unittest
 
-from baudolo.generation import FILES_DIR, MANIFEST_FILE, MANIFEST_SCHEMA, SQL_DIR
+from baudolo.generation import (
+    CLUSTER_SUFFIX,
+    DUMP_COMPRESSION,
+    DUMP_SUFFIX,
+    FILES_DIR,
+    MANIFEST_FILE,
+    MANIFEST_SCHEMA,
+    SQL_DIR,
+)
 
 from .helpers import (
     POSTGRES_DATA_DIR,
     POSTGRES_IMAGE,
+    assert_dumps_in_sees_both_suffixes,
     backup_path,
     cleanup_docker,
     create_minimal_compose_dir,
+    dumps_in,
     ensure_empty_dir,
     latest_version_dir,
     require_docker,
@@ -143,7 +153,7 @@ class TestE2EOnlySqlFallbackToFiles(unittest.TestCase):
         self.assertTrue(p.is_dir(), f"Expected files backup dir at: {p}")
 
     def test_sql_dump_not_present(self) -> None:
-        # There should be no sql dumps because databases.csv had no matching entry.
+        assert_dumps_in_sees_both_suffixes(self, f"/tmp/{self.prefix}")
         sql_dir = (
             backup_path(
                 self.backups_dir,
@@ -151,16 +161,10 @@ class TestE2EOnlySqlFallbackToFiles(unittest.TestCase):
                 self.version,
                 self.pg_volume,
             )
-            / "sql"
+            / SQL_DIR
         )
-        # Could exist (dir created) in some edge cases, but should contain no *.sql dumps.
-        if sql_dir.exists():
-            dumps = list(sql_dir.glob("*.sql"))
-            self.assertEqual(
-                len(dumps),
-                0,
-                f"Did not expect SQL dump files, found: {dumps}",
-            )
+        dumps = dumps_in(sql_dir)
+        self.assertEqual(dumps, [], f"Did not expect SQL dump files, found: {dumps}")
 
     def manifest(self) -> dict:
         generation = backup_path(
@@ -183,6 +187,9 @@ class TestE2EOnlySqlFallbackToFiles(unittest.TestCase):
         self.assertTrue((volume_dir / layout["files_dir"]).is_dir())
         self.assertEqual(layout["files_dir"], FILES_DIR)
         self.assertEqual(layout["sql_dir"], SQL_DIR)
+        self.assertEqual(layout["dump_suffix"], DUMP_SUFFIX)
+        self.assertEqual(layout["cluster_suffix"], CLUSTER_SUFFIX)
+        self.assertEqual(layout["dump_compression"], DUMP_COMPRESSION)
 
     def test_the_manifest_states_a_schema_a_reader_can_check(self) -> None:
         self.assertEqual(self.manifest()["schema"], MANIFEST_SCHEMA)

tests/e2e/test_e2e_postgres_cluster_restore.py

@@ -1,5 +1,7 @@
 import unittest
 
+from baudolo.generation import CLUSTER_SUFFIX, SQL_DIR
+
 from .helpers import (
     POSTGRES_DATA_DIR,
     POSTGRES_IMAGE,
@@ -85,8 +87,8 @@ class TestE2EPostgresClusterRestore(unittest.TestCase):
         cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
         cls.dump = (
             backup_path(cls.backups_dir, cls.repo_name, cls.version, cls.pg_volume)
-            / "sql"
-            / f"{cls.pg_container}.cluster.backup.sql"
+            / SQL_DIR
+            / f"{cls.pg_container}{CLUSTER_SUFFIX}"
         )
 
         run(cls._restore_argv(), stdin_text="pgpw")

tests/e2e/test_e2e_postgres_full.py

@@ -1,5 +1,7 @@
 import unittest
 
+from baudolo.generation import DUMP_SUFFIX, SQL_DIR
+
 from .helpers import (
     POSTGRES_DATA_DIR,
     POSTGRES_IMAGE,
@@ -7,6 +9,7 @@ from .helpers import (
     backup_run,
     cleanup_docker,
     create_minimal_compose_dir,
+    dump_text,
     ensure_empty_dir,
     latest_version_dir,
     require_docker,
@@ -124,10 +127,11 @@ class TestE2EPostgresFull(unittest.TestCase):
     def test_dump_file_exists(self) -> None:
         p = (
             backup_path(self.backups_dir, self.repo_name, self.version, self.pg_volume)
-            / "sql"
-            / "appdb.backup.sql"
+            / SQL_DIR
+            / f"appdb{DUMP_SUFFIX}"
         )
         self.assertTrue(p.is_file(), f"Expected dump file at: {p}")
+        self.assertIn("Dumped by pg_dump", dump_text(p))
 
     def test_the_deprecated_password_argument_still_restores_but_warns(self) -> None:
         self.assertIn("visible in the process list", self.restored.stderr)

tests/e2e/test_e2e_postgres_password_required.py

@@ -19,6 +19,7 @@ from .helpers import (
     backup_run,
     cleanup_docker,
     create_minimal_compose_dir,
+    dump_text,
     ensure_empty_dir,
     latest_version_dir,
     require_docker,
@@ -137,19 +138,19 @@ class TestE2EPostgresPasswordRequired(unittest.TestCase):
     def test_the_configured_database_was_dumped(self) -> None:
         dump = self.volume_dir() / SQL_DIR / f"appdb{DUMP_SUFFIX}"
         self.assertTrue(dump.is_file(), f"expected a dump at {dump}")
-        self.assertIn("Dumped by pg_dump", dump.read_text(encoding="utf-8"))
+        self.assertIn("Dumped by pg_dump", dump_text(dump))
 
     def test_the_dump_carries_the_payload(self) -> None:
         """pg_dump emits table data as COPY ... FROM stdin, so the row reads as
         tab-separated values rather than as an INSERT literal."""
-        dump = self.volume_dir() / SQL_DIR / f"appdb{DUMP_SUFFIX}"
-        self.assertIn("COPY public.t (id, v) FROM stdin;", dump.read_text("utf-8"))
-        self.assertIn("1\tok", dump.read_text(encoding="utf-8"))
+        text = dump_text(self.volume_dir() / SQL_DIR / f"appdb{DUMP_SUFFIX}")
+        self.assertIn("COPY public.t (id, v) FROM stdin;", text)
+        self.assertIn("1\tok", text)
 
     def test_the_cluster_row_was_dumped_too(self) -> None:
         cluster = self.volume_dir() / SQL_DIR / f"{self.pg_container}{CLUSTER_SUFFIX}"
         self.assertTrue(cluster.is_file(), f"expected a cluster dump at {cluster}")
-        self.assertIn("CREATE DATABASE", cluster.read_text(encoding="utf-8"))
+        self.assertIn("CREATE DATABASE", dump_text(cluster))
 
     def test_only_sql_left_no_file_copy_behind(self) -> None:
         self.assertFalse((self.volume_dir() / FILES_DIR).exists())

tests/e2e/test_e2e_seed_star_and_db_entries_backup_postgres.py

@@ -1,11 +1,14 @@
 import unittest
 
+from baudolo.generation import CLUSTER_SUFFIX, DUMP_SUFFIX
+
 from .helpers import (
     POSTGRES_DATA_DIR,
     POSTGRES_IMAGE,
     backup_path,
     cleanup_docker,
     create_minimal_compose_dir,
+    dump_text,
     ensure_empty_dir,
     latest_version_dir,
     require_docker,
@@ -197,15 +200,15 @@ class TestE2ESeedStarAndDbEntriesBackupPostgres(unittest.TestCase):
         )
 
         # Cluster dump file produced by '*' entry
-        cluster = sql_dir / f"{self.pg_container}.cluster.backup.sql"
+        cluster = sql_dir / f"{self.pg_container}{CLUSTER_SUFFIX}"
         self.assertTrue(cluster.is_file(), f"Expected cluster dump file at: {cluster}")
 
         # Concrete DB dump produced by normal entry
-        db1 = sql_dir / f"{self.pg_db1}.backup.sql"
+        db1 = sql_dir / f"{self.pg_db1}{DUMP_SUFFIX}"
         self.assertTrue(db1.is_file(), f"Expected db dump file at: {db1}")
 
         # Basic sanity: cluster dump usually contains CREATE DATABASE statements
-        txt = cluster.read_text(encoding="utf-8", errors="ignore")
+        txt = dump_text(cluster)
         self.assertIn(
             "CREATE DATABASE",
             txt,

tests/e2e/test_e2e_version_gate.py

@@ -10,6 +10,8 @@ import re
 import unittest
 from pathlib import Path
 
+from baudolo.generation import DUMP_SUFFIX, SQL_DIR
+
 from .helpers import (
     MARIADB_DATA_DIR,
     MARIADB_IMAGE,
@@ -19,6 +21,7 @@ from .helpers import (
     backup_run,
     cleanup_docker,
     create_minimal_compose_dir,
+    dump_text,
     ensure_empty_dir,
     latest_version_dir,
     require_docker,
@@ -28,6 +31,7 @@ from .helpers import (
     wait_for_mariadb_sql,
     wait_for_postgres,
     write_databases_csv,
+    write_dump_text,
 )
 
 PAYLOAD = "gate-payload"
@@ -36,14 +40,14 @@ FUTURE = "99.0"
 
 def rewrite_version(dump: Path, pattern: str, version: str) -> str:
     """Make the dump claim ``version``; return what it claimed before."""
-    text = dump.read_text(encoding="utf-8", errors="replace")
+    text = dump_text(dump)
     found = re.search(pattern, text)
     if not found:
         raise AssertionError(f"{dump} carries no version header matching {pattern}")
     claimed = found.group(1)
-    dump.write_text(
+    write_dump_text(
+        dump,
         text.replace(found.group(0), found.group(0).replace(claimed, version), 1),
-        encoding="utf-8",
     )
     return claimed
 
@@ -103,8 +107,8 @@ class GateCase:
         cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
         cls.dump = (
             backup_path(cls.backups_dir, cls.repo_name, cls.version, cls.volume)
-            / "sql"
-            / f"{cls.db_name}.backup.sql"
+            / SQL_DIR
+            / f"{cls.db_name}{DUMP_SUFFIX}"
         )
 
         cls.truthful_version = rewrite_version(cls.dump, cls.pattern, FUTURE)