mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2026-08-20 13:12:48 +00:00
build(lint): gate make test on a clean ruff run
ruff was never wired into this repository: no target, no CI step, no pin. It reported 45 findings across sources and tests, so nothing enforced what the codebase already mostly followed. Adds `make ruff` (check + format --check), `make ruff-fix`, and `make lint` as its alias, and makes `make test` run lint as a fourth parallel spur. The CI workflow calls `make test`, so it is covered there too. The linter is pinned in a `lint` extra: a ruff minor bump changes which rules fire, and with the suite gating on a clean run an unpinned linter would fail it on an unrelated day. The 45 findings are fixed rather than configured away. Three needed a decision instead of the mechanical fix: - The generation timestamp keeps its local wall clock (DTZ005 waived). Generations sort by that name, and UTC would order new ones before the existing ones wherever the offset is positive - "newest generation" is what every restore path selects on. - The per-volume `copy` closure now binds volume_name and vol_dir as default arguments (B023). It only worked because it is called inside the same iteration. - The two CLI top-level handlers keep their blind except (BLE001 waived): turning any failure into exit 1 is what a CLI boundary is for. The two in run.py did not need it and were narrowed to what they actually catch. Also drops the comments that restate the code: the section banners in restore/__main__.py, the filename repeated as line 1 of nine test files, step narration above the statement it narrates, and a block in app.py documenting parameters that had moved to another module. What names a trip-wire stays - the snapshot destination rule, the mysql-binary absence in MariaDB 11 images, the session-scoped FOREIGN_KEY_CHECKS, the spooled temp file for multi-GB dumps, and the negative control that loses its discriminating power if it ever passes. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
"""Shared e2e helpers, re-exported so tests import one name."""
|
||||
|
||||
from .fixtures import * # noqa: F401,F403
|
||||
from .process import * # noqa: F401,F403
|
||||
from .fixtures import *
|
||||
from .process import *
|
||||
|
||||
@@ -97,8 +97,7 @@ def write_databases_csv(path: str, rows: list[tuple[str, str, str, str]]) -> Non
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write("instance;database;username;password\n")
|
||||
for inst, db, user, pw in rows:
|
||||
f.write(f"{inst};{db};{user};{pw}\n")
|
||||
f.writelines(f"{inst};{db};{user};{pw}\n" for inst, db, user, pw in rows)
|
||||
|
||||
|
||||
def cleanup_docker(*, containers: list[str], volumes: list[str]) -> None:
|
||||
|
||||
@@ -12,8 +12,8 @@ import sys
|
||||
|
||||
sys.path.insert(0, "/src")
|
||||
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot # noqa: E402
|
||||
from baudolo.backup.volume import backup_volume # noqa: E402
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
||||
from baudolo.backup.volume import backup_volume
|
||||
|
||||
SUBJECT = "/subject/docker"
|
||||
VOLUME = "mariadb_data"
|
||||
@@ -23,7 +23,9 @@ GENERATION = f"{VERSIONS}/20260731"
|
||||
|
||||
|
||||
def shell(command: str) -> list[str]:
|
||||
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
||||
proc = subprocess.run(
|
||||
command, shell=True, capture_output=True, text=True, check=False
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise SnapshotError(
|
||||
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||
|
||||
@@ -12,7 +12,7 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, "/src")
|
||||
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot # noqa: E402
|
||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
||||
|
||||
KIND = sys.argv[1]
|
||||
SUBJECT = sys.argv[2]
|
||||
@@ -20,7 +20,9 @@ EXPECT = sys.argv[3]
|
||||
|
||||
|
||||
def shell(command: str) -> list[str]:
|
||||
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
||||
proc = subprocess.run(
|
||||
command, shell=True, capture_output=True, text=True, check=False
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise SnapshotError(
|
||||
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
# tests/e2e/test_e2e_dump_only_fallback_to_files.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
@@ -12,8 +11,8 @@ from .helpers import (
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
@@ -37,7 +36,6 @@ class TestE2EDumpOnlyFallbackToFiles(unittest.TestCase):
|
||||
|
||||
run(["docker", "volume", "create", cls.pg_volume])
|
||||
|
||||
# Start Postgres (creates a real DB volume)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
@@ -35,7 +35,6 @@ class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
|
||||
cls.containers: list[str] = []
|
||||
cls.volumes = [cls.db_volume, cls.files_volume]
|
||||
|
||||
# Create volumes
|
||||
run(["docker", "volume", "create", cls.db_volume])
|
||||
run(["docker", "volume", "create", cls.files_volume])
|
||||
|
||||
@@ -114,7 +113,6 @@ class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
|
||||
[(cls.pg_container, cls.pg_db, cls.pg_user, cls.pg_password)],
|
||||
)
|
||||
|
||||
# Run baudolo with dump-only-sql
|
||||
cmd = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
backup_run,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
REGISTRY_HOST = "svc-db-mariadb-swarm-mgr-01:5000"
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
backup_run,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
)
|
||||
|
||||
|
||||
@@ -30,7 +30,6 @@ class TestE2EFilesFull(unittest.TestCase):
|
||||
cls.containers = []
|
||||
cls.volumes = [cls.volume_src, cls.volume_dst]
|
||||
|
||||
# create source volume with a file
|
||||
run(["docker", "volume", "create", cls.volume_src])
|
||||
run(
|
||||
[
|
||||
@@ -50,7 +49,6 @@ class TestE2EFilesFull(unittest.TestCase):
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(cls.databases_csv, [])
|
||||
|
||||
# Run backup (files should be copied)
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
@@ -97,7 +95,6 @@ class TestE2EFilesFull(unittest.TestCase):
|
||||
]
|
||||
)
|
||||
|
||||
# verify restored file exists in dst volume
|
||||
p = run(
|
||||
[
|
||||
"docker",
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
backup_run,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
run,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
)
|
||||
|
||||
|
||||
@@ -29,7 +29,6 @@ class TestE2EFilesNoCopy(unittest.TestCase):
|
||||
cls.containers: list[str] = []
|
||||
cls.volumes = [cls.volume_src]
|
||||
|
||||
# Create source volume and write a marker file
|
||||
run(["docker", "volume", "create", cls.volume_src])
|
||||
run(
|
||||
[
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
# tests/e2e/test_e2e_images_no_backup_required_early_skip.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
@@ -34,11 +33,9 @@ class TestE2EImagesNoBackupRequiredEarlySkip(unittest.TestCase):
|
||||
cls.containers = [cls.redis_container]
|
||||
cls.volumes = [cls.ignored_volume, cls.normal_volume]
|
||||
|
||||
# Create volumes
|
||||
run(["docker", "volume", "create", cls.ignored_volume])
|
||||
run(["docker", "volume", "create", cls.normal_volume])
|
||||
|
||||
# Start redis container using the ignored volume
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
@@ -71,7 +68,6 @@ class TestE2EImagesNoBackupRequiredEarlySkip(unittest.TestCase):
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(cls.databases_csv, [])
|
||||
|
||||
# Run baudolo with images-no-backup-required redis
|
||||
cmd = [
|
||||
"baudolo",
|
||||
"--compose-dir",
|
||||
|
||||
@@ -30,8 +30,8 @@ import pandas
|
||||
from baudolo.backup import db as db_mod
|
||||
|
||||
from .helpers import (
|
||||
MARIADB_IMAGE,
|
||||
MARIADB_DATA_DIR,
|
||||
MARIADB_IMAGE,
|
||||
cleanup_docker,
|
||||
require_docker,
|
||||
run,
|
||||
|
||||
@@ -1,21 +1,20 @@
|
||||
# tests/e2e/test_e2e_mariadb_full.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
MARIADB_IMAGE,
|
||||
MARIADB_DATA_DIR,
|
||||
backup_run,
|
||||
MARIADB_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
unique,
|
||||
wait_for_mariadb,
|
||||
wait_for_mariadb_sql,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
@@ -71,7 +70,6 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
||||
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
||||
)
|
||||
|
||||
# Create table + data via the dedicated user (TCP)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
@@ -79,9 +77,11 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\""
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -112,8 +112,10 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "DROP TABLE {cls.db_name}.t;"',
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "DROP TABLE {cls.db_name}.t;"'
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -161,8 +163,10 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
||||
self.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"',
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"'
|
||||
),
|
||||
]
|
||||
)
|
||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||
|
||||
@@ -1,21 +1,20 @@
|
||||
# tests/e2e/test_e2e_mariadb_no_copy.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
MARIADB_IMAGE,
|
||||
MARIADB_DATA_DIR,
|
||||
backup_run,
|
||||
MARIADB_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
unique,
|
||||
wait_for_mariadb,
|
||||
wait_for_mariadb_sql,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
@@ -69,7 +68,6 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
||||
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
||||
)
|
||||
|
||||
# Create table + data (TCP)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
@@ -77,9 +75,11 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\""
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -110,8 +110,10 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "DROP TABLE {cls.db_name}.t;"',
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f'-e "DROP TABLE {cls.db_name}.t;"'
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -158,8 +160,10 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
||||
self.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"',
|
||||
(
|
||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"'
|
||||
),
|
||||
]
|
||||
)
|
||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||
|
||||
@@ -16,15 +16,8 @@ from .helpers import (
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
# A `database = '*'` row makes the backup side write one pg_dumpall stream for
|
||||
# the whole instance instead of a dump per database - the shape an application
|
||||
# with several databases in one engine produces. This proves the stream is
|
||||
# replayable: two databases and their owning role are dropped outright, and the
|
||||
# cluster restore has to bring all three back. Before the cluster subcommand
|
||||
# existed the dump was stored and unreadable.
|
||||
# Each statement runs on its own: psql wraps a multi-statement -c in one
|
||||
# transaction, and CREATE DATABASE is forbidden inside one - the same rule that
|
||||
# keeps the cluster replay out of --single-transaction.
|
||||
# One statement per entry: psql wraps a multi-statement -c in a transaction,
|
||||
# and CREATE DATABASE is forbidden inside one.
|
||||
SEED_SQL = (
|
||||
"CREATE ROLE app LOGIN PASSWORD 'apppw'",
|
||||
"CREATE DATABASE first OWNER app",
|
||||
@@ -96,7 +89,6 @@ class TestE2EPostgresClusterRestore(unittest.TestCase):
|
||||
/ f"{cls.pg_container}.cluster.backup.sql"
|
||||
)
|
||||
|
||||
# The disaster: both databases and the role that owns them are gone.
|
||||
for statement in DROP_SQL:
|
||||
cls._psql("postgres", statement)
|
||||
|
||||
@@ -156,9 +148,6 @@ class TestE2EPostgresClusterRestore(unittest.TestCase):
|
||||
self.assertEqual(self._psql("second", "SELECT v FROM t"), "second-payload")
|
||||
|
||||
def test_the_superusers_own_create_was_filtered(self) -> None:
|
||||
# The dump recreates every role including the one the replay connects
|
||||
# as; only its ALTER may survive, or the stream dies on the first
|
||||
# statement with ON_ERROR_STOP.
|
||||
self.assertEqual(
|
||||
self._psql(
|
||||
"postgres", "SELECT rolsuper FROM pg_roles WHERE rolname = 'postgres'"
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
# tests/e2e/test_e2e_postgres_empty_drop_hard.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
|
||||
@@ -1,20 +1,19 @@
|
||||
# tests/e2e/test_e2e_postgres_full.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
backup_run,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
@@ -55,7 +54,6 @@ class TestE2EPostgresFull(unittest.TestCase):
|
||||
)
|
||||
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||
|
||||
# Create a table + data
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
|
||||
@@ -1,20 +1,19 @@
|
||||
# tests/e2e/test_e2e_postgres_no_copy.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
backup_run,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
unique,
|
||||
wait_for_postgres,
|
||||
write_databases_csv,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
# tests/e2e/test_e2e_postgres_single_transaction_live_writer.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_run,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_IMAGE,
|
||||
POSTGRES_DATA_DIR,
|
||||
POSTGRES_IMAGE,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
# tests/e2e/test_e2e_swarm_task_skip.py
|
||||
#
|
||||
# Reproduces the swarm flake fixed on this branch: baudolo used to stop a
|
||||
# swarm task container around the volume file backup because its image was
|
||||
# not whitelisted; the orchestrator immediately replaced the stopped task and
|
||||
|
||||
Reference in New Issue
Block a user