kevinveenbirkenbach/docker-volume-backup
1
0
Fork 0
mirror of https://github.com/kevinveenbirkenbach/docker-volume-backup.git synced 2026-08-20 21:22:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

build(lint): gate make test on a clean ruff run

Browse Source 
ruff was never wired into this repository: no target, no CI step, no pin.
It reported 45 findings across sources and tests, so nothing enforced
what the codebase already mostly followed.

Adds `make ruff` (check + format --check), `make ruff-fix`, and `make
lint` as its alias, and makes `make test` run lint as a fourth parallel
spur. The CI workflow calls `make test`, so it is covered there too. The
linter is pinned in a `lint` extra: a ruff minor bump changes which rules
fire, and with the suite gating on a clean run an unpinned linter would
fail it on an unrelated day.

The 45 findings are fixed rather than configured away. Three needed a
decision instead of the mechanical fix:

- The generation timestamp keeps its local wall clock (DTZ005 waived).
  Generations sort by that name, and UTC would order new ones before the
  existing ones wherever the offset is positive - "newest generation" is
  what every restore path selects on.
- The per-volume `copy` closure now binds volume_name and vol_dir as
  default arguments (B023). It only worked because it is called inside
  the same iteration.
- The two CLI top-level handlers keep their blind except (BLE001
  waived): turning any failure into exit 1 is what a CLI boundary is
  for. The two in run.py did not need it and were narrowed to what they
  actually catch.

Also drops the comments that restate the code: the section banners in
restore/__main__.py, the filename repeated as line 1 of nine test files,
step narration above the statement it narrates, and a block in app.py
documenting parameters that had moved to another module. What names a
trip-wire stays - the snapshot destination rule, the mysql-binary
absence in MariaDB 11 images, the session-scoped FOREIGN_KEY_CHECKS, the
spooled temp file for multi-GB dumps, and the negative control that
loses its discriminating power if it ever passes.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-Birkenbach
2026-08-17 04:35:50 +02:00
parent a0204fd3ea
commit 2129c5e362
41 changed files with 199 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/baudolo/backup/__main__.py
View File

@@ -1,9 +1,6 @@
#!/usr/bin/env python3
from __future__ import annotations
from .app import main
if __name__ == "__main__":
raise SystemExit(main())

41
src/baudolo/backup/app.py
View File

@@ -30,17 +30,13 @@ def main() -> int:
args = parse_args()
machine_id = get_machine_id()
backup_time = datetime.now().strftime("%Y%m%d%H%M%S")
# Local wall clock on purpose: generations sort by this name, and UTC would
# order new ones before the existing ones wherever the offset is positive.
backup_time = datetime.now().strftime("%Y%m%d%H%M%S") # noqa: DTZ005
versions_dir = os.path.join(args.backups_dir, machine_id, args.repo_name)
version_dir = create_version_directory(versions_dir, backup_time)
# IMPORTANT:
# - keep_default_na=False prevents empty fields from turning into NaN
# - dtype=str keeps all columns stable for comparisons/validation
#
# Robust behavior:
# - if the file is missing or empty, we continue without DB dumps.
databases_df = load_databases_df(args.databases_csv)
print("💾 Start volume backups...", flush=True)
@@ -80,24 +76,29 @@ def main() -> int:
database_containers=args.database_containers,
)
if args.dump_only_sql:
if found_db:
if not dumped_any:
print(
f"WARNING: dump-only-sql requested but no DB dump was produced for DB volume '{volume_name}'. "
"Falling back to file backup.",
flush=True,
)
else:
continue
if args.dump_only_sql and found_db:
if not dumped_any:
print(
f"WARNING: dump-only-sql requested but no DB dump was produced for DB volume '{volume_name}'. "
"Falling back to file backup.",
flush=True,
)
else:
continue
live_source = get_storage_path(volume_name)
def copy(*, authoritative: bool, source: str = live_source) -> None:
def copy(
*,
authoritative: bool,
source: str = live_source,
volume: str = volume_name,
target: str = vol_dir,
) -> None:
backup_volume(
versions_dir,
volume_name,
vol_dir,
volume,
target,
authoritative=authoritative,
source=source,
)

5
src/baudolo/backup/compose.py
View File

@@ -4,10 +4,9 @@ import os
import shutil
import subprocess
from pathlib import Path
from typing import List, Optional
def _build_compose_cmd(project_dir: str, passthrough: List[str]) -> List[str]:
def _build_compose_cmd(project_dir: str, passthrough: list[str]) -> list[str]:
"""
Build the compose command for this project directory.
@@ -30,7 +29,7 @@ def _build_compose_cmd(project_dir: str, passthrough: List[str]) -> List[str]:
raise RuntimeError("Neither 'compose' nor 'docker' found in PATH")
def _find_compose_file(project_dir: str) -> Optional[Path]:
def _find_compose_file(project_dir: str) -> Path | None:
"""
Detect a compose file in `project_dir` (case-insensitive).

10
src/baudolo/backup/db.py
View File

@@ -1,10 +1,9 @@
from __future__ import annotations
import logging
import os
import pathlib
import re
import logging
from typing import Optional
import pandas
@@ -22,7 +21,7 @@ def get_instance(container: str, database_containers: list[str]) -> str:
return re.split(r"(_|-)(database|db|postgres)", container)[0]
def _validate_database_value(value: Optional[str], *, instance: str) -> str:
def _validate_database_value(value: str | None, *, instance: str) -> str:
"""
Enforce explicit database semantics:
@@ -70,7 +69,7 @@ def backup_database(
container: str,
volume_dir: str,
db_type: str,
databases_df: "pandas.DataFrame",
databases_df: pandas.DataFrame,
database_containers: list[str],
) -> bool:
"""
@@ -97,7 +96,6 @@ def backup_database(
db_value = _validate_database_value(raw_db, instance=instance_name)
# Explicit: dump ALL databases
if db_value == "*":
if db_type != "postgres":
raise ValueError(
@@ -110,7 +108,6 @@ def backup_database(
produced = True
continue
# Concrete database dump
db_name = db_value
dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
@@ -135,7 +132,6 @@ def backup_database(
_atomic_write_cmd(cmd, dump_file)
produced = True
except BackupException as e:
# Explicit DB dump failed -> hard error
raise BackupException(
f"Postgres dump failed for instance '{instance_name}', "
f"database '{db_name}'. This database was explicitly configured "

2
src/baudolo/backup/docker.py
View File

@@ -98,5 +98,5 @@ def docker_volume_exists(volume: str) -> bool:
f"docker volume inspect {volume} >/dev/null 2>&1 && echo OK"
)
return True
except Exception:
except BackupException:
return False

8
src/baudolo/backup/dumps.py
View File

@@ -15,7 +15,7 @@ def backup_mariadb_or_postgres(
*,
container: str,
volume_dir: str,
databases_df: "pandas.DataFrame",
databases_df: pandas.DataFrame,
database_containers: list[str],
) -> tuple[bool, bool]:
"""
@@ -34,7 +34,7 @@ def backup_mariadb_or_postgres(
return False, False
def _empty_databases_df() -> "pandas.DataFrame":
def _empty_databases_df() -> pandas.DataFrame:
"""
Create an empty DataFrame with the expected schema for databases.csv.
@@ -44,7 +44,7 @@ def _empty_databases_df() -> "pandas.DataFrame":
return pandas.DataFrame(columns=["instance", "database", "username", "password"])
def load_databases_df(csv_path: str) -> "pandas.DataFrame":
def load_databases_df(csv_path: str) -> pandas.DataFrame:
"""
Load databases.csv robustly.
@@ -74,7 +74,7 @@ def backup_dumps_for_volume(
*,
containers: list[str],
vol_dir: str,
databases_df: "pandas.DataFrame",
databases_df: pandas.DataFrame,
database_containers: list[str],
) -> tuple[bool, bool]:
"""

16
src/baudolo/restore/__main__.py
View File

@@ -34,9 +34,6 @@ def main(argv: list[str] | None = None) -> int:
)
sub = parser.add_subparsers(dest="cmd", required=True)
# ------------------------------------------------------------------
# files
# ------------------------------------------------------------------
p_files = sub.add_parser("files", help="Restore files into a docker volume")
_add_common_backup_args(p_files)
p_files.add_argument(
@@ -49,9 +46,6 @@ def main(argv: list[str] | None = None) -> int:
),
)
# ------------------------------------------------------------------
# postgres
# ------------------------------------------------------------------
p_pg = sub.add_parser("postgres", help="Restore a single PostgreSQL database dump")
_add_common_backup_args(p_pg)
p_pg.add_argument("--container", required=True)
@@ -60,9 +54,6 @@ def main(argv: list[str] | None = None) -> int:
p_pg.add_argument("--db-password", required=True)
p_pg.add_argument("--empty", action="store_true")
# ------------------------------------------------------------------
# cluster
# ------------------------------------------------------------------
p_cluster = sub.add_parser(
"cluster", help="Restore a full PostgreSQL cluster dump (pg_dumpall)"
)
@@ -81,9 +72,6 @@ def main(argv: list[str] | None = None) -> int:
p_cluster.add_argument("--db-password", required=True)
p_cluster.add_argument("--empty", action="store_true")
# ------------------------------------------------------------------
# mariadb
# ------------------------------------------------------------------
p_mdb = sub.add_parser(
"mariadb", help="Restore a single MariaDB/MySQL-compatible dump"
)
@@ -98,8 +86,6 @@ def main(argv: list[str] | None = None) -> int:
try:
if args.cmd == "files":
# target volume = args.volume_name
# source volume (backup key) defaults to target volume
source_volume = args.source_volume or args.volume_name
bp_files = BackupPaths(
@@ -170,7 +156,7 @@ def main(argv: list[str] | None = None) -> int:
parser.error("Unhandled command")
return 2
except Exception as e:
except Exception as e: # noqa: BLE001 - CLI boundary: any failure becomes exit 1
print(f"ERROR: {e}", file=sys.stderr)
return 1

8
src/baudolo/restore/db/mariadb.py
View File

@@ -22,11 +22,11 @@ exit 42
if not out:
raise RuntimeError("empty client detection output")
return out
except Exception as e:
except Exception:
print(
"ERROR: neither 'mariadb' nor 'mysql' found in container.", file=sys.stderr
)
raise e
raise
def restore_mariadb_sql(
@@ -44,9 +44,7 @@ def restore_mariadb_sql(
raise FileNotFoundError(sql_path)
if empty:
# IMPORTANT:
# Do NOT hardcode 'mysql' here. Use the detected client.
# MariaDB 11 images may not contain the mysql binary at all.
# Do not hardcode 'mysql': MariaDB 11 images may not ship that binary.
result = docker_exec(
container,
[

1
src/baudolo/restore/db/postgres.py
View File

@@ -50,7 +50,6 @@ def restore_postgres_sql(
if not os.path.isfile(sql_path):
raise FileNotFoundError(sql_path)
# Make password available INSIDE the container for psql.
docker_env = {"PGPASSWORD": password}
if empty:

29
src/baudolo/restore/run.py
View File

@@ -2,7 +2,6 @@ from __future__ import annotations
import subprocess
import sys
from typing import Optional
def run(
@@ -10,7 +9,7 @@ def run(
*,
stdin=None,
capture: bool = False,
env: Optional[dict] = None,
env: dict | None = None,
) -> subprocess.CompletedProcess:
try:
kwargs: dict = {
@@ -26,21 +25,18 @@ def run(
else:
kwargs["stdin"] = stdin
return subprocess.run(cmd, **kwargs)
return subprocess.run(cmd, **kwargs) # noqa: PLW1510 - check lives in kwargs
except subprocess.CalledProcessError as e:
msg = f"ERROR: command failed ({e.returncode}): {' '.join(cmd)}"
print(msg, file=sys.stderr)
if e.stdout:
for stream in (e.stdout, e.stderr):
if not stream:
continue
try:
print(e.stdout.decode(), file=sys.stderr)
except Exception:
print(e.stdout, file=sys.stderr)
if e.stderr:
try:
print(e.stderr.decode(), file=sys.stderr)
except Exception:
print(e.stderr, file=sys.stderr)
print(stream.decode(), file=sys.stderr)
except (UnicodeDecodeError, AttributeError):
print(stream, file=sys.stderr)
raise
@@ -50,8 +46,8 @@ def docker_exec(
*,
stdin=None,
capture: bool = False,
env: Optional[dict] = None,
docker_env: Optional[dict[str, str]] = None,
env: dict | None = None,
docker_env: dict[str, str] | None = None,
) -> subprocess.CompletedProcess:
cmd: list[str] = ["docker", "exec", "-i"]
if docker_env:
@@ -67,8 +63,8 @@ def docker_exec_sh(
*,
stdin=None,
capture: bool = False,
env: Optional[dict] = None,
docker_env: Optional[dict[str, str]] = None,
env: dict | None = None,
docker_env: dict[str, str] | None = None,
) -> subprocess.CompletedProcess:
return docker_exec(
container,
@@ -85,5 +81,6 @@ def docker_volume_exists(volume: str) -> bool:
["docker", "volume", "inspect", volume],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False,
)
return p.returncode == 0

9
src/baudolo/seed/__main__.py
View File

@@ -1,18 +1,17 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import os
import re
import sys
import pandas as pd
from typing import Optional
from pandas.errors import EmptyDataError
DB_NAME_RE = re.compile(r"^[a-zA-Z0-9_][a-zA-Z0-9_-]*$")
def _validate_database_value(value: Optional[str], *, instance: str) -> str:
def _validate_database_value(value: str | None, *, instance: str) -> str:
v = (value or "").strip()
if v == "":
raise ValueError(
@@ -40,7 +39,7 @@ def _empty_df() -> pd.DataFrame:
def check_and_add_entry(
file_path: str,
instance: str,
database: Optional[str],
database: str | None,
username: str,
password: str,
) -> None:
@@ -108,7 +107,7 @@ def main() -> None:
username=args.username,
password=args.password,
)
except Exception as exc:
except Exception as exc: # noqa: BLE001 - CLI boundary: any failure becomes exit 1
print(f"ERROR: {exc}", file=sys.stderr)
sys.exit(1)