mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2026-08-20 13:12:48 +00:00
build(lint): gate make test on a clean ruff run
ruff was never wired into this repository: no target, no CI step, no pin. It reported 45 findings across sources and tests, so nothing enforced what the codebase already mostly followed. Adds `make ruff` (check + format --check), `make ruff-fix`, and `make lint` as its alias, and makes `make test` run lint as a fourth parallel spur. The CI workflow calls `make test`, so it is covered there too. The linter is pinned in a `lint` extra: a ruff minor bump changes which rules fire, and with the suite gating on a clean run an unpinned linter would fail it on an unrelated day. The 45 findings are fixed rather than configured away. Three needed a decision instead of the mechanical fix: - The generation timestamp keeps its local wall clock (DTZ005 waived). Generations sort by that name, and UTC would order new ones before the existing ones wherever the offset is positive - "newest generation" is what every restore path selects on. - The per-volume `copy` closure now binds volume_name and vol_dir as default arguments (B023). It only worked because it is called inside the same iteration. - The two CLI top-level handlers keep their blind except (BLE001 waived): turning any failure into exit 1 is what a CLI boundary is for. The two in run.py did not need it and were narrowed to what they actually catch. Also drops the comments that restate the code: the section banners in restore/__main__.py, the filename repeated as line 1 of nine test files, step narration above the statement it narrates, and a block in app.py documenting parameters that had moved to another module. What names a trip-wire stays - the snapshot destination rule, the mysql-binary absence in MariaDB 11 images, the session-scoped FOREIGN_KEY_CHECKS, the spooled temp file for multi-GB dumps, and the negative control that loses its discriminating power if it ever passes. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
27
Makefile
27
Makefile
@@ -1,4 +1,4 @@
|
|||||||
.PHONY: install build clean \
|
.PHONY: install install-lint build clean lint ruff ruff-fix \
|
||||||
test test-unit test-integration test-e2e \
|
test test-unit test-integration test-e2e \
|
||||||
test-unit-run test-integration-run test-e2e-run
|
test-unit-run test-integration-run test-e2e-run
|
||||||
|
|
||||||
@@ -34,13 +34,30 @@ build:
|
|||||||
clean:
|
clean:
|
||||||
git clean -fdX .
|
git clean -fdX .
|
||||||
|
|
||||||
# clean + build run once and in order, then the three suites run concurrently
|
# Separate from `install` so the test image does not have to carry the linter.
|
||||||
# via -j3; the *-run targets carry no clean/build prereq so the sub-make cannot
|
install-lint:
|
||||||
# race a second clean against build.
|
@$(PY_DEFAULT) -m pip install -q -e ".[lint]"
|
||||||
|
|
||||||
|
# Runs on the host, not in the image, so it also covers what the Dockerfile
|
||||||
|
# does not copy.
|
||||||
|
ruff: install-lint
|
||||||
|
@echo ">> Running ruff over the whole repository"
|
||||||
|
@$(PY_DEFAULT) -m ruff check .
|
||||||
|
@$(PY_DEFAULT) -m ruff format --check .
|
||||||
|
|
||||||
|
ruff-fix: install-lint
|
||||||
|
@$(PY_DEFAULT) -m ruff check --fix .
|
||||||
|
@$(PY_DEFAULT) -m ruff format .
|
||||||
|
|
||||||
|
lint: ruff
|
||||||
|
|
||||||
|
# clean + build run once and in order, then lint and the three suites run
|
||||||
|
# concurrently via -j4; the *-run targets carry no clean/build prereq so the
|
||||||
|
# sub-make cannot race a second clean against build.
|
||||||
test:
|
test:
|
||||||
@$(MAKE) clean
|
@$(MAKE) clean
|
||||||
@$(MAKE) build
|
@$(MAKE) build
|
||||||
@$(MAKE) -j3 test-unit-run test-integration-run test-e2e-run
|
@$(MAKE) -j4 lint test-unit-run test-integration-run test-e2e-run
|
||||||
|
|
||||||
test-unit: clean build test-unit-run
|
test-unit: clean build test-unit-run
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,11 @@ dependencies = [
|
|||||||
"dirval",
|
"dirval",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
# Pinned: a ruff minor bump changes which rules fire, and `make test` gates on
|
||||||
|
# a clean run, so an unpinned lint would fail the suite on an unrelated day.
|
||||||
|
lint = ["ruff==0.16.1"]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
baudolo = "baudolo.backup.__main__:main"
|
baudolo = "baudolo.backup.__main__:main"
|
||||||
baudolo-restore = "baudolo.restore.__main__:main"
|
baudolo-restore = "baudolo.restore.__main__:main"
|
||||||
|
|||||||
@@ -1,9 +1,6 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from .app import main
|
from .app import main
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
raise SystemExit(main())
|
raise SystemExit(main())
|
||||||
|
|||||||
@@ -30,17 +30,13 @@ def main() -> int:
|
|||||||
args = parse_args()
|
args = parse_args()
|
||||||
|
|
||||||
machine_id = get_machine_id()
|
machine_id = get_machine_id()
|
||||||
backup_time = datetime.now().strftime("%Y%m%d%H%M%S")
|
# Local wall clock on purpose: generations sort by this name, and UTC would
|
||||||
|
# order new ones before the existing ones wherever the offset is positive.
|
||||||
|
backup_time = datetime.now().strftime("%Y%m%d%H%M%S") # noqa: DTZ005
|
||||||
|
|
||||||
versions_dir = os.path.join(args.backups_dir, machine_id, args.repo_name)
|
versions_dir = os.path.join(args.backups_dir, machine_id, args.repo_name)
|
||||||
version_dir = create_version_directory(versions_dir, backup_time)
|
version_dir = create_version_directory(versions_dir, backup_time)
|
||||||
|
|
||||||
# IMPORTANT:
|
|
||||||
# - keep_default_na=False prevents empty fields from turning into NaN
|
|
||||||
# - dtype=str keeps all columns stable for comparisons/validation
|
|
||||||
#
|
|
||||||
# Robust behavior:
|
|
||||||
# - if the file is missing or empty, we continue without DB dumps.
|
|
||||||
databases_df = load_databases_df(args.databases_csv)
|
databases_df = load_databases_df(args.databases_csv)
|
||||||
|
|
||||||
print("💾 Start volume backups...", flush=True)
|
print("💾 Start volume backups...", flush=True)
|
||||||
@@ -80,24 +76,29 @@ def main() -> int:
|
|||||||
database_containers=args.database_containers,
|
database_containers=args.database_containers,
|
||||||
)
|
)
|
||||||
|
|
||||||
if args.dump_only_sql:
|
if args.dump_only_sql and found_db:
|
||||||
if found_db:
|
if not dumped_any:
|
||||||
if not dumped_any:
|
print(
|
||||||
print(
|
f"WARNING: dump-only-sql requested but no DB dump was produced for DB volume '{volume_name}'. "
|
||||||
f"WARNING: dump-only-sql requested but no DB dump was produced for DB volume '{volume_name}'. "
|
"Falling back to file backup.",
|
||||||
"Falling back to file backup.",
|
flush=True,
|
||||||
flush=True,
|
)
|
||||||
)
|
else:
|
||||||
else:
|
continue
|
||||||
continue
|
|
||||||
|
|
||||||
live_source = get_storage_path(volume_name)
|
live_source = get_storage_path(volume_name)
|
||||||
|
|
||||||
def copy(*, authoritative: bool, source: str = live_source) -> None:
|
def copy(
|
||||||
|
*,
|
||||||
|
authoritative: bool,
|
||||||
|
source: str = live_source,
|
||||||
|
volume: str = volume_name,
|
||||||
|
target: str = vol_dir,
|
||||||
|
) -> None:
|
||||||
backup_volume(
|
backup_volume(
|
||||||
versions_dir,
|
versions_dir,
|
||||||
volume_name,
|
volume,
|
||||||
vol_dir,
|
target,
|
||||||
authoritative=authoritative,
|
authoritative=authoritative,
|
||||||
source=source,
|
source=source,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -4,10 +4,9 @@ import os
|
|||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import List, Optional
|
|
||||||
|
|
||||||
|
|
||||||
def _build_compose_cmd(project_dir: str, passthrough: List[str]) -> List[str]:
|
def _build_compose_cmd(project_dir: str, passthrough: list[str]) -> list[str]:
|
||||||
"""
|
"""
|
||||||
Build the compose command for this project directory.
|
Build the compose command for this project directory.
|
||||||
|
|
||||||
@@ -30,7 +29,7 @@ def _build_compose_cmd(project_dir: str, passthrough: List[str]) -> List[str]:
|
|||||||
raise RuntimeError("Neither 'compose' nor 'docker' found in PATH")
|
raise RuntimeError("Neither 'compose' nor 'docker' found in PATH")
|
||||||
|
|
||||||
|
|
||||||
def _find_compose_file(project_dir: str) -> Optional[Path]:
|
def _find_compose_file(project_dir: str) -> Path | None:
|
||||||
"""
|
"""
|
||||||
Detect a compose file in `project_dir` (case-insensitive).
|
Detect a compose file in `project_dir` (case-insensitive).
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
import re
|
import re
|
||||||
import logging
|
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
import pandas
|
import pandas
|
||||||
|
|
||||||
@@ -22,7 +21,7 @@ def get_instance(container: str, database_containers: list[str]) -> str:
|
|||||||
return re.split(r"(_|-)(database|db|postgres)", container)[0]
|
return re.split(r"(_|-)(database|db|postgres)", container)[0]
|
||||||
|
|
||||||
|
|
||||||
def _validate_database_value(value: Optional[str], *, instance: str) -> str:
|
def _validate_database_value(value: str | None, *, instance: str) -> str:
|
||||||
"""
|
"""
|
||||||
Enforce explicit database semantics:
|
Enforce explicit database semantics:
|
||||||
|
|
||||||
@@ -70,7 +69,7 @@ def backup_database(
|
|||||||
container: str,
|
container: str,
|
||||||
volume_dir: str,
|
volume_dir: str,
|
||||||
db_type: str,
|
db_type: str,
|
||||||
databases_df: "pandas.DataFrame",
|
databases_df: pandas.DataFrame,
|
||||||
database_containers: list[str],
|
database_containers: list[str],
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""
|
"""
|
||||||
@@ -97,7 +96,6 @@ def backup_database(
|
|||||||
|
|
||||||
db_value = _validate_database_value(raw_db, instance=instance_name)
|
db_value = _validate_database_value(raw_db, instance=instance_name)
|
||||||
|
|
||||||
# Explicit: dump ALL databases
|
|
||||||
if db_value == "*":
|
if db_value == "*":
|
||||||
if db_type != "postgres":
|
if db_type != "postgres":
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
@@ -110,7 +108,6 @@ def backup_database(
|
|||||||
produced = True
|
produced = True
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Concrete database dump
|
|
||||||
db_name = db_value
|
db_name = db_value
|
||||||
dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
|
dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
|
||||||
|
|
||||||
@@ -135,7 +132,6 @@ def backup_database(
|
|||||||
_atomic_write_cmd(cmd, dump_file)
|
_atomic_write_cmd(cmd, dump_file)
|
||||||
produced = True
|
produced = True
|
||||||
except BackupException as e:
|
except BackupException as e:
|
||||||
# Explicit DB dump failed -> hard error
|
|
||||||
raise BackupException(
|
raise BackupException(
|
||||||
f"Postgres dump failed for instance '{instance_name}', "
|
f"Postgres dump failed for instance '{instance_name}', "
|
||||||
f"database '{db_name}'. This database was explicitly configured "
|
f"database '{db_name}'. This database was explicitly configured "
|
||||||
|
|||||||
@@ -98,5 +98,5 @@ def docker_volume_exists(volume: str) -> bool:
|
|||||||
f"docker volume inspect {volume} >/dev/null 2>&1 && echo OK"
|
f"docker volume inspect {volume} >/dev/null 2>&1 && echo OK"
|
||||||
)
|
)
|
||||||
return True
|
return True
|
||||||
except Exception:
|
except BackupException:
|
||||||
return False
|
return False
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ def backup_mariadb_or_postgres(
|
|||||||
*,
|
*,
|
||||||
container: str,
|
container: str,
|
||||||
volume_dir: str,
|
volume_dir: str,
|
||||||
databases_df: "pandas.DataFrame",
|
databases_df: pandas.DataFrame,
|
||||||
database_containers: list[str],
|
database_containers: list[str],
|
||||||
) -> tuple[bool, bool]:
|
) -> tuple[bool, bool]:
|
||||||
"""
|
"""
|
||||||
@@ -34,7 +34,7 @@ def backup_mariadb_or_postgres(
|
|||||||
return False, False
|
return False, False
|
||||||
|
|
||||||
|
|
||||||
def _empty_databases_df() -> "pandas.DataFrame":
|
def _empty_databases_df() -> pandas.DataFrame:
|
||||||
"""
|
"""
|
||||||
Create an empty DataFrame with the expected schema for databases.csv.
|
Create an empty DataFrame with the expected schema for databases.csv.
|
||||||
|
|
||||||
@@ -44,7 +44,7 @@ def _empty_databases_df() -> "pandas.DataFrame":
|
|||||||
return pandas.DataFrame(columns=["instance", "database", "username", "password"])
|
return pandas.DataFrame(columns=["instance", "database", "username", "password"])
|
||||||
|
|
||||||
|
|
||||||
def load_databases_df(csv_path: str) -> "pandas.DataFrame":
|
def load_databases_df(csv_path: str) -> pandas.DataFrame:
|
||||||
"""
|
"""
|
||||||
Load databases.csv robustly.
|
Load databases.csv robustly.
|
||||||
|
|
||||||
@@ -74,7 +74,7 @@ def backup_dumps_for_volume(
|
|||||||
*,
|
*,
|
||||||
containers: list[str],
|
containers: list[str],
|
||||||
vol_dir: str,
|
vol_dir: str,
|
||||||
databases_df: "pandas.DataFrame",
|
databases_df: pandas.DataFrame,
|
||||||
database_containers: list[str],
|
database_containers: list[str],
|
||||||
) -> tuple[bool, bool]:
|
) -> tuple[bool, bool]:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -34,9 +34,6 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
)
|
)
|
||||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
# files
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
p_files = sub.add_parser("files", help="Restore files into a docker volume")
|
p_files = sub.add_parser("files", help="Restore files into a docker volume")
|
||||||
_add_common_backup_args(p_files)
|
_add_common_backup_args(p_files)
|
||||||
p_files.add_argument(
|
p_files.add_argument(
|
||||||
@@ -49,9 +46,6 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
# postgres
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
p_pg = sub.add_parser("postgres", help="Restore a single PostgreSQL database dump")
|
p_pg = sub.add_parser("postgres", help="Restore a single PostgreSQL database dump")
|
||||||
_add_common_backup_args(p_pg)
|
_add_common_backup_args(p_pg)
|
||||||
p_pg.add_argument("--container", required=True)
|
p_pg.add_argument("--container", required=True)
|
||||||
@@ -60,9 +54,6 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
p_pg.add_argument("--db-password", required=True)
|
p_pg.add_argument("--db-password", required=True)
|
||||||
p_pg.add_argument("--empty", action="store_true")
|
p_pg.add_argument("--empty", action="store_true")
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
# cluster
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
p_cluster = sub.add_parser(
|
p_cluster = sub.add_parser(
|
||||||
"cluster", help="Restore a full PostgreSQL cluster dump (pg_dumpall)"
|
"cluster", help="Restore a full PostgreSQL cluster dump (pg_dumpall)"
|
||||||
)
|
)
|
||||||
@@ -81,9 +72,6 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
p_cluster.add_argument("--db-password", required=True)
|
p_cluster.add_argument("--db-password", required=True)
|
||||||
p_cluster.add_argument("--empty", action="store_true")
|
p_cluster.add_argument("--empty", action="store_true")
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
# mariadb
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
p_mdb = sub.add_parser(
|
p_mdb = sub.add_parser(
|
||||||
"mariadb", help="Restore a single MariaDB/MySQL-compatible dump"
|
"mariadb", help="Restore a single MariaDB/MySQL-compatible dump"
|
||||||
)
|
)
|
||||||
@@ -98,8 +86,6 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
if args.cmd == "files":
|
if args.cmd == "files":
|
||||||
# target volume = args.volume_name
|
|
||||||
# source volume (backup key) defaults to target volume
|
|
||||||
source_volume = args.source_volume or args.volume_name
|
source_volume = args.source_volume or args.volume_name
|
||||||
|
|
||||||
bp_files = BackupPaths(
|
bp_files = BackupPaths(
|
||||||
@@ -170,7 +156,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
parser.error("Unhandled command")
|
parser.error("Unhandled command")
|
||||||
return 2
|
return 2
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e: # noqa: BLE001 - CLI boundary: any failure becomes exit 1
|
||||||
print(f"ERROR: {e}", file=sys.stderr)
|
print(f"ERROR: {e}", file=sys.stderr)
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
|||||||
@@ -22,11 +22,11 @@ exit 42
|
|||||||
if not out:
|
if not out:
|
||||||
raise RuntimeError("empty client detection output")
|
raise RuntimeError("empty client detection output")
|
||||||
return out
|
return out
|
||||||
except Exception as e:
|
except Exception:
|
||||||
print(
|
print(
|
||||||
"ERROR: neither 'mariadb' nor 'mysql' found in container.", file=sys.stderr
|
"ERROR: neither 'mariadb' nor 'mysql' found in container.", file=sys.stderr
|
||||||
)
|
)
|
||||||
raise e
|
raise
|
||||||
|
|
||||||
|
|
||||||
def restore_mariadb_sql(
|
def restore_mariadb_sql(
|
||||||
@@ -44,9 +44,7 @@ def restore_mariadb_sql(
|
|||||||
raise FileNotFoundError(sql_path)
|
raise FileNotFoundError(sql_path)
|
||||||
|
|
||||||
if empty:
|
if empty:
|
||||||
# IMPORTANT:
|
# Do not hardcode 'mysql': MariaDB 11 images may not ship that binary.
|
||||||
# Do NOT hardcode 'mysql' here. Use the detected client.
|
|
||||||
# MariaDB 11 images may not contain the mysql binary at all.
|
|
||||||
result = docker_exec(
|
result = docker_exec(
|
||||||
container,
|
container,
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -50,7 +50,6 @@ def restore_postgres_sql(
|
|||||||
if not os.path.isfile(sql_path):
|
if not os.path.isfile(sql_path):
|
||||||
raise FileNotFoundError(sql_path)
|
raise FileNotFoundError(sql_path)
|
||||||
|
|
||||||
# Make password available INSIDE the container for psql.
|
|
||||||
docker_env = {"PGPASSWORD": password}
|
docker_env = {"PGPASSWORD": password}
|
||||||
|
|
||||||
if empty:
|
if empty:
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
|
|
||||||
def run(
|
def run(
|
||||||
@@ -10,7 +9,7 @@ def run(
|
|||||||
*,
|
*,
|
||||||
stdin=None,
|
stdin=None,
|
||||||
capture: bool = False,
|
capture: bool = False,
|
||||||
env: Optional[dict] = None,
|
env: dict | None = None,
|
||||||
) -> subprocess.CompletedProcess:
|
) -> subprocess.CompletedProcess:
|
||||||
try:
|
try:
|
||||||
kwargs: dict = {
|
kwargs: dict = {
|
||||||
@@ -26,21 +25,18 @@ def run(
|
|||||||
else:
|
else:
|
||||||
kwargs["stdin"] = stdin
|
kwargs["stdin"] = stdin
|
||||||
|
|
||||||
return subprocess.run(cmd, **kwargs)
|
return subprocess.run(cmd, **kwargs) # noqa: PLW1510 - check lives in kwargs
|
||||||
|
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
msg = f"ERROR: command failed ({e.returncode}): {' '.join(cmd)}"
|
msg = f"ERROR: command failed ({e.returncode}): {' '.join(cmd)}"
|
||||||
print(msg, file=sys.stderr)
|
print(msg, file=sys.stderr)
|
||||||
if e.stdout:
|
for stream in (e.stdout, e.stderr):
|
||||||
|
if not stream:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
print(e.stdout.decode(), file=sys.stderr)
|
print(stream.decode(), file=sys.stderr)
|
||||||
except Exception:
|
except (UnicodeDecodeError, AttributeError):
|
||||||
print(e.stdout, file=sys.stderr)
|
print(stream, file=sys.stderr)
|
||||||
if e.stderr:
|
|
||||||
try:
|
|
||||||
print(e.stderr.decode(), file=sys.stderr)
|
|
||||||
except Exception:
|
|
||||||
print(e.stderr, file=sys.stderr)
|
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
@@ -50,8 +46,8 @@ def docker_exec(
|
|||||||
*,
|
*,
|
||||||
stdin=None,
|
stdin=None,
|
||||||
capture: bool = False,
|
capture: bool = False,
|
||||||
env: Optional[dict] = None,
|
env: dict | None = None,
|
||||||
docker_env: Optional[dict[str, str]] = None,
|
docker_env: dict[str, str] | None = None,
|
||||||
) -> subprocess.CompletedProcess:
|
) -> subprocess.CompletedProcess:
|
||||||
cmd: list[str] = ["docker", "exec", "-i"]
|
cmd: list[str] = ["docker", "exec", "-i"]
|
||||||
if docker_env:
|
if docker_env:
|
||||||
@@ -67,8 +63,8 @@ def docker_exec_sh(
|
|||||||
*,
|
*,
|
||||||
stdin=None,
|
stdin=None,
|
||||||
capture: bool = False,
|
capture: bool = False,
|
||||||
env: Optional[dict] = None,
|
env: dict | None = None,
|
||||||
docker_env: Optional[dict[str, str]] = None,
|
docker_env: dict[str, str] | None = None,
|
||||||
) -> subprocess.CompletedProcess:
|
) -> subprocess.CompletedProcess:
|
||||||
return docker_exec(
|
return docker_exec(
|
||||||
container,
|
container,
|
||||||
@@ -85,5 +81,6 @@ def docker_volume_exists(volume: str) -> bool:
|
|||||||
["docker", "volume", "inspect", volume],
|
["docker", "volume", "inspect", volume],
|
||||||
stdout=subprocess.DEVNULL,
|
stdout=subprocess.DEVNULL,
|
||||||
stderr=subprocess.DEVNULL,
|
stderr=subprocess.DEVNULL,
|
||||||
|
check=False,
|
||||||
)
|
)
|
||||||
return p.returncode == 0
|
return p.returncode == 0
|
||||||
|
|||||||
@@ -1,18 +1,17 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from typing import Optional
|
|
||||||
from pandas.errors import EmptyDataError
|
from pandas.errors import EmptyDataError
|
||||||
|
|
||||||
DB_NAME_RE = re.compile(r"^[a-zA-Z0-9_][a-zA-Z0-9_-]*$")
|
DB_NAME_RE = re.compile(r"^[a-zA-Z0-9_][a-zA-Z0-9_-]*$")
|
||||||
|
|
||||||
|
|
||||||
def _validate_database_value(value: Optional[str], *, instance: str) -> str:
|
def _validate_database_value(value: str | None, *, instance: str) -> str:
|
||||||
v = (value or "").strip()
|
v = (value or "").strip()
|
||||||
if v == "":
|
if v == "":
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
@@ -40,7 +39,7 @@ def _empty_df() -> pd.DataFrame:
|
|||||||
def check_and_add_entry(
|
def check_and_add_entry(
|
||||||
file_path: str,
|
file_path: str,
|
||||||
instance: str,
|
instance: str,
|
||||||
database: Optional[str],
|
database: str | None,
|
||||||
username: str,
|
username: str,
|
||||||
password: str,
|
password: str,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -108,7 +107,7 @@ def main() -> None:
|
|||||||
username=args.username,
|
username=args.username,
|
||||||
password=args.password,
|
password=args.password,
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc: # noqa: BLE001 - CLI boundary: any failure becomes exit 1
|
||||||
print(f"ERROR: {exc}", file=sys.stderr)
|
print(f"ERROR: {exc}", file=sys.stderr)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""Shared e2e helpers, re-exported so tests import one name."""
|
"""Shared e2e helpers, re-exported so tests import one name."""
|
||||||
|
|
||||||
from .fixtures import * # noqa: F401,F403
|
from .fixtures import *
|
||||||
from .process import * # noqa: F401,F403
|
from .process import *
|
||||||
|
|||||||
@@ -97,8 +97,7 @@ def write_databases_csv(path: str, rows: list[tuple[str, str, str, str]]) -> Non
|
|||||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
with open(path, "w", encoding="utf-8") as f:
|
with open(path, "w", encoding="utf-8") as f:
|
||||||
f.write("instance;database;username;password\n")
|
f.write("instance;database;username;password\n")
|
||||||
for inst, db, user, pw in rows:
|
f.writelines(f"{inst};{db};{user};{pw}\n" for inst, db, user, pw in rows)
|
||||||
f.write(f"{inst};{db};{user};{pw}\n")
|
|
||||||
|
|
||||||
|
|
||||||
def cleanup_docker(*, containers: list[str], volumes: list[str]) -> None:
|
def cleanup_docker(*, containers: list[str], volumes: list[str]) -> None:
|
||||||
|
|||||||
@@ -12,8 +12,8 @@ import sys
|
|||||||
|
|
||||||
sys.path.insert(0, "/src")
|
sys.path.insert(0, "/src")
|
||||||
|
|
||||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot # noqa: E402
|
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
||||||
from baudolo.backup.volume import backup_volume # noqa: E402
|
from baudolo.backup.volume import backup_volume
|
||||||
|
|
||||||
SUBJECT = "/subject/docker"
|
SUBJECT = "/subject/docker"
|
||||||
VOLUME = "mariadb_data"
|
VOLUME = "mariadb_data"
|
||||||
@@ -23,7 +23,9 @@ GENERATION = f"{VERSIONS}/20260731"
|
|||||||
|
|
||||||
|
|
||||||
def shell(command: str) -> list[str]:
|
def shell(command: str) -> list[str]:
|
||||||
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
proc = subprocess.run(
|
||||||
|
command, shell=True, capture_output=True, text=True, check=False
|
||||||
|
)
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
raise SnapshotError(
|
raise SnapshotError(
|
||||||
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from pathlib import Path
|
|||||||
|
|
||||||
sys.path.insert(0, "/src")
|
sys.path.insert(0, "/src")
|
||||||
|
|
||||||
from baudolo.backup.snapshot import SnapshotError, volume_snapshot # noqa: E402
|
from baudolo.backup.snapshot import SnapshotError, volume_snapshot
|
||||||
|
|
||||||
KIND = sys.argv[1]
|
KIND = sys.argv[1]
|
||||||
SUBJECT = sys.argv[2]
|
SUBJECT = sys.argv[2]
|
||||||
@@ -20,7 +20,9 @@ EXPECT = sys.argv[3]
|
|||||||
|
|
||||||
|
|
||||||
def shell(command: str) -> list[str]:
|
def shell(command: str) -> list[str]:
|
||||||
proc = subprocess.run(command, shell=True, capture_output=True, text=True)
|
proc = subprocess.run(
|
||||||
|
command, shell=True, capture_output=True, text=True, check=False
|
||||||
|
)
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
raise SnapshotError(
|
raise SnapshotError(
|
||||||
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
# tests/e2e/test_e2e_dump_only_fallback_to_files.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
POSTGRES_IMAGE,
|
|
||||||
POSTGRES_DATA_DIR,
|
POSTGRES_DATA_DIR,
|
||||||
|
POSTGRES_IMAGE,
|
||||||
backup_path,
|
backup_path,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
@@ -12,8 +11,8 @@ from .helpers import (
|
|||||||
require_docker,
|
require_docker,
|
||||||
run,
|
run,
|
||||||
unique,
|
unique,
|
||||||
write_databases_csv,
|
|
||||||
wait_for_postgres,
|
wait_for_postgres,
|
||||||
|
write_databases_csv,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -37,7 +36,6 @@ class TestE2EDumpOnlyFallbackToFiles(unittest.TestCase):
|
|||||||
|
|
||||||
run(["docker", "volume", "create", cls.pg_volume])
|
run(["docker", "volume", "create", cls.pg_volume])
|
||||||
|
|
||||||
# Start Postgres (creates a real DB volume)
|
|
||||||
run(
|
run(
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
POSTGRES_IMAGE,
|
|
||||||
POSTGRES_DATA_DIR,
|
POSTGRES_DATA_DIR,
|
||||||
|
POSTGRES_IMAGE,
|
||||||
backup_path,
|
backup_path,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
@@ -35,7 +35,6 @@ class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
|
|||||||
cls.containers: list[str] = []
|
cls.containers: list[str] = []
|
||||||
cls.volumes = [cls.db_volume, cls.files_volume]
|
cls.volumes = [cls.db_volume, cls.files_volume]
|
||||||
|
|
||||||
# Create volumes
|
|
||||||
run(["docker", "volume", "create", cls.db_volume])
|
run(["docker", "volume", "create", cls.db_volume])
|
||||||
run(["docker", "volume", "create", cls.files_volume])
|
run(["docker", "volume", "create", cls.files_volume])
|
||||||
|
|
||||||
@@ -114,7 +113,6 @@ class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
|
|||||||
[(cls.pg_container, cls.pg_db, cls.pg_user, cls.pg_password)],
|
[(cls.pg_container, cls.pg_db, cls.pg_user, cls.pg_password)],
|
||||||
)
|
)
|
||||||
|
|
||||||
# Run baudolo with dump-only-sql
|
|
||||||
cmd = [
|
cmd = [
|
||||||
"baudolo",
|
"baudolo",
|
||||||
"--compose-dir",
|
"--compose-dir",
|
||||||
|
|||||||
@@ -1,19 +1,19 @@
|
|||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
POSTGRES_IMAGE,
|
|
||||||
POSTGRES_DATA_DIR,
|
POSTGRES_DATA_DIR,
|
||||||
backup_run,
|
POSTGRES_IMAGE,
|
||||||
backup_path,
|
backup_path,
|
||||||
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
ensure_empty_dir,
|
ensure_empty_dir,
|
||||||
latest_version_dir,
|
latest_version_dir,
|
||||||
require_docker,
|
require_docker,
|
||||||
unique,
|
|
||||||
write_databases_csv,
|
|
||||||
run,
|
run,
|
||||||
|
unique,
|
||||||
wait_for_postgres,
|
wait_for_postgres,
|
||||||
|
write_databases_csv,
|
||||||
)
|
)
|
||||||
|
|
||||||
REGISTRY_HOST = "svc-db-mariadb-swarm-mgr-01:5000"
|
REGISTRY_HOST = "svc-db-mariadb-swarm-mgr-01:5000"
|
||||||
|
|||||||
@@ -1,16 +1,16 @@
|
|||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
backup_run,
|
|
||||||
backup_path,
|
backup_path,
|
||||||
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
ensure_empty_dir,
|
ensure_empty_dir,
|
||||||
latest_version_dir,
|
latest_version_dir,
|
||||||
require_docker,
|
require_docker,
|
||||||
|
run,
|
||||||
unique,
|
unique,
|
||||||
write_databases_csv,
|
write_databases_csv,
|
||||||
run,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -30,7 +30,6 @@ class TestE2EFilesFull(unittest.TestCase):
|
|||||||
cls.containers = []
|
cls.containers = []
|
||||||
cls.volumes = [cls.volume_src, cls.volume_dst]
|
cls.volumes = [cls.volume_src, cls.volume_dst]
|
||||||
|
|
||||||
# create source volume with a file
|
|
||||||
run(["docker", "volume", "create", cls.volume_src])
|
run(["docker", "volume", "create", cls.volume_src])
|
||||||
run(
|
run(
|
||||||
[
|
[
|
||||||
@@ -50,7 +49,6 @@ class TestE2EFilesFull(unittest.TestCase):
|
|||||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
write_databases_csv(cls.databases_csv, [])
|
write_databases_csv(cls.databases_csv, [])
|
||||||
|
|
||||||
# Run backup (files should be copied)
|
|
||||||
backup_run(
|
backup_run(
|
||||||
backups_dir=cls.backups_dir,
|
backups_dir=cls.backups_dir,
|
||||||
repo_name=cls.repo_name,
|
repo_name=cls.repo_name,
|
||||||
@@ -97,7 +95,6 @@ class TestE2EFilesFull(unittest.TestCase):
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
# verify restored file exists in dst volume
|
|
||||||
p = run(
|
p = run(
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
|
|||||||
@@ -1,16 +1,16 @@
|
|||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
backup_run,
|
|
||||||
backup_path,
|
backup_path,
|
||||||
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
ensure_empty_dir,
|
ensure_empty_dir,
|
||||||
latest_version_dir,
|
latest_version_dir,
|
||||||
require_docker,
|
require_docker,
|
||||||
|
run,
|
||||||
unique,
|
unique,
|
||||||
write_databases_csv,
|
write_databases_csv,
|
||||||
run,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -29,7 +29,6 @@ class TestE2EFilesNoCopy(unittest.TestCase):
|
|||||||
cls.containers: list[str] = []
|
cls.containers: list[str] = []
|
||||||
cls.volumes = [cls.volume_src]
|
cls.volumes = [cls.volume_src]
|
||||||
|
|
||||||
# Create source volume and write a marker file
|
|
||||||
run(["docker", "volume", "create", cls.volume_src])
|
run(["docker", "volume", "create", cls.volume_src])
|
||||||
run(
|
run(
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
# tests/e2e/test_e2e_images_no_backup_required_early_skip.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
@@ -34,11 +33,9 @@ class TestE2EImagesNoBackupRequiredEarlySkip(unittest.TestCase):
|
|||||||
cls.containers = [cls.redis_container]
|
cls.containers = [cls.redis_container]
|
||||||
cls.volumes = [cls.ignored_volume, cls.normal_volume]
|
cls.volumes = [cls.ignored_volume, cls.normal_volume]
|
||||||
|
|
||||||
# Create volumes
|
|
||||||
run(["docker", "volume", "create", cls.ignored_volume])
|
run(["docker", "volume", "create", cls.ignored_volume])
|
||||||
run(["docker", "volume", "create", cls.normal_volume])
|
run(["docker", "volume", "create", cls.normal_volume])
|
||||||
|
|
||||||
# Start redis container using the ignored volume
|
|
||||||
run(
|
run(
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
@@ -71,7 +68,6 @@ class TestE2EImagesNoBackupRequiredEarlySkip(unittest.TestCase):
|
|||||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
write_databases_csv(cls.databases_csv, [])
|
write_databases_csv(cls.databases_csv, [])
|
||||||
|
|
||||||
# Run baudolo with images-no-backup-required redis
|
|
||||||
cmd = [
|
cmd = [
|
||||||
"baudolo",
|
"baudolo",
|
||||||
"--compose-dir",
|
"--compose-dir",
|
||||||
|
|||||||
@@ -30,8 +30,8 @@ import pandas
|
|||||||
from baudolo.backup import db as db_mod
|
from baudolo.backup import db as db_mod
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
MARIADB_IMAGE,
|
|
||||||
MARIADB_DATA_DIR,
|
MARIADB_DATA_DIR,
|
||||||
|
MARIADB_IMAGE,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
require_docker,
|
require_docker,
|
||||||
run,
|
run,
|
||||||
|
|||||||
@@ -1,21 +1,20 @@
|
|||||||
# tests/e2e/test_e2e_mariadb_full.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
MARIADB_IMAGE,
|
|
||||||
MARIADB_DATA_DIR,
|
MARIADB_DATA_DIR,
|
||||||
backup_run,
|
MARIADB_IMAGE,
|
||||||
backup_path,
|
backup_path,
|
||||||
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
ensure_empty_dir,
|
ensure_empty_dir,
|
||||||
latest_version_dir,
|
latest_version_dir,
|
||||||
require_docker,
|
require_docker,
|
||||||
unique,
|
|
||||||
write_databases_csv,
|
|
||||||
run,
|
run,
|
||||||
|
unique,
|
||||||
wait_for_mariadb,
|
wait_for_mariadb,
|
||||||
wait_for_mariadb_sql,
|
wait_for_mariadb_sql,
|
||||||
|
write_databases_csv,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -71,7 +70,6 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
|||||||
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
||||||
)
|
)
|
||||||
|
|
||||||
# Create table + data via the dedicated user (TCP)
|
|
||||||
run(
|
run(
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
@@ -79,9 +77,11 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
|||||||
cls.db_container,
|
cls.db_container,
|
||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
(
|
||||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||||
|
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\""
|
||||||
|
),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -112,8 +112,10 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
|||||||
cls.db_container,
|
cls.db_container,
|
||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
(
|
||||||
f'-e "DROP TABLE {cls.db_name}.t;"',
|
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||||
|
f'-e "DROP TABLE {cls.db_name}.t;"'
|
||||||
|
),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -161,8 +163,10 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
|||||||
self.db_container,
|
self.db_container,
|
||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
(
|
||||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"',
|
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||||
|
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"'
|
||||||
|
),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||||
|
|||||||
@@ -1,21 +1,20 @@
|
|||||||
# tests/e2e/test_e2e_mariadb_no_copy.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
MARIADB_IMAGE,
|
|
||||||
MARIADB_DATA_DIR,
|
MARIADB_DATA_DIR,
|
||||||
backup_run,
|
MARIADB_IMAGE,
|
||||||
backup_path,
|
backup_path,
|
||||||
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
ensure_empty_dir,
|
ensure_empty_dir,
|
||||||
latest_version_dir,
|
latest_version_dir,
|
||||||
require_docker,
|
require_docker,
|
||||||
unique,
|
|
||||||
write_databases_csv,
|
|
||||||
run,
|
run,
|
||||||
|
unique,
|
||||||
wait_for_mariadb,
|
wait_for_mariadb,
|
||||||
wait_for_mariadb_sql,
|
wait_for_mariadb_sql,
|
||||||
|
write_databases_csv,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -69,7 +68,6 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
|||||||
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
||||||
)
|
)
|
||||||
|
|
||||||
# Create table + data (TCP)
|
|
||||||
run(
|
run(
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
@@ -77,9 +75,11 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
|||||||
cls.db_container,
|
cls.db_container,
|
||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
(
|
||||||
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||||
|
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\""
|
||||||
|
),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -110,8 +110,10 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
|||||||
cls.db_container,
|
cls.db_container,
|
||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
(
|
||||||
f'-e "DROP TABLE {cls.db_name}.t;"',
|
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||||
|
f'-e "DROP TABLE {cls.db_name}.t;"'
|
||||||
|
),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -158,8 +160,10 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
|||||||
self.db_container,
|
self.db_container,
|
||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
(
|
||||||
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"',
|
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||||
|
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"'
|
||||||
|
),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||||
|
|||||||
@@ -16,15 +16,8 @@ from .helpers import (
|
|||||||
write_databases_csv,
|
write_databases_csv,
|
||||||
)
|
)
|
||||||
|
|
||||||
# A `database = '*'` row makes the backup side write one pg_dumpall stream for
|
# One statement per entry: psql wraps a multi-statement -c in a transaction,
|
||||||
# the whole instance instead of a dump per database - the shape an application
|
# and CREATE DATABASE is forbidden inside one.
|
||||||
# with several databases in one engine produces. This proves the stream is
|
|
||||||
# replayable: two databases and their owning role are dropped outright, and the
|
|
||||||
# cluster restore has to bring all three back. Before the cluster subcommand
|
|
||||||
# existed the dump was stored and unreadable.
|
|
||||||
# Each statement runs on its own: psql wraps a multi-statement -c in one
|
|
||||||
# transaction, and CREATE DATABASE is forbidden inside one - the same rule that
|
|
||||||
# keeps the cluster replay out of --single-transaction.
|
|
||||||
SEED_SQL = (
|
SEED_SQL = (
|
||||||
"CREATE ROLE app LOGIN PASSWORD 'apppw'",
|
"CREATE ROLE app LOGIN PASSWORD 'apppw'",
|
||||||
"CREATE DATABASE first OWNER app",
|
"CREATE DATABASE first OWNER app",
|
||||||
@@ -96,7 +89,6 @@ class TestE2EPostgresClusterRestore(unittest.TestCase):
|
|||||||
/ f"{cls.pg_container}.cluster.backup.sql"
|
/ f"{cls.pg_container}.cluster.backup.sql"
|
||||||
)
|
)
|
||||||
|
|
||||||
# The disaster: both databases and the role that owns them are gone.
|
|
||||||
for statement in DROP_SQL:
|
for statement in DROP_SQL:
|
||||||
cls._psql("postgres", statement)
|
cls._psql("postgres", statement)
|
||||||
|
|
||||||
@@ -156,9 +148,6 @@ class TestE2EPostgresClusterRestore(unittest.TestCase):
|
|||||||
self.assertEqual(self._psql("second", "SELECT v FROM t"), "second-payload")
|
self.assertEqual(self._psql("second", "SELECT v FROM t"), "second-payload")
|
||||||
|
|
||||||
def test_the_superusers_own_create_was_filtered(self) -> None:
|
def test_the_superusers_own_create_was_filtered(self) -> None:
|
||||||
# The dump recreates every role including the one the replay connects
|
|
||||||
# as; only its ALTER may survive, or the stream dies on the first
|
|
||||||
# statement with ON_ERROR_STOP.
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self._psql(
|
self._psql(
|
||||||
"postgres", "SELECT rolsuper FROM pg_roles WHERE rolname = 'postgres'"
|
"postgres", "SELECT rolsuper FROM pg_roles WHERE rolname = 'postgres'"
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
# tests/e2e/test_e2e_postgres_empty_drop_hard.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
POSTGRES_IMAGE,
|
|
||||||
POSTGRES_DATA_DIR,
|
POSTGRES_DATA_DIR,
|
||||||
|
POSTGRES_IMAGE,
|
||||||
backup_run,
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
|
|||||||
@@ -1,20 +1,19 @@
|
|||||||
# tests/e2e/test_e2e_postgres_full.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
POSTGRES_IMAGE,
|
|
||||||
POSTGRES_DATA_DIR,
|
POSTGRES_DATA_DIR,
|
||||||
backup_run,
|
POSTGRES_IMAGE,
|
||||||
backup_path,
|
backup_path,
|
||||||
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
ensure_empty_dir,
|
ensure_empty_dir,
|
||||||
latest_version_dir,
|
latest_version_dir,
|
||||||
require_docker,
|
require_docker,
|
||||||
unique,
|
|
||||||
write_databases_csv,
|
|
||||||
run,
|
run,
|
||||||
|
unique,
|
||||||
wait_for_postgres,
|
wait_for_postgres,
|
||||||
|
write_databases_csv,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -55,7 +54,6 @@ class TestE2EPostgresFull(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||||
|
|
||||||
# Create a table + data
|
|
||||||
run(
|
run(
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
|
|||||||
@@ -1,20 +1,19 @@
|
|||||||
# tests/e2e/test_e2e_postgres_no_copy.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
POSTGRES_IMAGE,
|
|
||||||
POSTGRES_DATA_DIR,
|
POSTGRES_DATA_DIR,
|
||||||
backup_run,
|
POSTGRES_IMAGE,
|
||||||
backup_path,
|
backup_path,
|
||||||
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
ensure_empty_dir,
|
ensure_empty_dir,
|
||||||
latest_version_dir,
|
latest_version_dir,
|
||||||
require_docker,
|
require_docker,
|
||||||
unique,
|
|
||||||
write_databases_csv,
|
|
||||||
run,
|
run,
|
||||||
|
unique,
|
||||||
wait_for_postgres,
|
wait_for_postgres,
|
||||||
|
write_databases_csv,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
# tests/e2e/test_e2e_postgres_single_transaction_live_writer.py
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
POSTGRES_IMAGE,
|
|
||||||
POSTGRES_DATA_DIR,
|
POSTGRES_DATA_DIR,
|
||||||
|
POSTGRES_IMAGE,
|
||||||
backup_run,
|
backup_run,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
POSTGRES_IMAGE,
|
|
||||||
POSTGRES_DATA_DIR,
|
POSTGRES_DATA_DIR,
|
||||||
|
POSTGRES_IMAGE,
|
||||||
backup_path,
|
backup_path,
|
||||||
cleanup_docker,
|
cleanup_docker,
|
||||||
create_minimal_compose_dir,
|
create_minimal_compose_dir,
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
# tests/e2e/test_e2e_swarm_task_skip.py
|
|
||||||
#
|
|
||||||
# Reproduces the swarm flake fixed on this branch: baudolo used to stop a
|
# Reproduces the swarm flake fixed on this branch: baudolo used to stop a
|
||||||
# swarm task container around the volume file backup because its image was
|
# swarm task container around the volume file backup because its image was
|
||||||
# not whitelisted; the orchestrator immediately replaced the stopped task and
|
# not whitelisted; the orchestrator immediately replaced the stopped task and
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ import pandas as pd
|
|||||||
# Adjust if your package name/import path differs.
|
# Adjust if your package name/import path differs.
|
||||||
from baudolo.backup.dumps import load_databases_df
|
from baudolo.backup.dumps import load_databases_df
|
||||||
|
|
||||||
|
|
||||||
EXPECTED_COLUMNS = ["instance", "database", "username", "password"]
|
EXPECTED_COLUMNS = ["instance", "database", "username", "password"]
|
||||||
|
|
||||||
|
|
||||||
@@ -33,7 +32,6 @@ class TestLoadDatabasesDf(unittest.TestCase):
|
|||||||
def test_empty_csv_is_handled_with_warning_and_empty_df(self) -> None:
|
def test_empty_csv_is_handled_with_warning_and_empty_df(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as td:
|
with tempfile.TemporaryDirectory() as td:
|
||||||
empty_path = os.path.join(td, "databases.csv")
|
empty_path = os.path.join(td, "databases.csv")
|
||||||
# Create an empty file (0 bytes)
|
|
||||||
with open(empty_path, "w", encoding="utf-8") as f:
|
with open(empty_path, "w", encoding="utf-8") as f:
|
||||||
f.write("")
|
f.write("")
|
||||||
|
|
||||||
|
|||||||
@@ -69,14 +69,18 @@ class TestSnapshotFlags(unittest.TestCase):
|
|||||||
|
|
||||||
class TestRequiredFlags(unittest.TestCase):
|
class TestRequiredFlags(unittest.TestCase):
|
||||||
def test_backups_dir_is_required(self) -> None:
|
def test_backups_dir_is_required(self) -> None:
|
||||||
with mock.patch("sys.argv", ["baudolo", "--compose-dir", "/compose"]):
|
with (
|
||||||
with self.assertRaises(SystemExit):
|
mock.patch("sys.argv", ["baudolo", "--compose-dir", "/compose"]),
|
||||||
parse_args()
|
self.assertRaises(SystemExit),
|
||||||
|
):
|
||||||
|
parse_args()
|
||||||
|
|
||||||
def test_compose_dir_is_required(self) -> None:
|
def test_compose_dir_is_required(self) -> None:
|
||||||
with mock.patch("sys.argv", ["baudolo", "--backups-dir", "/backups"]):
|
with (
|
||||||
with self.assertRaises(SystemExit):
|
mock.patch("sys.argv", ["baudolo", "--backups-dir", "/backups"]),
|
||||||
parse_args()
|
self.assertRaises(SystemExit),
|
||||||
|
):
|
||||||
|
parse_args()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ from __future__ import annotations
|
|||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import List
|
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from .compose_fixture import setup_compose_dir as _setup_compose_dir
|
from .compose_fixture import setup_compose_dir as _setup_compose_dir
|
||||||
@@ -99,7 +98,7 @@ class TestCompose(unittest.TestCase):
|
|||||||
str(d), ["up", "-d", "--force-recreate"]
|
str(d), ["up", "-d", "--force-recreate"]
|
||||||
)
|
)
|
||||||
|
|
||||||
expected: List[str] = [
|
expected: list[str] = [
|
||||||
"/usr/bin/docker",
|
"/usr/bin/docker",
|
||||||
"compose",
|
"compose",
|
||||||
"--chdir",
|
"--chdir",
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
from typing import List
|
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
|
||||||
@@ -11,7 +10,7 @@ class HardRestartArgTests(unittest.TestCase):
|
|||||||
the dir is a stack whose overlay network collides with compose up, pass
|
the dir is a stack whose overlay network collides with compose up, pass
|
||||||
nothing."""
|
nothing."""
|
||||||
|
|
||||||
def _parse(self, extra: List[str]):
|
def _parse(self, extra: list[str]):
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from baudolo.backup import cli
|
from baudolo.backup import cli
|
||||||
|
|||||||
@@ -20,15 +20,17 @@ def _capture_commands(*, db_type, rows, container):
|
|||||||
captured.append(cmd)
|
captured.append(cmd)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as td:
|
with (
|
||||||
with patch.object(db_mod, "execute_shell_command", side_effect=_capture):
|
tempfile.TemporaryDirectory() as td,
|
||||||
db_mod.backup_database(
|
patch.object(db_mod, "execute_shell_command", side_effect=_capture),
|
||||||
container=container,
|
):
|
||||||
volume_dir=td,
|
db_mod.backup_database(
|
||||||
db_type=db_type,
|
container=container,
|
||||||
databases_df=_df(rows),
|
volume_dir=td,
|
||||||
database_containers=[container],
|
db_type=db_type,
|
||||||
)
|
databases_df=_df(rows),
|
||||||
|
database_containers=[container],
|
||||||
|
)
|
||||||
return captured
|
return captured
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -61,9 +61,11 @@ class TestBtrfs(unittest.TestCase):
|
|||||||
|
|
||||||
def test_it_removes_the_snapshot_even_when_the_body_raises(self) -> None:
|
def test_it_removes_the_snapshot_even_when_the_body_raises(self) -> None:
|
||||||
run = Runner()
|
run = Runner()
|
||||||
with self.assertRaises(ZeroDivisionError):
|
with (
|
||||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
|
self.assertRaises(ZeroDivisionError),
|
||||||
raise ZeroDivisionError
|
volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run),
|
||||||
|
):
|
||||||
|
raise ZeroDivisionError
|
||||||
self.assertTrue(run.calls[-1].startswith("btrfs subvolume delete"))
|
self.assertTrue(run.calls[-1].startswith("btrfs subvolume delete"))
|
||||||
|
|
||||||
|
|
||||||
@@ -93,26 +95,30 @@ class TestZfs(unittest.TestCase):
|
|||||||
|
|
||||||
def test_an_unmounted_dataset_is_an_error(self) -> None:
|
def test_an_unmounted_dataset_is_an_error(self) -> None:
|
||||||
run = Runner({"zfs list": [""]})
|
run = Runner({"zfs list": [""]})
|
||||||
with self.assertRaises(SnapshotError):
|
with (
|
||||||
with volume_snapshot("zfs", "/var/lib/docker", "20260731", run=run):
|
self.assertRaises(SnapshotError),
|
||||||
pass
|
volume_snapshot("zfs", "/var/lib/docker", "20260731", run=run),
|
||||||
|
):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
class TestRejections(unittest.TestCase):
|
class TestRejections(unittest.TestCase):
|
||||||
def test_an_unknown_kind_is_rejected(self) -> None:
|
def test_an_unknown_kind_is_rejected(self) -> None:
|
||||||
run = Runner()
|
run = Runner()
|
||||||
with self.assertRaises(SnapshotError):
|
with (
|
||||||
with volume_snapshot("ext4", "/var/lib/docker", "20260731", run=run):
|
self.assertRaises(SnapshotError),
|
||||||
pass
|
volume_snapshot("ext4", "/var/lib/docker", "20260731", run=run),
|
||||||
|
):
|
||||||
|
pass
|
||||||
self.assertEqual(run.calls, [])
|
self.assertEqual(run.calls, [])
|
||||||
|
|
||||||
def test_a_path_outside_the_subject_is_rejected(self) -> None:
|
def test_a_path_outside_the_subject_is_rejected(self) -> None:
|
||||||
run = Runner()
|
run = Runner()
|
||||||
with volume_snapshot(
|
with (
|
||||||
"btrfs", "/var/lib/docker", "20260731", run=run
|
volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve,
|
||||||
) as resolve:
|
self.assertRaises(SnapshotError),
|
||||||
with self.assertRaises(SnapshotError):
|
):
|
||||||
resolve("/etc/passwd")
|
resolve("/etc/passwd")
|
||||||
|
|
||||||
def test_the_subject_itself_resolves_to_the_snapshot_root(self) -> None:
|
def test_the_subject_itself_resolves_to_the_snapshot_root(self) -> None:
|
||||||
run = Runner()
|
run = Runner()
|
||||||
@@ -137,9 +143,11 @@ class TestRemovalFailure(unittest.TestCase):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
def test_a_failed_removal_does_not_mask_the_body(self) -> None:
|
def test_a_failed_removal_does_not_mask_the_body(self) -> None:
|
||||||
with self.assertRaises(ZeroDivisionError):
|
with (
|
||||||
with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=Busy()):
|
self.assertRaises(ZeroDivisionError),
|
||||||
raise ZeroDivisionError
|
volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=Busy()),
|
||||||
|
):
|
||||||
|
raise ZeroDivisionError
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -65,10 +65,7 @@ class TestClusterReplay(unittest.TestCase):
|
|||||||
self.assertIn("rolname <> current_user", preclean)
|
self.assertIn("rolname <> current_user", preclean)
|
||||||
|
|
||||||
def test_only_the_connecting_role_loses_its_create(self) -> None:
|
def test_only_the_connecting_role_loses_its_create(self) -> None:
|
||||||
# Captured from pg_dumpall 17: the bootstrap superuser is recreated like
|
# Captured from pg_dumpall 17.
|
||||||
# any other role, and the pre-clean cannot drop the one holding the
|
|
||||||
# session - so that single CREATE always collides while its ALTER, which
|
|
||||||
# carries the attributes and the password, must survive.
|
|
||||||
dump = [
|
dump = [
|
||||||
b"CREATE ROLE app;\n",
|
b"CREATE ROLE app;\n",
|
||||||
b"ALTER ROLE app WITH NOSUPERUSER INHERIT LOGIN PASSWORD 'SCRAM-SHA-256$...';\n",
|
b"ALTER ROLE app WITH NOSUPERUSER INHERIT LOGIN PASSWORD 'SCRAM-SHA-256$...';\n",
|
||||||
|
|||||||
@@ -131,7 +131,6 @@ class TestSeedMain(unittest.TestCase):
|
|||||||
warning_calls,
|
warning_calls,
|
||||||
"Expected a WARNING print when databases.csv is empty, but none was found.",
|
"Expected a WARNING print when databases.csv is empty, but none was found.",
|
||||||
)
|
)
|
||||||
# Ensure the warning goes to stderr
|
|
||||||
_, warn_kwargs = warning_calls[0]
|
_, warn_kwargs = warning_calls[0]
|
||||||
self.assertEqual(warn_kwargs.get("file"), seed_main.sys.stderr)
|
self.assertEqual(warn_kwargs.get("file"), seed_main.sys.stderr)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user