kevinveenbirkenbach/docker-volume-backup
1
0
Fork 0
mirror of https://github.com/kevinveenbirkenbach/docker-volume-backup.git synced 2025-12-29 03:42:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(backup): log missing db config instead of raising

Browse Source 
- Use module logger in backup/db.py
- Skip db dump when no databases.csv entry is present
- Apply black/formatting cleanup across backup/restore/tests

https://chatgpt.com/share/69519d45-b0dc-800f-acb6-6fb8504e9b46
This commit is contained in:
Kevin Veen-Birkenbach
2025-12-28 22:12:31 +01:00
parent 88b35ee923
commit 6adafe6b1f
21 changed files with 421 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/baudolo/backup/app.py
View File

@@ -51,7 +51,9 @@ def is_image_ignored(container: str, images_no_backup_required: list[str]) -> bo
return any(pat in img for pat in images_no_backup_required)
def volume_is_fully_ignored(containers: list[str], images_no_backup_required: list[str]) -> bool:
def volume_is_fully_ignored(
containers: list[str], images_no_backup_required: list[str]
) -> bool:
"""
Skip file backup only if all containers linked to the volume are ignored.
"""
@@ -178,6 +180,8 @@ def main() -> int:
print("Finished volume backups.", flush=True)
print("Handling Docker Compose services...", flush=True)
handle_docker_compose_services(args.compose_dir, args.docker_compose_hard_restart_required)
handle_docker_compose_services(
args.compose_dir, args.docker_compose_hard_restart_required
)
return 0

4
src/baudolo/backup/cli.py
View File

@@ -2,7 +2,7 @@ from __future__ import annotations
import argparse
import os
from pathlib import Path
def parse_args() -> argparse.Namespace:
dirname = os.path.dirname(__file__)
@@ -25,7 +25,7 @@ def parse_args() -> argparse.Namespace:
p.add_argument(
"--repo-name",
default='backup-docker-to-local',
default="backup-docker-to-local",
help="Backup repo folder name under <backups-dir>/<machine-id>/ (default: git repo folder name)",
)
p.add_argument(

4
src/baudolo/backup/compose.py
View File

@@ -10,7 +10,9 @@ def hard_restart_docker_services(dir_path: str) -> None:
subprocess.run(["docker-compose", "up", "-d"], cwd=dir_path, check=True)
def handle_docker_compose_services(parent_directory: str, hard_restart_required: list[str]) -> None:
def handle_docker_compose_services(
parent_directory: str, hard_restart_required: list[str]
) -> None:
for entry in os.scandir(parent_directory):
if not entry.is_dir():
continue

15
src/baudolo/backup/db.py
View File

@@ -5,9 +5,12 @@ import pathlib
import re
import pandas
import logging
from .shell import BackupException, execute_shell_command
log = logging.getLogger(__name__)
def get_instance(container: str, database_containers: list[str]) -> str:
if container in database_containers:
@@ -15,7 +18,9 @@ def get_instance(container: str, database_containers: list[str]) -> str:
return re.split(r"(_|-)(database|db|postgres)", container)[0]
def fallback_pg_dumpall(container: str, username: str, password: str, out_file: str) -> None:
def fallback_pg_dumpall(
container: str, username: str, password: str, out_file: str
) -> None:
cmd = (
f"PGPASSWORD={password} docker exec -i {container} "
f"pg_dumpall -U {username} -h localhost > {out_file}"
@@ -34,7 +39,8 @@ def backup_database(
instance_name = get_instance(container, database_containers)
entries = databases_df.loc[databases_df["instance"] == instance_name]
if entries.empty:
raise BackupException(f"No entry found for instance '{instance_name}'")
log.warning("No entry found for instance '%s'", instance_name)
return
out_dir = os.path.join(volume_dir, "sql")
pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True)
@@ -68,6 +74,9 @@ def backup_database(
execute_shell_command(cmd)
except BackupException as e:
print(f"pg_dump failed: {e}", flush=True)
print(f"Falling back to pg_dumpall for instance '{instance_name}'", flush=True)
print(
f"Falling back to pg_dumpall for instance '{instance_name}'",
flush=True,
)
fallback_pg_dumpall(container, user, password, cluster_file)
continue

4
src/baudolo/backup/docker.py
View File

@@ -37,7 +37,9 @@ def change_containers_status(containers: list[str], status: str) -> None:
def docker_volume_exists(volume: str) -> bool:
# Avoid throwing exceptions for exists checks.
try:
execute_shell_command(f"docker volume inspect {volume} >/dev/null 2>&1 && echo OK")
execute_shell_command(
f"docker volume inspect {volume} >/dev/null 2>&1 && echo OK"
)
return True
except Exception:
return False

8
src/baudolo/backup/volume.py
View File

@@ -13,7 +13,9 @@ def get_storage_path(volume_name: str) -> str:
return f"{path}/"
def get_last_backup_dir(versions_dir: str, volume_name: str, current_backup_dir: str) -> str | None:
def get_last_backup_dir(
versions_dir: str, volume_name: str, current_backup_dir: str
) -> str | None:
versions = sorted(os.listdir(versions_dir), reverse=True)
for version in versions:
candidate = os.path.join(versions_dir, version, volume_name, "files", "")
@@ -37,6 +39,8 @@ def backup_volume(versions_dir: str, volume_name: str, volume_dir: str) -> None:
execute_shell_command(cmd)
except BackupException as e:
if "file has vanished" in str(e):
print("Warning: Some files vanished before transfer. Continuing.", flush=True)
print(
"Warning: Some files vanished before transfer. Continuing.", flush=True
)
else:
raise

2
src/baudolo/restore/__init__.py
View File

@@ -1 +1 @@
__all__ = ["main"]
__all__ = ["main"]

4
src/baudolo/restore/__main__.py
View File

@@ -66,7 +66,9 @@ def main(argv: list[str] | None = None) -> int:
# ------------------------------------------------------------------
# mariadb
# ------------------------------------------------------------------
p_mdb = sub.add_parser("mariadb", help="Restore a single MariaDB/MySQL-compatible dump")
p_mdb = sub.add_parser(
"mariadb", help="Restore a single MariaDB/MySQL-compatible dump"
)
_add_common_backup_args(p_mdb)
p_mdb.add_argument("--container", required=True)
p_mdb.add_argument("--db-name", required=True)

2
src/baudolo/restore/db/__init__.py
View File

@@ -1 +1 @@
"""Database restore handlers (Postgres, MariaDB/MySQL)."""
"""Database restore handlers (Postgres, MariaDB/MySQL)."""

26
src/baudolo/restore/db/mariadb.py
View File

@@ -23,7 +23,9 @@ exit 42
raise RuntimeError("empty client detection output")
return out
except Exception as e:
print("ERROR: neither 'mariadb' nor 'mysql' found in container.", file=sys.stderr)
print(
"ERROR: neither 'mariadb' nor 'mysql' found in container.", file=sys.stderr
)
raise e
@@ -47,7 +49,14 @@ def restore_mariadb_sql(
# MariaDB 11 images may not contain the mysql binary at all.
docker_exec(
container,
[client, "-u", user, f"--password={password}", "-e", "SET FOREIGN_KEY_CHECKS=0;"],
[
client,
"-u",
user,
f"--password={password}",
"-e",
"SET FOREIGN_KEY_CHECKS=0;",
],
)
result = docker_exec(
@@ -80,10 +89,19 @@ def restore_mariadb_sql(
docker_exec(
container,
[client, "-u", user, f"--password={password}", "-e", "SET FOREIGN_KEY_CHECKS=1;"],
[
client,
"-u",
user,
f"--password={password}",
"-e",
"SET FOREIGN_KEY_CHECKS=1;",
],
)
with open(sql_path, "rb") as f:
docker_exec(container, [client, "-u", user, f"--password={password}", db_name], stdin=f)
docker_exec(
container, [client, "-u", user, f"--password={password}", db_name], stdin=f
)
print(f"MariaDB/MySQL restore complete for db '{db_name}'.")

4
src/baudolo/restore/files.py
View File

@@ -6,7 +6,9 @@ import sys
from .run import run, docker_volume_exists
def restore_volume_files(volume_name: str, backup_files_dir: str, *, rsync_image: str) -> int:
def restore_volume_files(
volume_name: str, backup_files_dir: str, *, rsync_image: str
) -> int:
if not os.path.isdir(backup_files_dir):
print(f"ERROR: backup files dir not found: {backup_files_dir}", file=sys.stderr)
return 2

40
src/baudolo/seed/__main__.py
View File

@@ -2,21 +2,24 @@ import pandas as pd
import argparse
import os
def check_and_add_entry(file_path, instance, database, username, password):
# Check if the file exists and is not empty
if os.path.exists(file_path) and os.path.getsize(file_path) > 0:
# Read the existing CSV file with header
df = pd.read_csv(file_path, sep=';')
df = pd.read_csv(file_path, sep=";")
else:
# Create a new DataFrame with columns if file does not exist
df = pd.DataFrame(columns=['instance', 'database', 'username', 'password'])
df = pd.DataFrame(columns=["instance", "database", "username", "password"])
# Check if the entry exists and remove it
mask = (
(df['instance'] == instance) &
((df['database'] == database) |
(((df['database'].isna()) | (df['database'] == '')) & (database == ''))) &
(df['username'] == username)
(df["instance"] == instance)
& (
(df["database"] == database)
| (((df["database"].isna()) | (df["database"] == "")) & (database == ""))
)
& (df["username"] == username)
)
if not df[mask].empty:
@@ -26,25 +29,40 @@ def check_and_add_entry(file_path, instance, database, username, password):
print("Adding new entry.")
# Create a new DataFrame for the new entry
new_entry = pd.DataFrame([{'instance': instance, 'database': database, 'username': username, 'password': password}])
new_entry = pd.DataFrame(
[
{
"instance": instance,
"database": database,
"username": username,
"password": password,
}
]
)
# Add (or replace) the entry using concat
df = pd.concat([df, new_entry], ignore_index=True)
# Save the updated CSV file
df.to_csv(file_path, sep=';', index=False)
df.to_csv(file_path, sep=";", index=False)
def main():
parser = argparse.ArgumentParser(description="Check and replace (or add) a database entry in a CSV file.")
parser = argparse.ArgumentParser(
description="Check and replace (or add) a database entry in a CSV file."
)
parser.add_argument("file_path", help="Path to the CSV file")
parser.add_argument("instance", help="Database instance")
parser.add_argument("database", help="Database name")
parser.add_argument("username", help="Username")
parser.add_argument("password", nargs='?', default="", help="Password (optional)")
parser.add_argument("password", nargs="?", default="", help="Password (optional)")
args = parser.parse_args()
check_and_add_entry(args.file_path, args.instance, args.database, args.username, args.password)
check_and_add_entry(
args.file_path, args.instance, args.database, args.username, args.password
)
if __name__ == "__main__":
main()