diff --git a/backup-docker-to-local.py b/backup-docker-to-local.py index d93e4af..a506244 100644 --- a/backup-docker-to-local.py +++ b/backup-docker-to-local.py @@ -1,6 +1,6 @@ #!/bin/python # Backups volumes of running containers -# + import subprocess import os import re @@ -8,114 +8,80 @@ import pathlib import pandas from datetime import datetime -class RsyncCode24Exception(Exception): - """Exception for rsync error code 24.""" - """rsync warning: some files vanished before they could be transferred""" +class BackupException(Exception): + """Generic exception for backup errors.""" pass -def bash(command): +def execute_shell_command(command): + """Execute a shell command and return its output.""" print(command) process = subprocess.Popen([command], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) out, err = process.communicate() - stdout = out.splitlines() - stderr = err.decode("utf-8") - output = [line.decode("utf-8") for line in stdout] + if process.returncode != 0: + raise BackupException(f"Error in command: {command}\nOutput: {out}\nError: {err}\nExit code: {process.returncode}") + return [line.decode("utf-8") for line in out.splitlines()] - exitcode = process.wait() - if exitcode != 0: - print(f"Error in command: {command}\nOutput: {out}\nError: {err}\nExit code: {exitcode}") - - if "rsync" in command and exitcode == 24: - raise RsyncCode24Exception(f"rsync error code 24 encountered: {stderr}") +def get_machine_id(): + """Get the machine identifier.""" + return execute_shell_command("sha256sum /etc/machine-id")[0][0:64] - raise Exception("Exit code is greater than 0") +def create_backup_directories(base_dir, machine_id, repository_name, backup_time): + """Create necessary directories for backup.""" + version_dir = os.path.join(base_dir, machine_id, repository_name, backup_time) + pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True) + return version_dir - return output +def get_database_name(container): + """Extract the database name from the container name.""" + return re.split("(_|-)(database|db)", container)[0] -def print_bash(command): - output = bash(command) - print(list_to_string(output)) - return output +def backup_database(container, databases, version_dir): + """Backup database if applicable.""" + database_name = get_database_name(container) + database_entry = databases.loc[databases['database'] == database_name].iloc[0] + mysqldump_destination_dir = os.path.join(version_dir, "sql") + pathlib.Path(mysqldump_destination_dir).mkdir(parents=True, exist_ok=True) + mysqldump_destination_file = os.path.join(mysqldump_destination_dir, "backup.sql") + database_backup_command = f"docker exec {container} /usr/bin/mariadb-dump -u {database_entry['username']} -p{database_entry['password']} {database_entry['database']} > {mysqldump_destination_file}" + execute_shell_command(database_backup_command) +def backup_volume(volume_name, version_dir): + """Backup files of a volume.""" + files_rsync_destination_path = os.path.join(version_dir, volume_name, "files") + pathlib.Path(files_rsync_destination_path).mkdir(parents=True, exist_ok=True) + source_dir = f"/var/lib/docker/volumes/{volume_name}/_data/" + rsync_command = f"rsync -abP --delete --delete-excluded {source_dir} {files_rsync_destination_path}" + execute_shell_command(rsync_command) -def list_to_string(list): - return str(' '.join(list)) +def main(): + print('Start backup routine...') + dirname = os.path.dirname(__file__) + repository_name = os.path.basename(dirname) + machine_id = get_machine_id() + backups_dir = '/Backups/' + backup_time = datetime.now().strftime("%Y%m%d%H%M%S") + version_dir = create_backup_directories(backups_dir, machine_id, repository_name, backup_time) + print('Start volume backups...') + databases = pandas.read_csv(os.path.join(dirname, "databases.csv"), sep=";") + volume_names = execute_shell_command("docker volume ls --format '{{.Name}}'") + + for volume_name in volume_names: + print(f'Start backup routine for volume: {volume_name}') + containers = execute_shell_command(f"docker ps --filter volume=\"{volume_name}\" --format '{{.Names}}'") + if not containers: + print('Skipped due to no running containers using this volume.') + continue -print('start backup routine...') + for container in containers: + if container != 'akaunting': + backup_database(container, databases, version_dir) + backup_volume(volume_name, version_dir) -dirname = os.path.dirname(__file__) -repository_name = os.path.basename(dirname) -# identifier of this backups -machine_id = bash("sha256sum /etc/machine-id")[0][0:64] -# Folder in which all Backups are stored -backups_dir = '/Backups/' -# Folder in which the versions off docker volume backups are stored -versions_dir = backups_dir + machine_id + "/" + repository_name + "/" -# Time when the backup started -backup_time = datetime.now().strftime("%Y%m%d%H%M%S") -# Folder containing the current version -version_dir = versions_dir + backup_time + "/" + print('Finished volume backups.') + print('Restart docker service...') + execute_shell_command("systemctl restart docker") + print('Finished backup routine.') -# Create folder to store version in -pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True) - -print('start volume backups...') -print('load connection data...') -databases = pandas.read_csv(dirname + "/databases.csv", sep=";") -volume_names = bash("docker volume ls --format '{{.Name}}'") -for volume_name in volume_names: - print('start backup routine for volume: ' + volume_name) - containers = bash("docker ps --filter volume=\"" + volume_name + "\" --format '{{.Names}}'") - if len(containers) == 0: - print('skipped due to no running containers using this volume.') - else: - container = containers[0] - # Folder to which the volumes are copied - volume_destination_dir = version_dir + volume_name - # Database name - database_name = re.split("(_|-)(database|db)", container)[0] - # Entries with database login data concerning this container - databases_entries = databases.loc[databases['database'] == database_name] - # Exception for akaunting due to fast implementation - if len(databases_entries) == 1 and container != 'akaunting': - print("Backup database...") - mysqldump_destination_dir = volume_destination_dir + "/sql" - mysqldump_destination_file = mysqldump_destination_dir + "/backup.sql" - pathlib.Path(mysqldump_destination_dir).mkdir(parents=True, exist_ok=True) - database_entry = databases_entries.iloc[0] - database_backup_command = "docker exec " + container + " /usr/bin/mariadb-dump -u " + database_entry["username"] + " -p" + database_entry["password"] + " " + database_entry["database"] + " > " + mysqldump_destination_file - print_bash(database_backup_command) - print("Backup files...") - files_rsync_destination_path = volume_destination_dir + "/files" - pathlib.Path(files_rsync_destination_path).mkdir(parents=True, exist_ok=True) - versions = os.listdir(versions_dir) - versions.sort(reverse=True) - if len(versions) > 1: - last_version = versions[1] - last_version_files_dir = versions_dir + last_version + "/" + volume_name + "/files" - if os.path.isdir(last_version_files_dir): - link_dest_parameter="--link-dest='" + last_version_files_dir + "' " - else: - print("No previous version exists in path "+ last_version_files_dir + ".") - link_dest_parameter="" - else: - print("No previous version exists in path "+ last_version_files_dir + ".") - link_dest_parameter="" - source_dir = "/var/lib/docker/volumes/" + volume_name + "/_data/" - rsync_command = "rsync -abP --delete --delete-excluded " + link_dest_parameter + source_dir + " " + files_rsync_destination_path - try: - print_bash(rsync_command) - except RsyncCode24Exception: - print("Ignoring rsync error code 24, proceeding with the next command.") - print("stop containers...") - print("Backup data after container is stopped...") - print_bash("docker stop " + list_to_string(containers)) - print_bash(rsync_command) - print("start containers...") - print_bash("docker start " + list_to_string(containers)) - print("end backup routine for volume:" + volume_name) -print('finished volume backups.') -print('restart docker service...') -print_bash("systemctl restart docker") -print('finished backup routine.') +if __name__ == "__main__": + main()