mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2024-11-22 00:11:03 +01:00
refactored. untested.
This commit is contained in:
parent
b83e481d01
commit
a1c33c1747
@ -1,6 +1,6 @@
|
||||
#!/bin/python
|
||||
# Backups volumes of running containers
|
||||
#
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
import re
|
||||
@ -8,114 +8,80 @@ import pathlib
|
||||
import pandas
|
||||
from datetime import datetime
|
||||
|
||||
class RsyncCode24Exception(Exception):
|
||||
"""Exception for rsync error code 24."""
|
||||
"""rsync warning: some files vanished before they could be transferred"""
|
||||
class BackupException(Exception):
|
||||
"""Generic exception for backup errors."""
|
||||
pass
|
||||
|
||||
def bash(command):
|
||||
def execute_shell_command(command):
|
||||
"""Execute a shell command and return its output."""
|
||||
print(command)
|
||||
process = subprocess.Popen([command], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
|
||||
out, err = process.communicate()
|
||||
stdout = out.splitlines()
|
||||
stderr = err.decode("utf-8")
|
||||
output = [line.decode("utf-8") for line in stdout]
|
||||
if process.returncode != 0:
|
||||
raise BackupException(f"Error in command: {command}\nOutput: {out}\nError: {err}\nExit code: {process.returncode}")
|
||||
return [line.decode("utf-8") for line in out.splitlines()]
|
||||
|
||||
exitcode = process.wait()
|
||||
if exitcode != 0:
|
||||
print(f"Error in command: {command}\nOutput: {out}\nError: {err}\nExit code: {exitcode}")
|
||||
|
||||
if "rsync" in command and exitcode == 24:
|
||||
raise RsyncCode24Exception(f"rsync error code 24 encountered: {stderr}")
|
||||
def get_machine_id():
|
||||
"""Get the machine identifier."""
|
||||
return execute_shell_command("sha256sum /etc/machine-id")[0][0:64]
|
||||
|
||||
raise Exception("Exit code is greater than 0")
|
||||
def create_backup_directories(base_dir, machine_id, repository_name, backup_time):
|
||||
"""Create necessary directories for backup."""
|
||||
version_dir = os.path.join(base_dir, machine_id, repository_name, backup_time)
|
||||
pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
|
||||
return version_dir
|
||||
|
||||
return output
|
||||
def get_database_name(container):
|
||||
"""Extract the database name from the container name."""
|
||||
return re.split("(_|-)(database|db)", container)[0]
|
||||
|
||||
def print_bash(command):
|
||||
output = bash(command)
|
||||
print(list_to_string(output))
|
||||
return output
|
||||
def backup_database(container, databases, version_dir):
|
||||
"""Backup database if applicable."""
|
||||
database_name = get_database_name(container)
|
||||
database_entry = databases.loc[databases['database'] == database_name].iloc[0]
|
||||
mysqldump_destination_dir = os.path.join(version_dir, "sql")
|
||||
pathlib.Path(mysqldump_destination_dir).mkdir(parents=True, exist_ok=True)
|
||||
mysqldump_destination_file = os.path.join(mysqldump_destination_dir, "backup.sql")
|
||||
database_backup_command = f"docker exec {container} /usr/bin/mariadb-dump -u {database_entry['username']} -p{database_entry['password']} {database_entry['database']} > {mysqldump_destination_file}"
|
||||
execute_shell_command(database_backup_command)
|
||||
|
||||
def backup_volume(volume_name, version_dir):
|
||||
"""Backup files of a volume."""
|
||||
files_rsync_destination_path = os.path.join(version_dir, volume_name, "files")
|
||||
pathlib.Path(files_rsync_destination_path).mkdir(parents=True, exist_ok=True)
|
||||
source_dir = f"/var/lib/docker/volumes/{volume_name}/_data/"
|
||||
rsync_command = f"rsync -abP --delete --delete-excluded {source_dir} {files_rsync_destination_path}"
|
||||
execute_shell_command(rsync_command)
|
||||
|
||||
def list_to_string(list):
|
||||
return str(' '.join(list))
|
||||
def main():
|
||||
print('Start backup routine...')
|
||||
dirname = os.path.dirname(__file__)
|
||||
repository_name = os.path.basename(dirname)
|
||||
machine_id = get_machine_id()
|
||||
backups_dir = '/Backups/'
|
||||
backup_time = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
version_dir = create_backup_directories(backups_dir, machine_id, repository_name, backup_time)
|
||||
|
||||
print('Start volume backups...')
|
||||
databases = pandas.read_csv(os.path.join(dirname, "databases.csv"), sep=";")
|
||||
volume_names = execute_shell_command("docker volume ls --format '{{.Name}}'")
|
||||
|
||||
for volume_name in volume_names:
|
||||
print(f'Start backup routine for volume: {volume_name}')
|
||||
containers = execute_shell_command(f"docker ps --filter volume=\"{volume_name}\" --format '{{.Names}}'")
|
||||
if not containers:
|
||||
print('Skipped due to no running containers using this volume.')
|
||||
continue
|
||||
|
||||
print('start backup routine...')
|
||||
for container in containers:
|
||||
if container != 'akaunting':
|
||||
backup_database(container, databases, version_dir)
|
||||
backup_volume(volume_name, version_dir)
|
||||
|
||||
dirname = os.path.dirname(__file__)
|
||||
repository_name = os.path.basename(dirname)
|
||||
# identifier of this backups
|
||||
machine_id = bash("sha256sum /etc/machine-id")[0][0:64]
|
||||
# Folder in which all Backups are stored
|
||||
backups_dir = '/Backups/'
|
||||
# Folder in which the versions off docker volume backups are stored
|
||||
versions_dir = backups_dir + machine_id + "/" + repository_name + "/"
|
||||
# Time when the backup started
|
||||
backup_time = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
# Folder containing the current version
|
||||
version_dir = versions_dir + backup_time + "/"
|
||||
print('Finished volume backups.')
|
||||
print('Restart docker service...')
|
||||
execute_shell_command("systemctl restart docker")
|
||||
print('Finished backup routine.')
|
||||
|
||||
# Create folder to store version in
|
||||
pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print('start volume backups...')
|
||||
print('load connection data...')
|
||||
databases = pandas.read_csv(dirname + "/databases.csv", sep=";")
|
||||
volume_names = bash("docker volume ls --format '{{.Name}}'")
|
||||
for volume_name in volume_names:
|
||||
print('start backup routine for volume: ' + volume_name)
|
||||
containers = bash("docker ps --filter volume=\"" + volume_name + "\" --format '{{.Names}}'")
|
||||
if len(containers) == 0:
|
||||
print('skipped due to no running containers using this volume.')
|
||||
else:
|
||||
container = containers[0]
|
||||
# Folder to which the volumes are copied
|
||||
volume_destination_dir = version_dir + volume_name
|
||||
# Database name
|
||||
database_name = re.split("(_|-)(database|db)", container)[0]
|
||||
# Entries with database login data concerning this container
|
||||
databases_entries = databases.loc[databases['database'] == database_name]
|
||||
# Exception for akaunting due to fast implementation
|
||||
if len(databases_entries) == 1 and container != 'akaunting':
|
||||
print("Backup database...")
|
||||
mysqldump_destination_dir = volume_destination_dir + "/sql"
|
||||
mysqldump_destination_file = mysqldump_destination_dir + "/backup.sql"
|
||||
pathlib.Path(mysqldump_destination_dir).mkdir(parents=True, exist_ok=True)
|
||||
database_entry = databases_entries.iloc[0]
|
||||
database_backup_command = "docker exec " + container + " /usr/bin/mariadb-dump -u " + database_entry["username"] + " -p" + database_entry["password"] + " " + database_entry["database"] + " > " + mysqldump_destination_file
|
||||
print_bash(database_backup_command)
|
||||
print("Backup files...")
|
||||
files_rsync_destination_path = volume_destination_dir + "/files"
|
||||
pathlib.Path(files_rsync_destination_path).mkdir(parents=True, exist_ok=True)
|
||||
versions = os.listdir(versions_dir)
|
||||
versions.sort(reverse=True)
|
||||
if len(versions) > 1:
|
||||
last_version = versions[1]
|
||||
last_version_files_dir = versions_dir + last_version + "/" + volume_name + "/files"
|
||||
if os.path.isdir(last_version_files_dir):
|
||||
link_dest_parameter="--link-dest='" + last_version_files_dir + "' "
|
||||
else:
|
||||
print("No previous version exists in path "+ last_version_files_dir + ".")
|
||||
link_dest_parameter=""
|
||||
else:
|
||||
print("No previous version exists in path "+ last_version_files_dir + ".")
|
||||
link_dest_parameter=""
|
||||
source_dir = "/var/lib/docker/volumes/" + volume_name + "/_data/"
|
||||
rsync_command = "rsync -abP --delete --delete-excluded " + link_dest_parameter + source_dir + " " + files_rsync_destination_path
|
||||
try:
|
||||
print_bash(rsync_command)
|
||||
except RsyncCode24Exception:
|
||||
print("Ignoring rsync error code 24, proceeding with the next command.")
|
||||
print("stop containers...")
|
||||
print("Backup data after container is stopped...")
|
||||
print_bash("docker stop " + list_to_string(containers))
|
||||
print_bash(rsync_command)
|
||||
print("start containers...")
|
||||
print_bash("docker start " + list_to_string(containers))
|
||||
print("end backup routine for volume:" + volume_name)
|
||||
print('finished volume backups.')
|
||||
print('restart docker service...')
|
||||
print_bash("systemctl restart docker")
|
||||
print('finished backup routine.')
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user