backup-docker-to-local.py

@@ -7,7 +7,6 @@ import re
 import pathlib
 import pandas
 from datetime import datetime
-import argparse
 
 class BackupException(Exception):
     """Generic exception for backup errors."""
@@ -22,58 +21,28 @@ def execute_shell_command(command):
         raise BackupException(f"Error in command: {command}\nOutput: {out}\nError: {err}\nExit code: {process.returncode}")
     return [line.decode("utf-8") for line in out.splitlines()]
 
-def create_version_directory():
-    """Create necessary directories for backup."""
-    version_dir = os.path.join(VERSIONS_DIR, BACKUP_TIME)
-    pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
-    return version_dir
-
 def get_machine_id():
     """Get the machine identifier."""
     return execute_shell_command("sha256sum /etc/machine-id")[0][0:64]
 
-### GLOBAL CONFIGURATION ###
-
-IMAGES_NO_STOP_REQUIRED = [
-    # 'baserow', Doesn't use an extra database
-    'element',
-    'gitea',
-    'listmonk',
-    'mastodon',
-    'matomo',
-    'nextcloud',
-    'openproject',
-    'pixelfed',
-    'wordpress' 
-]
-
-IMAGES_NO_BACKUP_REQUIRED = [
-    'redis', 
-    'memcached'
-    ]
-
-DIRNAME = os.path.dirname(__file__)
-
-DATABASES = pandas.read_csv(os.path.join(DIRNAME, "databases.csv"), sep=";")
-REPOSITORY_NAME = os.path.basename(DIRNAME)
-MACHINE_ID = get_machine_id()
-BACKUPS_DIR = '/Backups/'
-VERSIONS_DIR = os.path.join(BACKUPS_DIR, MACHINE_ID, REPOSITORY_NAME)
-BACKUP_TIME = datetime.now().strftime("%Y%m%d%H%M%S")
-VERSION_DIR = create_version_directory()
+def create_version_directory(versions_dir, backup_time):
+    """Create necessary directories for backup."""
+    version_dir = os.path.join(versions_dir, backup_time)
+    pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
+    return version_dir
 
 def get_instance(container):
     instance_name = re.split("(_|-)(database|db|postgres)", container)[0]
     print(f"Extracted instance name: {instance_name}")
     return instance_name
 
-def backup_database(container, volume_dir, db_type):
+def backup_database(container, databases, volume_dir, db_type):
     """Backup database (MariaDB or PostgreSQL) if applicable."""
     print(f"Starting database backup for {container} using {db_type}...")
     instance_name = get_instance(container)
 
     # Filter the DataFrame for the given instance_name
-    database_entries = DATABASES.loc[DATABASES['instance'] == instance_name]
+    database_entries = databases.loc[databases['instance'] == instance_name]
 
     # Check if there are more than one entries
     if len(database_entries) > 1:
@@ -111,11 +80,11 @@ def backup_database(container, volume_dir, db_type):
     execute_shell_command(backup_command)
     print(f"Database backup for {container} completed.")
 
-def get_last_backup_dir(volume_name, current_backup_dir):
+def get_last_backup_dir(versions_dir, volume_name, current_backup_dir):
     """Get the most recent backup directory for the specified volume."""
-    versions = sorted(os.listdir(VERSIONS_DIR), reverse=True)
+    versions = sorted(os.listdir(versions_dir), reverse=True)
     for version in versions:
-        backup_dir = os.path.join(VERSIONS_DIR, version, volume_name, "files")
+        backup_dir = os.path.join(versions_dir, version, volume_name, "files")
         # Ignore current backup dir
         if backup_dir != current_backup_dir:
             if os.path.isdir(backup_dir):
@@ -123,13 +92,13 @@ def get_last_backup_dir(volume_name, current_backup_dir):
     print(f"No previous backups available for volume: {volume_name}")
     return None
 
-def backup_volume(volume_name, volume_dir):
+def backup_volume(volume_name, volume_dir, versions_dir):
     """Backup files of a volume with incremental backups."""
     print(f"Starting backup routine for volume: {volume_name}")
     files_rsync_destination_path = os.path.join(volume_dir, "files")
     pathlib.Path(files_rsync_destination_path).mkdir(parents=True, exist_ok=True)
 
-    last_backup_dir = get_last_backup_dir(volume_name, files_rsync_destination_path)
+    last_backup_dir = get_last_backup_dir(versions_dir, volume_name, files_rsync_destination_path)
     link_dest_option = f"--link-dest='{last_backup_dir}'" if last_backup_dir else ""
 
     source_dir = f"/var/lib/docker/volumes/{volume_name}/_data/"
@@ -173,91 +142,94 @@ def is_image_whitelisted(container, images):
             return True
     return False
 
-def is_container_stop_required(containers):
+def is_any_image_not_whitelisted(containers, images):
     """Check if any of the containers are using images that are not whitelisted."""
-    return any(not is_image_whitelisted(container, IMAGES_NO_STOP_REQUIRED) for container in containers)
+    return any(not is_image_whitelisted(container, images) for container in containers)
 
-def create_volume_directory(volume_name):
+def create_volume_directory(version_dir,volume_name):
     """Create necessary directories for backup."""
-    volume_dir = os.path.join(VERSION_DIR, volume_name)
+    volume_dir = os.path.join(version_dir, volume_name)
     pathlib.Path(volume_dir).mkdir(parents=True, exist_ok=True)
     return volume_dir
 
-def is_image_ignored(container):
+def is_image_ignored(container, ignored_images):
     """Check if the container's image is one of the ignored images."""
-    for image in IMAGES_NO_BACKUP_REQUIRED:
+    for image in ignored_images:
         if has_image(container, image):
             return True
     return False
 
-def backup_with_containers_paused(volume_name, volume_dir, containers):
-    stop_containers(containers)
-    backup_volume(volume_name, volume_dir)
-    start_containers(containers)
-
-def backup_mariadb_or_postgres(container, volume_dir):
-    '''Performs database image specific backup procedures'''
-    for image in ['mariadb','postgres']:
-        if has_image(container, image):
-            backup_database(container, volume_dir, image)
-            return True
-    return False
-
-
-def default_backup_routine_for_volume(volume_name, containers):
+def backup_routine_for_volume(volume_name, containers, databases, version_dir, whitelisted_images, versions_dir):
     """Perform backup routine for a given volume."""
     volume_dir=""
     for container in containers:
         
         # Skip ignored images
-        if is_image_ignored(container):
+        if is_image_ignored(container, ['redis', 'memcached']):
             print(f"Ignoring volume '{volume_name}' linked to container '{container}' with ignored image.")
             continue 
 
         # Directory which contains files and sqls
-        volume_dir = create_volume_directory(volume_name)
+        volume_dir = create_volume_directory(version_dir, volume_name)
         
-        # Execute Database backup and exit if successfull
-        if backup_mariadb_or_postgres(container, volume_dir):
+        # Execute MariaDB procedure
+        if has_image(container, 'mariadb'):
+            backup_database(container, databases, volume_dir, 'mariadb')
+            return
+
+        # Execute Postgres procedure
+        if has_image(container, 'postgres'):
+            backup_database(container, databases, volume_dir, 'postgres')
             return
 
     # Execute backup if image is not ignored
     if volume_dir:    
-        backup_volume(volume_name, volume_dir)
-        if is_container_stop_required(containers):
-            backup_with_containers_paused(volume_name, volume_dir, containers)
+        backup_volume(volume_name, volume_dir, versions_dir)
+        if is_any_image_not_whitelisted(containers, whitelisted_images):
+            stop_containers(containers)
+            backup_volume(volume_name, volume_dir, versions_dir)
+            start_containers(containers)
 
-def backup_everything(volume_name, containers):
-    """Perform file backup routine for a given volume."""
-    volume_dir=create_volume_directory(volume_name)
 
-    # Execute sql dumps
-    for container in containers:
-        backup_mariadb_or_postgres(container, volume_dir)
-
-    # Execute file backups
-    backup_volume(volume_name, volume_dir)
-    backup_with_containers_paused(volume_name, volume_dir, containers)
 
 def main():
-    parser = argparse.ArgumentParser(description='Backup Docker volumes.')
-    parser.add_argument('--everything', action='store_true',
-                        help='Force file backup for all volumes and additional execute database dumps')
-    args = parser.parse_args()
+    print('Start backup routine...')
+    dirname = os.path.dirname(__file__)
+    repository_name = os.path.basename(dirname)
+    machine_id = get_machine_id()
+    backups_dir = '/Backups/'
+    versions_dir = os.path.join(backups_dir, machine_id, repository_name)
+    backup_time = datetime.now().strftime("%Y%m%d%H%M%S")
+    version_dir = create_version_directory(versions_dir, backup_time)
 
     print('Start volume backups...')
+    databases = pandas.read_csv(os.path.join(dirname, "databases.csv"), sep=";")
     volume_names = execute_shell_command("docker volume ls --format '{{.Name}}'")
     
+    # This whitelist is configurated for https://github.com/kevinveenbirkenbach/backup-docker-to-local 
+    stop_and_restart_not_needed = [
+        # 'baserow', Doesn't use an extra database
+        'element',
+        'gitea',
+        'listmonk',
+        'mastodon',
+        'matomo',
+        'memcached',
+        'nextcloud',
+        'openproject',
+        'pixelfed',
+        'redis',
+        'wordpress' 
+    ]
+    
     for volume_name in volume_names:
         print(f'Start backup routine for volume: {volume_name}')
         containers = execute_shell_command(f"docker ps --filter volume=\"{volume_name}\" --format '{{{{.Names}}}}'")
         if not containers:
             print('Skipped due to no running containers using this volume.')
             continue
-        if args.everything:
-            backup_everything(volume_name, containers)
-        else:    
-            default_backup_routine_for_volume(volume_name, containers)
+        
+        backup_routine_for_volume(volume_name, containers, databases, version_dir, stop_and_restart_not_needed, versions_dir)
 
     print('Finished volume backups.')