removed parameter bug

backup-docker-to-local.py

@@ -7,6 +7,7 @@ import re
 import pathlib
 import pandas
 from datetime import datetime
+import argparse
 
 class BackupException(Exception):
     """Generic exception for backup errors."""
@@ -21,28 +22,58 @@ def execute_shell_command(command):
         raise BackupException(f"Error in command: {command}\nOutput: {out}\nError: {err}\nExit code: {process.returncode}")
     return [line.decode("utf-8") for line in out.splitlines()]
 
+def create_version_directory():
+    """Create necessary directories for backup."""
+    version_dir = os.path.join(VERSIONS_DIR, BACKUP_TIME)
+    pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
+    return version_dir
+
 def get_machine_id():
     """Get the machine identifier."""
     return execute_shell_command("sha256sum /etc/machine-id")[0][0:64]
 
-def create_version_directory(versions_dir, backup_time):
-    """Create necessary directories for backup."""
-    version_dir = os.path.join(versions_dir, backup_time)
-    pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
-    return version_dir
+### GLOBAL CONFIGURATION ###
+
+IMAGES_NO_STOP_REQUIRED = [
+    # 'baserow', Doesn't use an extra database
+    'element',
+    'gitea',
+    'listmonk',
+    'mastodon',
+    'matomo',
+    'nextcloud',
+    'openproject',
+    'pixelfed',
+    'wordpress' 
+]
+
+IMAGES_NO_BACKUP_REQUIRED = [
+    'redis', 
+    'memcached'
+    ]
+
+DIRNAME = os.path.dirname(__file__)
+
+DATABASES = pandas.read_csv(os.path.join(DIRNAME, "databases.csv"), sep=";")
+REPOSITORY_NAME = os.path.basename(DIRNAME)
+MACHINE_ID = get_machine_id()
+BACKUPS_DIR = '/Backups/'
+VERSIONS_DIR = os.path.join(BACKUPS_DIR, MACHINE_ID, REPOSITORY_NAME)
+BACKUP_TIME = datetime.now().strftime("%Y%m%d%H%M%S")
+VERSION_DIR = create_version_directory()
 
 def get_instance(container):
     instance_name = re.split("(_|-)(database|db|postgres)", container)[0]
     print(f"Extracted instance name: {instance_name}")
     return instance_name
 
-def backup_database(container, databases, volume_dir, db_type):
+def backup_database(container, volume_dir, db_type):
     """Backup database (MariaDB or PostgreSQL) if applicable."""
     print(f"Starting database backup for {container} using {db_type}...")
     instance_name = get_instance(container)
 
     # Filter the DataFrame for the given instance_name
-    database_entries = databases.loc[databases['instance'] == instance_name]
+    database_entries = DATABASES.loc[DATABASES['instance'] == instance_name]
 
     # Check if there are more than one entries
     if len(database_entries) > 1:
@@ -80,11 +111,11 @@ def backup_database(container, databases, volume_dir, db_type):
     execute_shell_command(backup_command)
     print(f"Database backup for {container} completed.")
 
-def get_last_backup_dir(versions_dir, volume_name, current_backup_dir):
+def get_last_backup_dir(volume_name, current_backup_dir):
     """Get the most recent backup directory for the specified volume."""
-    versions = sorted(os.listdir(versions_dir), reverse=True)
+    versions = sorted(os.listdir(VERSIONS_DIR), reverse=True)
     for version in versions:
-        backup_dir = os.path.join(versions_dir, version, volume_name, "files")
+        backup_dir = os.path.join(VERSIONS_DIR, version, volume_name, "files")
         # Ignore current backup dir
         if backup_dir != current_backup_dir:
             if os.path.isdir(backup_dir):
@@ -92,13 +123,13 @@ def get_last_backup_dir(versions_dir, volume_name, current_backup_dir):
     print(f"No previous backups available for volume: {volume_name}")
     return None
 
-def backup_volume(volume_name, volume_dir, versions_dir):
+def backup_volume(volume_name, volume_dir):
     """Backup files of a volume with incremental backups."""
     print(f"Starting backup routine for volume: {volume_name}")
     files_rsync_destination_path = os.path.join(volume_dir, "files")
     pathlib.Path(files_rsync_destination_path).mkdir(parents=True, exist_ok=True)
 
-    last_backup_dir = get_last_backup_dir(versions_dir, volume_name, files_rsync_destination_path)
+    last_backup_dir = get_last_backup_dir(volume_name, files_rsync_destination_path)
     link_dest_option = f"--link-dest='{last_backup_dir}'" if last_backup_dir else ""
 
     source_dir = f"/var/lib/docker/volumes/{volume_name}/_data/"
@@ -142,94 +173,91 @@ def is_image_whitelisted(container, images):
             return True
     return False
 
-def is_any_image_not_whitelisted(containers, images):
+def is_container_stop_required(containers):
     """Check if any of the containers are using images that are not whitelisted."""
-    return any(not is_image_whitelisted(container, images) for container in containers)
+    return any(not is_image_whitelisted(container, IMAGES_NO_STOP_REQUIRED) for container in containers)
 
-def create_volume_directory(version_dir,volume_name):
+def create_volume_directory(volume_name):
     """Create necessary directories for backup."""
-    volume_dir = os.path.join(version_dir, volume_name)
+    volume_dir = os.path.join(VERSION_DIR, volume_name)
     pathlib.Path(volume_dir).mkdir(parents=True, exist_ok=True)
     return volume_dir
 
-def is_image_ignored(container, ignored_images):
+def is_image_ignored(container):
     """Check if the container's image is one of the ignored images."""
-    for image in ignored_images:
+    for image in IMAGES_NO_BACKUP_REQUIRED:
         if has_image(container, image):
             return True
     return False
 
-def backup_routine_for_volume(volume_name, containers, databases, version_dir, whitelisted_images, versions_dir):
+def backup_with_containers_paused(volume_name, volume_dir, containers):
+    stop_containers(containers)
+    backup_volume(volume_name, volume_dir)
+    start_containers(containers)
+
+def backup_mariadb_or_postgres(container, volume_dir):
+    '''Performs database image specific backup procedures'''
+    for image in ['mariadb','postgres']:
+        if has_image(container, image):
+            backup_database(container, volume_dir, image)
+            return True
+    return False
+
+
+def default_backup_routine_for_volume(volume_name, containers):
     """Perform backup routine for a given volume."""
     volume_dir=""
     for container in containers:
         
         # Skip ignored images
-        if is_image_ignored(container, ['redis', 'memcached']):
+        if is_image_ignored(container):
             print(f"Ignoring volume '{volume_name}' linked to container '{container}' with ignored image.")
             continue 
 
         # Directory which contains files and sqls
-        volume_dir = create_volume_directory(version_dir, volume_name)
+        volume_dir = create_volume_directory(volume_name)
         
-        # Execute MariaDB procedure
-        if has_image(container, 'mariadb'):
-            backup_database(container, databases, volume_dir, 'mariadb')
-            return
-
-        # Execute Postgres procedure
-        if has_image(container, 'postgres'):
-            backup_database(container, databases, volume_dir, 'postgres')
+        # Execute Database backup and exit if successfull
+        if backup_mariadb_or_postgres(container, volume_dir):
             return
 
     # Execute backup if image is not ignored
     if volume_dir:    
-        backup_volume(volume_name, volume_dir, versions_dir)
-        if is_any_image_not_whitelisted(containers, whitelisted_images):
-            stop_containers(containers)
-            backup_volume(volume_name, volume_dir, versions_dir)
-            start_containers(containers)
+        backup_volume(volume_name, volume_dir)
+        if is_container_stop_required(containers):
+            backup_with_containers_paused(volume_name, volume_dir, containers)
 
+def backup_everything(volume_name, containers):
+    """Perform file backup routine for a given volume."""
+    volume_dir=create_volume_directory(volume_name)
     
+    # Execute sql dumps
+    for container in containers:
+        backup_mariadb_or_postgres(container, volume_dir)
+
+    # Execute file backups
+    backup_volume(volume_name, volume_dir)
+    backup_with_containers_paused(volume_name, volume_dir, containers)
 
 def main():
-    print('Start backup routine...')
-    dirname = os.path.dirname(__file__)
-    repository_name = os.path.basename(dirname)
-    machine_id = get_machine_id()
-    backups_dir = '/Backups/'
-    versions_dir = os.path.join(backups_dir, machine_id, repository_name)
-    backup_time = datetime.now().strftime("%Y%m%d%H%M%S")
-    version_dir = create_version_directory(versions_dir, backup_time)
+    parser = argparse.ArgumentParser(description='Backup Docker volumes.')
+    parser.add_argument('--everything', action='store_true',
+                        help='Force file backup for all volumes and additional execute database dumps')
+    args = parser.parse_args()
 
     print('Start volume backups...')
-    databases = pandas.read_csv(os.path.join(dirname, "databases.csv"), sep=";")
     volume_names = execute_shell_command("docker volume ls --format '{{.Name}}'")
     
-    # This whitelist is configurated for https://github.com/kevinveenbirkenbach/backup-docker-to-local 
-    stop_and_restart_not_needed = [
-        # 'baserow', Doesn't use an extra database
-        'element',
-        'gitea',
-        'listmonk',
-        'mastodon',
-        'matomo',
-        'memcached',
-        'nextcloud',
-        'openproject',
-        'pixelfed',
-        'redis',
-        'wordpress' 
-    ]
-    
     for volume_name in volume_names:
         print(f'Start backup routine for volume: {volume_name}')
         containers = execute_shell_command(f"docker ps --filter volume=\"{volume_name}\" --format '{{{{.Names}}}}'")
         if not containers:
             print('Skipped due to no running containers using this volume.')
             continue
-        
-        backup_routine_for_volume(volume_name, containers, databases, version_dir, stop_and_restart_not_needed, versions_dir)
+        if args.everything:
+            backup_everything(volume_name, containers)
+        else:    
+            default_backup_routine_for_volume(volume_name, containers)
 
     print('Finished volume backups.')