backup-docker-to-local.py

@@ -25,9 +25,9 @@ def get_machine_id():
     """Get the machine identifier."""
     return execute_shell_command("sha256sum /etc/machine-id")[0][0:64]
 
-def create_version_directory(versions_dir, backup_time):
+def create_backup_directories(base_dir, machine_id, repository_name, backup_time):
     """Create necessary directories for backup."""
-    version_dir = os.path.join(versions_dir, backup_time)
+    version_dir = os.path.join(base_dir, machine_id, repository_name, backup_time)
     pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
     return version_dir
 
@@ -36,7 +36,7 @@ def get_instance(container):
     print(f"Extracted instance name: {instance_name}")
     return instance_name
 
-def backup_database(container, databases, volume_dir, db_type):
+def backup_database(container, databases, version_dir, db_type):
     """Backup database (MariaDB or PostgreSQL) if applicable."""
     print(f"Starting database backup for {container} using {db_type}...")
     instance_name = get_instance(container)
@@ -55,7 +55,7 @@ def backup_database(container, databases, volume_dir, db_type):
     # Get the first (and only) entry
     database_entry = database_entries.iloc[0]
 
-    backup_destination_dir = os.path.join(volume_dir, "sql")
+    backup_destination_dir = os.path.join(version_dir, "sql")
     pathlib.Path(backup_destination_dir).mkdir(parents=True, exist_ok=True)
     backup_destination_file = os.path.join(backup_destination_dir, f"backup.sql")
     
@@ -80,38 +80,19 @@ def backup_database(container, databases, volume_dir, db_type):
     execute_shell_command(backup_command)
     print(f"Database backup for {container} completed.")
 
-def get_last_backup_dir(versions_dir, volume_name, current_backup_dir):
+def backup_volume(volume_name, version_dir):
-    """Get the most recent backup directory for the specified volume."""
+    """Backup files of a volume."""
-    versions = sorted(os.listdir(versions_dir), reverse=True)
-    for version in versions:
-        backup_dir = os.path.join(versions_dir, version, volume_name, "files")
-        # Ignore current backup dir
-        if backup_dir != current_backup_dir:
-            if os.path.isdir(backup_dir):
-                return backup_dir
-    print(f"No previous backups available for volume: {volume_name}")
-    return None
-
-def backup_volume(volume_name, volume_dir, versions_dir):
-    """Backup files of a volume with incremental backups."""
     print(f"Starting backup routine for volume: {volume_name}")
-    files_rsync_destination_path = os.path.join(volume_dir, "files")
+    files_rsync_destination_path = os.path.join(version_dir, volume_name, "files")
     pathlib.Path(files_rsync_destination_path).mkdir(parents=True, exist_ok=True)
-
-    last_backup_dir = get_last_backup_dir(versions_dir, volume_name, files_rsync_destination_path)
-    link_dest_option = f"--link-dest='{last_backup_dir}'" if last_backup_dir else ""
-
     source_dir = f"/var/lib/docker/volumes/{volume_name}/_data/"
-    rsync_command = f"rsync -abP --delete --delete-excluded {link_dest_option} {source_dir} {files_rsync_destination_path}"
+    rsync_command = f"rsync -abP --delete --delete-excluded {source_dir} {files_rsync_destination_path}"
     execute_shell_command(rsync_command)
     print(f"Backup routine for volume: {volume_name} completed.")
 
-def get_image_info(container):
-    return execute_shell_command(f"docker inspect {container} | jq -r '.[].Config.Image'")
-
 def has_image(container,image):
     """Check if the container is using the image"""
-    image_info = get_image_info(container)
+    image_info = execute_shell_command(f"docker inspect {container} | jq -r '.[].Config.Image'")
     return image in image_info[0]
 
 def stop_containers(containers):
@@ -134,7 +115,7 @@ def get_container_with_image(containers,image):
 
 def is_image_whitelisted(container, images):
     """Check if the container's image is one of the whitelisted images."""
-    image_info = get_image_info(container)
+    image_info = execute_shell_command(f"docker inspect {container} | jq -r '.[].Config.Image'")
     container_image = image_info[0]
 
     for image in images:
@@ -146,44 +127,20 @@ def is_any_image_not_whitelisted(containers, images):
     """Check if any of the containers are using images that are not whitelisted."""
     return any(not is_image_whitelisted(container, images) for container in containers)
 
-def create_volume_directory(version_dir,volume_name):
+def backup_routine_for_volume(volume_name, containers, databases, version_dir, whitelisted_images):
-    """Create necessary directories for backup."""
-    volume_dir = os.path.join(version_dir, volume_name)
-    pathlib.Path(volume_dir).mkdir(parents=True, exist_ok=True)
-    return volume_dir
-
-def is_image_ignored(container, ignored_images):
-    """Check if the container's image is one of the ignored images."""
-    for image in ignored_images:
-        if has_image(container, image):
-            return True
-    return False
-
-def backup_routine_for_volume(volume_name, containers, databases, version_dir, whitelisted_images, versions_dir):
     """Perform backup routine for a given volume."""
-    ignored_images = ['redis', 'memcached']
-
     for container in containers:
-        if is_image_ignored(container, ignored_images):
-            print(f"Ignoring volume '{volume_name}' linked to container '{container}' with ignored image.")
-            continue
-
-        volume_dir = create_volume_directory(version_dir, volume_name)
         if has_image(container, 'mariadb'):
-            backup_database(container, databases, volume_dir, 'mariadb')
+            backup_database(container, databases, version_dir, 'mariadb')
-            continue
+        elif has_image(container, 'postgres'):
-        
+            backup_database(container, databases, version_dir, 'postgres')
-        if has_image(container, 'postgres'):
+        else:
-            backup_database(container, databases, volume_dir, 'postgres')
+            if is_any_image_not_whitelisted(containers, whitelisted_images):
-            continue
+                stop_containers(containers)
-        
+                backup_volume(volume_name, version_dir)
-        backup_volume(volume_name, volume_dir, versions_dir)
+                start_containers(containers)
-        if is_any_image_not_whitelisted(containers, whitelisted_images):
+            else:
-            stop_containers(containers)
+                backup_volume(volume_name, version_dir)
-            backup_volume(volume_name, volume_dir, versions_dir)
-            start_containers(containers)
-
-
 
 def main():
     print('Start backup routine...')
@@ -191,9 +148,8 @@ def main():
     repository_name = os.path.basename(dirname)
     machine_id = get_machine_id()
     backups_dir = '/Backups/'
-    versions_dir = os.path.join(backups_dir, machine_id, repository_name)
     backup_time = datetime.now().strftime("%Y%m%d%H%M%S")
-    version_dir = create_version_directory(versions_dir, backup_time)
+    version_dir = create_backup_directories(backups_dir, machine_id, repository_name, backup_time)
 
     print('Start volume backups...')
     databases = pandas.read_csv(os.path.join(dirname, "databases.csv"), sep=";")
@@ -222,7 +178,7 @@ def main():
             print('Skipped due to no running containers using this volume.')
             continue
         
-        backup_routine_for_volume(volume_name, containers, databases, version_dir, stop_and_restart_not_needed, versions_dir)
+        backup_routine_for_volume(volume_name, containers, databases, version_dir, stop_and_restart_not_needed)
 
     print('Finished volume backups.')