Ignored images

backup-docker-to-local.py

@@ -25,9 +25,9 @@ def get_machine_id():
     """Get the machine identifier."""
     return execute_shell_command("sha256sum /etc/machine-id")[0][0:64]
 
-def create_backup_directories(base_dir, machine_id, repository_name, backup_time):
+def create_version_directory(versions_dir, backup_time):
     """Create necessary directories for backup."""
-    version_dir = os.path.join(base_dir, machine_id, repository_name, backup_time)
+    version_dir = os.path.join(versions_dir, backup_time)
     pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
     return version_dir
 
@@ -36,7 +36,7 @@ def get_instance(container):
     print(f"Extracted instance name: {instance_name}")
     return instance_name
 
-def backup_database(container, databases, version_dir, db_type):
+def backup_database(container, databases, volume_dir, db_type):
     """Backup database (MariaDB or PostgreSQL) if applicable."""
     print(f"Starting database backup for {container} using {db_type}...")
     instance_name = get_instance(container)
@@ -55,7 +55,7 @@ def backup_database(container, databases, version_dir, db_type):
     # Get the first (and only) entry
     database_entry = database_entries.iloc[0]
 
-    backup_destination_dir = os.path.join(version_dir, "sql")
+    backup_destination_dir = os.path.join(volume_dir, "sql")
     pathlib.Path(backup_destination_dir).mkdir(parents=True, exist_ok=True)
     backup_destination_file = os.path.join(backup_destination_dir, f"backup.sql")
     
@@ -80,19 +80,38 @@ def backup_database(container, databases, version_dir, db_type):
     execute_shell_command(backup_command)
     print(f"Database backup for {container} completed.")
 
-def backup_volume(volume_name, version_dir):
-    """Backup files of a volume."""
+def get_last_backup_dir(versions_dir, volume_name, current_backup_dir):
+    """Get the most recent backup directory for the specified volume."""
+    versions = sorted(os.listdir(versions_dir), reverse=True)
+    for version in versions:
+        backup_dir = os.path.join(versions_dir, version, volume_name, "files")
+        # Ignore current backup dir
+        if backup_dir != current_backup_dir:
+            if os.path.isdir(backup_dir):
+                return backup_dir
+    print(f"No previous backups available for volume: {volume_name}")
+    return None
+
+def backup_volume(volume_name, volume_dir, versions_dir):
+    """Backup files of a volume with incremental backups."""
     print(f"Starting backup routine for volume: {volume_name}")
-    files_rsync_destination_path = os.path.join(version_dir, volume_name, "files")
+    files_rsync_destination_path = os.path.join(volume_dir, "files")
     pathlib.Path(files_rsync_destination_path).mkdir(parents=True, exist_ok=True)
+
+    last_backup_dir = get_last_backup_dir(versions_dir, volume_name, files_rsync_destination_path)
+    link_dest_option = f"--link-dest='{last_backup_dir}'" if last_backup_dir else ""
+
     source_dir = f"/var/lib/docker/volumes/{volume_name}/_data/"
-    rsync_command = f"rsync -abP --delete --delete-excluded {source_dir} {files_rsync_destination_path}"
+    rsync_command = f"rsync -abP --delete --delete-excluded {link_dest_option} {source_dir} {files_rsync_destination_path}"
     execute_shell_command(rsync_command)
     print(f"Backup routine for volume: {volume_name} completed.")
 
+def get_image_info(container):
+    return execute_shell_command(f"docker inspect {container} | jq -r '.[].Config.Image'")
+
 def has_image(container,image):
     """Check if the container is using the image"""
-    image_info = execute_shell_command(f"docker inspect {container} | jq -r '.[].Config.Image'")
+    image_info = get_image_info(container)
     return image in image_info[0]
 
 def stop_containers(containers):
@@ -115,7 +134,7 @@ def get_container_with_image(containers,image):
 
 def is_image_whitelisted(container, images):
     """Check if the container's image is one of the whitelisted images."""
-    image_info = execute_shell_command(f"docker inspect {container} | jq -r '.[].Config.Image'")
+    image_info = get_image_info(container)
     container_image = image_info[0]
 
     for image in images:
@@ -127,20 +146,44 @@ def is_any_image_not_whitelisted(containers, images):
     """Check if any of the containers are using images that are not whitelisted."""
     return any(not is_image_whitelisted(container, images) for container in containers)
 
-def backup_routine_for_volume(volume_name, containers, databases, version_dir, whitelisted_images):
+def create_volume_directory(version_dir,volume_name):
+    """Create necessary directories for backup."""
+    volume_dir = os.path.join(version_dir, volume_name)
+    pathlib.Path(volume_dir).mkdir(parents=True, exist_ok=True)
+    return volume_dir
+
+def is_image_ignored(container, ignored_images):
+    """Check if the container's image is one of the ignored images."""
+    for image in ignored_images:
+        if has_image(container, image):
+            return True
+    return False
+
+def backup_routine_for_volume(volume_name, containers, databases, version_dir, whitelisted_images, versions_dir):
     """Perform backup routine for a given volume."""
+    ignored_images = ['redis', 'memcached']
+
     for container in containers:
+        if is_image_ignored(container, ignored_images):
+            print(f"Ignoring volume '{volume_name}' linked to container '{container}' with ignored image.")
+            continue
+
+        volume_dir = create_volume_directory(version_dir, volume_name)
         if has_image(container, 'mariadb'):
-            backup_database(container, databases, version_dir, 'mariadb')
-        elif has_image(container, 'postgres'):
-            backup_database(container, databases, version_dir, 'postgres')
-        else:
-            if is_any_image_not_whitelisted(containers, whitelisted_images):
-                stop_containers(containers)
-                backup_volume(volume_name, version_dir)
-                start_containers(containers)
-            else:
-                backup_volume(volume_name, version_dir)
+            backup_database(container, databases, volume_dir, 'mariadb')
+            continue
+        
+        if has_image(container, 'postgres'):
+            backup_database(container, databases, volume_dir, 'postgres')
+            continue
+        
+        backup_volume(volume_name, volume_dir, versions_dir)
+        if is_any_image_not_whitelisted(containers, whitelisted_images):
+            stop_containers(containers)
+            backup_volume(volume_name, volume_dir, versions_dir)
+            start_containers(containers)
+
+
 
 def main():
     print('Start backup routine...')
@@ -148,8 +191,9 @@ def main():
     repository_name = os.path.basename(dirname)
     machine_id = get_machine_id()
     backups_dir = '/Backups/'
+    versions_dir = os.path.join(backups_dir, machine_id, repository_name)
     backup_time = datetime.now().strftime("%Y%m%d%H%M%S")
-    version_dir = create_backup_directories(backups_dir, machine_id, repository_name, backup_time)
+    version_dir = create_version_directory(versions_dir, backup_time)
 
     print('Start volume backups...')
     databases = pandas.read_csv(os.path.join(dirname, "databases.csv"), sep=";")
@@ -178,7 +222,7 @@ def main():
             print('Skipped due to no running containers using this volume.')
             continue
         
-        backup_routine_for_volume(volume_name, containers, databases, version_dir, stop_and_restart_not_needed)
+        backup_routine_for_volume(volume_name, containers, databases, version_dir, stop_and_restart_not_needed, versions_dir)
 
     print('Finished volume backups.')