Implemented deletion of not fully pulled backups

README.md

@@ -35,7 +35,7 @@ This software shipts the following tools which are natively setup on the server:
 - [Docker Health Check](./roles/server_native-docker-health-check/) - Checks the health of docker containers
 - [Docker Reverse Proxy](./roles/server_native-docker-reverse-proxy/README.md) - Docker Reverse Proxy Solution
 - [Docker Volume Backup](./roles/server_native-docker-volume-backup/) - Backup Solution for Docker Volumes
-- [Pull Primary Backups](./roles/server_native-pull-primary-backups/README.md) - Pulls the backups from another server and stores them
+- [Pull Primary Backups](./roles/server_native-backups-consumer/README.md) - Pulls the backups from another server and stores them
 - [Wireguard](./roles/server_native-wireguard/README.md) - Integrates the server in an wireguard vpn
 
 ### Server Administration

playbook.yml

@@ -184,7 +184,7 @@
   hosts: replica_backup
   become: true
   roles:
-    - role: server_native-pull-primary-backups
+    - role: server_native-backups-consumer
 
 ## PC services
 - name: general host setup

roles/server_native-backups-cleanup/files/backups-cleanup.py

@@ -2,6 +2,7 @@ import psutil
 import shutil
 import os
 import argparse
+import sys
 
 # Validating arguments
 parser = argparse.ArgumentParser()
@@ -9,24 +10,41 @@ parser.add_argument('--maximum-backup-size-percent', type=int, dest='maximum_bac
 parser.add_argument('--backups-folder-path',type=str,dest='backups_folder_path',required=True, help="The folder in which the backups are stored")
 args = parser.parse_args()
 
-deleted = True
-while psutil.disk_usage(args.backups_folder_path).percent > args.maximum_backup_size_percent and deleted:
-    deleted = False
-    print("%d %% of disk %s are used. Freeing space..." % (psutil.disk_usage(args.backups_folder_path).percent,args.backups_folder_path))
-    for host_backup_directory_name in os.listdir(args.backups_folder_path):
-        host_backup_directory_path = os.path.join(args.backups_folder_path, host_backup_directory_name)
-        for application_directory in os.listdir(host_backup_directory_path):
-            
-            # The directory which contains all backup versions of the application
-            versions_directory = os.path.join(host_backup_directory_path, application_directory) + "/"
-                        
-            versions = os.listdir(versions_directory)
-            versions.sort(reverse=False)
-            if len(versions) >= 1:
-                delete_diff = versions_directory + versions[0]
-                print("Deleting %s..." % (delete_diff))
-                shutil.rmtree(delete_diff)
-                deleted = True
-if not deleted:
-    print("All versions had been deleted!")
-print("Cleaning up finished: %d %% of disk %s are used." % (psutil.disk_usage(args.backups_folder_path).percent,args.backups_folder_path))
+def print_used_disc_space():
+    print("%d %% of disk %s are used" % (psutil.disk_usage(args.backups_folder_path).percent,args.backups_folder_path))  
+    
+warning_counter=0
+
+for host_backup_directory_name in os.listdir(args.backups_folder_path):
+    host_backup_directory_path = os.path.join(args.backups_folder_path, host_backup_directory_name)
+    for application_directory in os.listdir(host_backup_directory_path):
+        
+        # The directory which contains all backup versions of the application
+        versions_directory = os.path.join(host_backup_directory_path, application_directory) + "/"
+                    
+        versions = os.listdir(versions_directory)
+        versions.sort(reverse=False)
+        versions_counter=len(versions)
+        
+        print_used_disc_space()  
+        for version in versions:
+            version_path=os.path.join(versions_directory, version)
+            version_status_pulling_path=os.path.join(versions_directory, version, ".pulling")
+            print("Checking directory %s ..." % (version_path))
+            if psutil.disk_usage(args.backups_folder_path).percent > args.maximum_backup_size_percent:
+                if versions_counter >= 1:
+                    print("Deleting %s to free space" % (version_path))
+                    shutil.rmtree(version_path)
+                    versions_counter-=1
+                else:
+                    print("Deletion not possible. There needs to be at least one backup version")
+                    warning_counter+=1
+            elif os.path.exists(version_status_pulling_path):
+                last_version=versions[-1]
+                if last_version != version:
+                    print("Deleting %s due to unfinished pull" % (version_path))
+                    shutil.rmtree(version_path)
+                    versions_counter-=1
+print_used_disc_space()            
+print("Cleaning up finished.")
+sys.exit(warning_counter)

roles/server_native-backups-consumer/README.md

@@ -1,4 +1,4 @@
-# role server_native-pull-primary-backups
+# role server_native-backups-consumer
 
 ## goal
 This script allows to pull backups from a remote server.

roles/server_native-backups-consumer/files/pull-remote-backup.sh

@@ -45,10 +45,19 @@ for backup_type in $remote_backup_types; do
     echo "creating local backup destination folder..." &&
     mkdir -vp "$local_backup_destination_path" &&
 
+    status_pulling_file="$local_backup_destination_path/.pulling" &&
+    echo "creating:                 $status_pulling_file" &&
+    echo "pulling backup since $(date)" > $status_pulling_file &&
+
     echo "starting backup..." &&
     rsync_command='rsync -abP --delete --delete-excluded --rsync-path="sudo rsync" --link-dest="'$local_previous_version_dir'" "'$remote_source_path'" "'$local_backup_destination_path'"' &&
     echo "executing:                $rsync_command" &&
-    eval "$rsync_command" || ((errors+=1));
+    eval "$rsync_command" &&
+    
+    echo "removing:                $status_pulling_file" &&
+    rm -vf $status_pulling_file
+
+    || ((errors+=1));
   fi
 done
 exit $errors;

roles/server_native-backups-provider-user/readme.md

@@ -1,4 +1,4 @@
-# role server_native-user-backup
+# role server_native-backups-provider-user
 User for backups
 
 ## todo

roles/server_native-backups-provider/README.md

@@ -1,4 +1,4 @@
-# role server_native-primary-backup-host
+# role server_native-backups-provider-host
 
 ## todo
 - add full system backup

roles/server_native-backups-provider/meta/main.yml

@@ -1,3 +1,3 @@
 dependencies:
-- server_native-user-backup
+- server_native-backups-provider-user
 - server_native-backups-cleanup

roles/server_native-docker-volume-backup/meta/main.yml

@@ -1,4 +1,4 @@
 dependencies:
   - server_native-git
-  - server_native-primary-backup
+  - server_native-backups-provider
   - server_native-systemd-email