removed backup links and versions folder

roles/server_native-backups-cleanup/files/backups-cleanup.py

@@ -13,11 +13,13 @@ deleted = True
 while psutil.disk_usage(args.backups_folder_path).percent > args.maximum_backup_size_percent and deleted:
     deleted = False
     print("%d %% of disk %s are used. Freeing space..." % (psutil.disk_usage(args.backups_folder_path).percent,args.backups_folder_path))
-    for primary_directory in os.listdir(args.backups_folder_path):
+    for host_backup_directory_name in os.listdir(args.backups_folder_path):
-        primary_directory = os.path.join(args.backups_folder_path, primary_directory)
+        host_backup_directory_path = os.path.join(args.backups_folder_path, host_backup_directory_name)
-        for application_directory in os.listdir(primary_directory):
+        for application_directory in os.listdir(host_backup_directory_path):
-            application_directory = os.path.join(primary_directory, application_directory)
+            
-            versions_directory = os.path.join(application_directory, "versions/")
+            # The directory which contains all backup versions of the application
+            versions_directory = os.path.join(host_backup_directory_path, application_directory)
+                        
             versions = os.listdir(versions_directory)
             versions.sort(reverse=False)
             if len(versions) >= 1:

roles/server_native-pull-primary-backups/files/pull-remote-backup.sh

@@ -1,63 +1,53 @@
 #!/bin/bash
 # @param $1 hostname from which backup should be pulled
 
-echo "pulling backups from: $1"
+echo "pulling backups from: $1" &&
 
 # error counter
-errors=0
+errors=0 &&
 
-echo "loading meta data..."
+echo "loading meta data..." &&
 
-remote_host="backup@$1"
+remote_host="backup@$1" &&
-echo "host address:         $remote_host"
+echo "host address:         $remote_host" &&
 
-remote_machine_id="$( (ssh "$remote_host" sha256sum /etc/machine-id) | head -c 64 )" || exit 1
+remote_machine_id="$( (ssh "$remote_host" sha256sum /etc/machine-id) | head -c 64 )" &&
-echo "remote machine id:    $remote_machine_id"
+echo "remote machine id:    $remote_machine_id" &&
 
-general_backup_machine_dir="/Backups/$remote_machine_id/"
+general_backup_machine_dir="/Backups/$remote_machine_id/" &&
-echo "backup dir:           $general_backup_machine_dir"
+echo "backup dir:           $general_backup_machine_dir" &&
 
-remote_backup_types="$(ssh "$remote_host" "find $general_backup_machine_dir -maxdepth 1 -type d -execdir basename {} ;")" || exit 1
+remote_backup_types="$(ssh "$remote_host" "find $general_backup_machine_dir -maxdepth 1 -type d -execdir basename {} ;")" &&
-echo "backuptypes:          $remote_backup_types"
+echo "backup types:          $remote_backup_types" || exit 1
 
 for backup_type in $remote_backup_types; do
   if [ "$backup_type" != "$remote_machine_id" ]; then
-    general_backup_type_dir="$general_backup_machine_dir""$backup_type/"
+    echo "backup type:              $backup_type" &&
-    # folder which contains versions
-    general_versions_dir="$general_backup_type_dir""versions/"
-    # link name of last backup
-    general_latest_version_link="$general_backup_type_dir""latest"
     
-    # this folder contains the last backup
+    general_backup_type_dir="$general_backup_machine_dir""$backup_type/" &&
-    local_latest_version_dir="$general_versions_dir$(date '+%Y%m%d%H%M%S')/"
+    general_versions_dir="$general_backup_type_dir" &&
-    # this is the link name of the previous version
+    local_previous_version_dir="$(ls -d $general_versions_dir* | tail -1)" &&
-    local_previous_version_link="$general_backup_type_dir""previous"
+    echo "last local backup:      $local_previous_version_dir" &&
 
-    #identifiy previous version
+    remote_backup_versions="$(ssh "$remote_host" ls -d "$general_backup_type_dir"\*)" &&
-    local_versions=( $(basename -a "$general_versions_dir"*/ | sort) )|| exit 1
+    echo "remote backup versions:   $remote_backup_versions" &&
-    local_last_version="${local_versions[-1]}" || exit 1
-    local_previous_version_dir="$general_versions_dir""$local_last_version/"
 
-    # source path of the backup files:
-    remote_last_version_dir="$(ssh "$remote_host" readlink -f $general_latest_version_link)"
-    echo "last remote backup: $remote_last_version_dir"
-    remote_source_path="$remote_host:$remote_last_version_dir/"
-    echo "source path:        $remote_source_path"
 
-    # create working folders:
+    remote_last_backup_dir=$(echo "$remote_backup_versions" | tail -1) &&
-    mkdir -vp "$local_latest_version_dir"
+    echo "last remote backup:       $remote_last_backup_dir" &&
 
-    # delete links
+    remote_source_path="$remote_host:$remote_last_backup_dir/" &&
-    rm -v "$general_latest_version_link"
+    echo "source path:              $remote_source_path" &&
-    rm -v "$local_previous_version_link"
 
-    # create links
+    local_backup_destination_path=$remote_last_backup_dir &&
-    ln -vs "$local_latest_version_dir" "$general_latest_version_link" || exit 1
+    echo "backup destination:       $local_backup_destination_path" &&
-    ln -vs "$local_previous_version_dir" "$local_previous_version_link" || exit 1
 
-    # do backup:
+    echo "creating local backup destination folder..." &&
-    rsync_command='rsync -abP --delete --delete-excluded --rsync-path="sudo rsync" --link-dest="'$local_previous_version_link'" "'$remote_source_path'" "'$general_latest_version_link'"'
+    mkdir -vp "$local_backup_destination_path" &&
-    echo "executing: $rsync_command"
+
+    echo "starting backup..." &&
+    rsync_command='rsync -abP --delete --delete-excluded --rsync-path="sudo rsync" --link-dest="'$local_previous_version_dir'" "'$remote_source_path'" "'$local_backup_destination_path'"' &&
+    echo "executing:                $rsync_command" &&
     eval "$rsync_command" || ((errors+=1));
   fi
 done

roles/server_native-user-backup/files/ssh-wrapper.sh

@@ -8,19 +8,22 @@ fi
 
 # define executable commands
 get_hashed_machine_id="sha256sum /etc/machine-id";
-get_backup_types="find /Backups/{{hashed_machine_id.stdout}}/ -maxdepth 1 -type d -execdir basename {} ;";
+hashed_machine_id="$($get_hashed_machine_id | head -c 64)"
+get_backup_types="find /Backups/$hashed_machine_id/ -maxdepth 1 -type d -execdir basename {} ;";
+
 
 # @todo This configuration is not scalable yet. If other backup services then docker-volume-backup are integrated, this logic needs to be optimized
-get_static_last_version_dir="readlink -f /Backups/{{hashed_machine_id.stdout}}/docker-volume-backup/latest"
+get_version_directories="ls -d /Backups/$hashed_machine_id/docker-volume-backup/*"
-rsync_command="sudo rsync --server --sender -blogDtpre.iLsfxCIvu . $($get_static_last_version_dir)/"
+last_version_directory="$($get_version_directories | tail -1)"
+rsync_command="sudo rsync --server --sender -blogDtpre.iLsfxCIvu . $last_version_directory/"
 
 # filter commands
 case "$SSH_ORIGINAL_COMMAND" in
 	"$get_hashed_machine_id")
 		$get_hashed_machine_id
 		;;
-	"$get_static_last_version_dir")
+	"$get_version_directories")
-		$get_static_last_version_dir
+		$get_version_directories
 		;;
 	"$get_backup_types")
 		$get_backup_types

roles/server_native-user-backup/tasks/main.yml

@@ -11,10 +11,6 @@
     group: backup
     mode: '0700'
 
-- name: register hashed_machine_id
-  shell: sha256sum /etc/machine-id | head -c 64
-  register: hashed_machine_id
-
 - name: create /home/backup/.ssh/authorized_keys
   template:
     src: "authorized_keys.j2"
@@ -24,8 +20,8 @@
     mode: '0644'
 
 - name: create /home/backup/ssh-wrapper.sh
-  template:
+  copy:
-    src: "ssh-wrapper.sh.j2"
+    src: "ssh-wrapper.sh"
     dest: /home/backup/ssh-wrapper.sh
     owner: backup
     group: backup