From d721d451e0bc2ebc8b3fb464ce7f1151198372d2 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 12 Apr 2023 20:32:52 +0200 Subject: [PATCH] solved pull backup bugs --- roles/native-pull-primary-backups/README.md | 1 + .../files/pull-remote-backup.sh | 25 +++++++++++++++---- .../tasks/main.yml | 7 +++++- .../templates/pull-remote-backups.sh.j2 | 2 +- .../templates/ssh-wrapper.sh.j2 | 4 +-- 5 files changed, 30 insertions(+), 9 deletions(-) diff --git a/roles/native-pull-primary-backups/README.md b/roles/native-pull-primary-backups/README.md index 49364e85..f5c754f9 100644 --- a/roles/native-pull-primary-backups/README.md +++ b/roles/native-pull-primary-backups/README.md @@ -25,3 +25,4 @@ To track what the service is doing execute the following command: ## see - https://superuser.com/questions/363444/how-do-i-get-the-output-and-exit-value-of-a-subshell-when-using-bash-e - https://gist.github.com/otkrsk/b0ffd4018e8a79b9010c461af298471e +- https://serverfault.com/questions/304125/rsync-seems-incompatible-with-bashrc-causes-is-your-shell-clean \ No newline at end of file diff --git a/roles/native-pull-primary-backups/files/pull-remote-backup.sh b/roles/native-pull-primary-backups/files/pull-remote-backup.sh index e82ef3d8..696527c1 100644 --- a/roles/native-pull-primary-backups/files/pull-remote-backup.sh +++ b/roles/native-pull-primary-backups/files/pull-remote-backup.sh @@ -1,13 +1,24 @@ #!/bin/bash # @param $1 hostname from which backup should be pulled +echo "pulling backups from: $1" + # error counter errors=0 +echo "loading meta data..." + remote_host="backup@$1" +echo "host address: $remote_host" + remote_machine_id="$( (ssh "$remote_host" sha256sum /etc/machine-id) | head -c 64 )" || exit 1 +echo "remote machine id: $remote_machine_id" + general_backup_machine_dir="/Backups/$remote_machine_id/" -remote_backup_types="$(ssh "$remote_host" find "$general_backup_machine_dir" -maxdepth 1 -type d -execdir basename {} ';')" || exit 1 +echo "backup dir: $general_backup_machine_dir" + +remote_backup_types="$(ssh "$remote_host" "find $general_backup_machine_dir -maxdepth 1 -type d -execdir basename {} ;")" || exit 1 +echo "backuptypes: $remote_backup_types" for backup_type in $remote_backup_types; do if [ "$backup_type" != "$remote_machine_id" ]; then @@ -25,11 +36,13 @@ for backup_type in $remote_backup_types; do #identifiy previous version local_versions=( $(basename -a "$general_versions_dir"*/ | sort) )|| exit 1 local_last_version="${local_versions[-1]}" || exit 1 - local_previous_version_dir="$general_versions_dir""$last_version/" + local_previous_version_dir="$general_versions_dir""$local_last_version/" # source path of the backup files: - remote_last_version_dir="$(readlink -f $general_latest_version_link)" + remote_last_version_dir="$(ssh "$remote_host" readlink -f $general_latest_version_link)" + echo "last remote backup: $remote_last_version_dir" remote_source_path="$remote_host:$remote_last_version_dir/" + echo "source path: $remote_source_path" # create working folders: mkdir -vp "$local_latest_version_dir" @@ -40,10 +53,12 @@ for backup_type in $remote_backup_types; do # create links ln -vs "$local_latest_version_dir" "$general_latest_version_link" || exit 1 - ln -vs "$local_previous_version_dir" "$local_previous_version_link"|| exit 1 + ln -vs "$local_previous_version_dir" "$local_previous_version_link" || exit 1 # do backup: - rsync -abP --delete --delete-excluded --rsync-path="sudo rsync" --link-dest="$local_previous_version_link" "$remote_source_path" "$general_latest_version_link" || ((errors+=1)); + rsync_command='rsync -abP --delete --delete-excluded --rsync-path="sudo rsync" --link-dest="'$local_previous_version_link'" "'$remote_source_path'" "'$general_latest_version_link'"' + echo "executing: $rsync_command" + eval "$rsync_command" || ((errors+=1)); fi done exit $errors; diff --git a/roles/native-pull-primary-backups/tasks/main.yml b/roles/native-pull-primary-backups/tasks/main.yml index 878d2118..68d587ef 100644 --- a/roles/native-pull-primary-backups/tasks/main.yml +++ b/roles/native-pull-primary-backups/tasks/main.yml @@ -8,6 +8,7 @@ copy: src: pull-remote-backup.sh dest: "{{docker_pull_primary_backups_folder}}pull-remote-backup.sh" + mode: 0755 - name: create pull-remote-backups.service template: src=pull-remote-backups.service.j2 dest=/etc/systemd/system/pull-remote-backups.service @@ -18,4 +19,8 @@ notify: restart pull-remote-backups timer - name: create pull-remote-backups.sh - template: src=pull-remote-backups.sh.j2 dest={{docker_pull_primary_backups_folder}}pull-remote-backups.sh + template: + src: pull-remote-backups.sh.j2 + dest: "{{docker_pull_primary_backups_folder}}pull-remote-backups.sh" + mode: 0755 + diff --git a/roles/native-pull-primary-backups/templates/pull-remote-backups.sh.j2 b/roles/native-pull-primary-backups/templates/pull-remote-backups.sh.j2 index eddb8427..bc0de715 100644 --- a/roles/native-pull-primary-backups/templates/pull-remote-backups.sh.j2 +++ b/roles/native-pull-primary-backups/templates/pull-remote-backups.sh.j2 @@ -3,6 +3,6 @@ hosts="{{pull_remote_backups_hosts}}"; errors=0 for host in $hosts; do - bash {{docker_pull_primary_backups_folder}}/pull-remote-backup.sh $host || ((errors+=1)); + bash {{docker_pull_primary_backups_folder}}pull-remote-backup.sh $host || ((errors+=1)); done; exit $errors; diff --git a/roles/native-user-backup/templates/ssh-wrapper.sh.j2 b/roles/native-user-backup/templates/ssh-wrapper.sh.j2 index 61a1171b..2eb92b9d 100644 --- a/roles/native-user-backup/templates/ssh-wrapper.sh.j2 +++ b/roles/native-user-backup/templates/ssh-wrapper.sh.j2 @@ -8,10 +8,10 @@ fi # define executable commands get_hashed_machine_id="sha256sum /etc/machine-id"; -get_backup_types="find /Backups/{{hashed_machine_id.stdout}}/ -maxdepth 1 -type d -execdir basename {} ';'"; +get_backup_types="find /Backups/{{hashed_machine_id.stdout}}/ -maxdepth 1 -type d -execdir basename {} ;"; # @todo This configuration is not scalable yet. If other backup services then docker-volume-backup are integrated, this logic needs to be optimized -get_static_last_version_dir="readlink -f /Backups/{{hashed_machine_id.stdout}}/docker-volume-backup/latest/" +get_static_last_version_dir="readlink -f /Backups/{{hashed_machine_id.stdout}}/docker-volume-backup/latest" rsync_command="sudo rsync --server --sender -blogDtpre.iLsfxCIvu . $($get_static_last_version_dir)/" # filter commands