renamed to backup-remote-to-local

roles/backup-remote-to-local/README.md

@@ -0,0 +1,34 @@
+# role backup-remote-to-local
+
+## goal
+This script allows to pull backups from a remote server.
+
+## scheme
+It is part of the following scheme:
+![backup scheme](https://www.veen.world/wp-content/uploads/2020/12/server-backup-768x567.jpg) <br />
+Further information you will find [in this blog post](https://www.veen.world/2020/12/26/how-i-backup-dedicated-root-servers/).
+
+## debug
+
+### live
+To track what the service is doing execute one of the following commands:
+
+#### systemctl
+```bash
+  watch -n2 "systemctl status backup-remote-to-local.service"
+```
+
+#### journalctl
+```bash
+  journalctl -fu backup-remote-to-local.service
+```  
+
+### history
+```bash
+  sudo journalctl -u backup-remote-to-local
+```
+
+## see
+- https://superuser.com/questions/363444/how-do-i-get-the-output-and-exit-value-of-a-subshell-when-using-bash-e
+- https://gist.github.com/otkrsk/b0ffd4018e8a79b9010c461af298471e
+- https://serverfault.com/questions/304125/rsync-seems-incompatible-with-bashrc-causes-is-your-shell-clean

roles/backup-remote-to-local/files/backup-remote-to-local.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+# @param $1 hostname from which backup should be pulled
+
+echo "pulling backups from: $1" &&
+
+# error counter
+errors=0 &&
+
+echo "loading meta data..." &&
+
+remote_host="backup@$1" &&
+echo "host address:         $remote_host" &&
+
+remote_machine_id="$( (ssh "$remote_host" sha256sum /etc/machine-id) | head -c 64 )" &&
+echo "remote machine id:    $remote_machine_id" &&
+
+general_backup_machine_dir="/Backups/$remote_machine_id/" &&
+echo "backup dir:           $general_backup_machine_dir" &&
+
+remote_backup_types="$(ssh "$remote_host" "find $general_backup_machine_dir -maxdepth 1 -type d -execdir basename {} ;")" &&
+echo "backup types:          $remote_backup_types" || exit 1
+
+for backup_type in $remote_backup_types; do
+  if [ "$backup_type" != "$remote_machine_id" ]; then
+    echo "backup type:              $backup_type" &&
+    
+    general_backup_type_dir="$general_backup_machine_dir""$backup_type/" &&
+    general_versions_dir="$general_backup_type_dir" &&
+    local_previous_version_dir="$(ls -d $general_versions_dir* | tail -1)" &&
+    echo "last local backup:      $local_previous_version_dir" &&
+
+    remote_backup_versions="$(ssh "$remote_host" ls -d "$general_backup_type_dir"\*)" &&
+    echo "remote backup versions:   $remote_backup_versions" &&
+
+
+    remote_last_backup_dir=$(echo "$remote_backup_versions" | tail -1) &&
+    echo "last remote backup:       $remote_last_backup_dir" &&
+
+    remote_source_path="$remote_host:$remote_last_backup_dir/" &&
+    echo "source path:              $remote_source_path" &&
+
+    local_backup_destination_path=$remote_last_backup_dir &&
+    echo "backup destination:       $local_backup_destination_path" &&
+
+    echo "creating local backup destination folder..." &&
+    mkdir -vp "$local_backup_destination_path" &&
+
+    echo "starting backup..."
+    rsync_command='rsync -abP --delete --delete-excluded --rsync-path="sudo rsync" --link-dest="'$local_previous_version_dir'" "'$remote_source_path'" "'$local_backup_destination_path'"'
+
+    echo "executing:                $rsync_command"
+
+    retry_count=0
+    max_retries=12
+    retry_delay=300  # Retry delay in seconds (5 minutes)
+    last_retry_start=0
+    max_retry_duration=43200  # Maximum duration for a single retry attempt (12 hours)
+
+    while [[ $retry_count -lt $max_retries ]]; do
+      echo "Retry attempt: $((retry_count + 1))"
+      if [[ $retry_count -gt 0 ]]; then
+        current_time=$(date +%s)
+        last_retry_duration=$((current_time - last_retry_start))
+        if [[ $last_retry_duration -ge $max_retry_duration ]]; then
+          echo "Last retry took more than 12 hours, increasing max retries to 12."
+          max_retries=12
+        fi
+      fi
+      last_retry_start=$(date +%s)
+      eval "$rsync_command"
+      rsync_exit_code=$?
+      if [[ $rsync_exit_code -eq 0 ]]; then
+        break
+      fi
+      retry_count=$((retry_count + 1))
+      sleep $retry_delay
+    done
+
+    if [[ $rsync_exit_code -ne 0 ]]; then
+      echo "Error: rsync failed after $max_retries attempts"
+      ((errors += 1))
+    fi
+  fi
+done
+exit $errors;

roles/backup-remote-to-local/handlers/main.yml

@@ -0,0 +1,11 @@
+- name: "reload backup-remote-to-local service"
+  systemd:
+    name: backup-remote-to-local.service
+    enabled: yes
+    daemon_reload: yes
+- name: "restart backup-remote-to-local timer"
+  systemd:
+    name: backup-remote-to-local.timer
+    state: started
+    enabled: yes
+    daemon_reload: yes

roles/backup-remote-to-local/meta/main.yml

@@ -0,0 +1,5 @@
+dependencies:
+  - git
+  - systemd_notifier
+  - cleanup-backups-timer
+  - cleanup-failed-docker-backups

roles/backup-remote-to-local/tasks/main.yml

@@ -0,0 +1,26 @@
+- name: "create {{docker_pull_primary_backups_folder}}"
+  file:
+    path: "{{docker_pull_primary_backups_folder}}"
+    state: directory
+    mode: 0755
+
+- name: create backup-remote-to-local.sh
+  copy:
+    src: backup-remote-to-local.sh
+    dest: "{{docker_pull_primary_backups_folder}}backup-remote-to-local.sh"
+    mode: 0755
+
+- name: create backup-remote-to-local.service
+  template: src=backup-remote-to-local.service.j2 dest=/etc/systemd/system/backup-remote-to-local.service
+  notify: reload backup-remote-to-local service
+
+- name: create backup-remote-to-local.timer
+  template: src=backup-remote-to-local.timer.j2 dest=/etc/systemd/system/backup-remote-to-local.timer
+  notify: restart backup-remote-to-local timer
+
+- name: create backup-remote-to-local.sh
+  template: 
+    src: backup-remote-to-local.sh.j2 
+    dest: "{{docker_pull_primary_backups_folder}}backup-remote-to-local.sh"
+    mode: 0755
+

roles/backup-remote-to-local/templates/backup-remote-to-local.service.j2

@@ -0,0 +1,7 @@
+[Unit]
+Description=pull remote backups
+OnFailure=systemd-notifier@%n.service cleanup-failed-docker-backups.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/bash {{docker_pull_primary_backups_folder}}backup-remote-to-local.sh

roles/backup-remote-to-local/templates/backup-remote-to-local.sh.j2

@@ -0,0 +1,8 @@
+#!/bin/bash
+# Pulls the remote backups from multiple hosts
+hosts="{{pull_remote_backups}}";
+errors=0
+for host in $hosts; do
+  bash {{docker_pull_primary_backups_folder}}backup-remote-to-local.sh $host || ((errors+=1));
+done;
+exit $errors;

roles/backup-remote-to-local/templates/backup-remote-to-local.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=starts pull remote backup timer
+
+[Timer]
+OnCalendar={{on_calendar_pull_primary_backups}}
+RandomizedDelaySec={{randomized_delay_sec}}
+Persistent=false
+
+[Install]
+WantedBy=timers.target

roles/backup-remote-to-local/vars/main.yml

@@ -0,0 +1 @@
+docker_pull_primary_backups_folder: "{{path_administrator_scripts}}pull-primary-backups/"