Implemented deletion of not fully pulled backups

This commit is contained in:
Kevin Veen-Birkenbach 2023-04-25 21:39:44 +02:00
parent 04671e283b
commit 36e41b8c99
22 changed files with 56 additions and 29 deletions

View File

@ -35,7 +35,7 @@ This software shipts the following tools which are natively setup on the server:
- [Docker Health Check](./roles/server_native-docker-health-check/) - Checks the health of docker containers - [Docker Health Check](./roles/server_native-docker-health-check/) - Checks the health of docker containers
- [Docker Reverse Proxy](./roles/server_native-docker-reverse-proxy/README.md) - Docker Reverse Proxy Solution - [Docker Reverse Proxy](./roles/server_native-docker-reverse-proxy/README.md) - Docker Reverse Proxy Solution
- [Docker Volume Backup](./roles/server_native-docker-volume-backup/) - Backup Solution for Docker Volumes - [Docker Volume Backup](./roles/server_native-docker-volume-backup/) - Backup Solution for Docker Volumes
- [Pull Primary Backups](./roles/server_native-pull-primary-backups/README.md) - Pulls the backups from another server and stores them - [Pull Primary Backups](./roles/server_native-backups-consumer/README.md) - Pulls the backups from another server and stores them
- [Wireguard](./roles/server_native-wireguard/README.md) - Integrates the server in an wireguard vpn - [Wireguard](./roles/server_native-wireguard/README.md) - Integrates the server in an wireguard vpn
### Server Administration ### Server Administration

View File

@ -184,7 +184,7 @@
hosts: replica_backup hosts: replica_backup
become: true become: true
roles: roles:
- role: server_native-pull-primary-backups - role: server_native-backups-consumer
## PC services ## PC services
- name: general host setup - name: general host setup

View File

@ -2,6 +2,7 @@ import psutil
import shutil import shutil
import os import os
import argparse import argparse
import sys
# Validating arguments # Validating arguments
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
@ -9,10 +10,11 @@ parser.add_argument('--maximum-backup-size-percent', type=int, dest='maximum_bac
parser.add_argument('--backups-folder-path',type=str,dest='backups_folder_path',required=True, help="The folder in which the backups are stored") parser.add_argument('--backups-folder-path',type=str,dest='backups_folder_path',required=True, help="The folder in which the backups are stored")
args = parser.parse_args() args = parser.parse_args()
deleted = True def print_used_disc_space():
while psutil.disk_usage(args.backups_folder_path).percent > args.maximum_backup_size_percent and deleted: print("%d %% of disk %s are used" % (psutil.disk_usage(args.backups_folder_path).percent,args.backups_folder_path))
deleted = False
print("%d %% of disk %s are used. Freeing space..." % (psutil.disk_usage(args.backups_folder_path).percent,args.backups_folder_path)) warning_counter=0
for host_backup_directory_name in os.listdir(args.backups_folder_path): for host_backup_directory_name in os.listdir(args.backups_folder_path):
host_backup_directory_path = os.path.join(args.backups_folder_path, host_backup_directory_name) host_backup_directory_path = os.path.join(args.backups_folder_path, host_backup_directory_name)
for application_directory in os.listdir(host_backup_directory_path): for application_directory in os.listdir(host_backup_directory_path):
@ -22,11 +24,27 @@ while psutil.disk_usage(args.backups_folder_path).percent > args.maximum_backup_
versions = os.listdir(versions_directory) versions = os.listdir(versions_directory)
versions.sort(reverse=False) versions.sort(reverse=False)
if len(versions) >= 1: versions_counter=len(versions)
delete_diff = versions_directory + versions[0]
print("Deleting %s..." % (delete_diff)) print_used_disc_space()
shutil.rmtree(delete_diff) for version in versions:
deleted = True version_path=os.path.join(versions_directory, version)
if not deleted: version_status_pulling_path=os.path.join(versions_directory, version, ".pulling")
print("All versions had been deleted!") print("Checking directory %s ..." % (version_path))
print("Cleaning up finished: %d %% of disk %s are used." % (psutil.disk_usage(args.backups_folder_path).percent,args.backups_folder_path)) if psutil.disk_usage(args.backups_folder_path).percent > args.maximum_backup_size_percent:
if versions_counter >= 1:
print("Deleting %s to free space" % (version_path))
shutil.rmtree(version_path)
versions_counter-=1
else:
print("Deletion not possible. There needs to be at least one backup version")
warning_counter+=1
elif os.path.exists(version_status_pulling_path):
last_version=versions[-1]
if last_version != version:
print("Deleting %s due to unfinished pull" % (version_path))
shutil.rmtree(version_path)
versions_counter-=1
print_used_disc_space()
print("Cleaning up finished.")
sys.exit(warning_counter)

View File

@ -1,4 +1,4 @@
# role server_native-pull-primary-backups # role server_native-backups-consumer
## goal ## goal
This script allows to pull backups from a remote server. This script allows to pull backups from a remote server.

View File

@ -45,10 +45,19 @@ for backup_type in $remote_backup_types; do
echo "creating local backup destination folder..." && echo "creating local backup destination folder..." &&
mkdir -vp "$local_backup_destination_path" && mkdir -vp "$local_backup_destination_path" &&
status_pulling_file="$local_backup_destination_path/.pulling" &&
echo "creating: $status_pulling_file" &&
echo "pulling backup since $(date)" > $status_pulling_file &&
echo "starting backup..." && echo "starting backup..." &&
rsync_command='rsync -abP --delete --delete-excluded --rsync-path="sudo rsync" --link-dest="'$local_previous_version_dir'" "'$remote_source_path'" "'$local_backup_destination_path'"' && rsync_command='rsync -abP --delete --delete-excluded --rsync-path="sudo rsync" --link-dest="'$local_previous_version_dir'" "'$remote_source_path'" "'$local_backup_destination_path'"' &&
echo "executing: $rsync_command" && echo "executing: $rsync_command" &&
eval "$rsync_command" || ((errors+=1)); eval "$rsync_command" &&
echo "removing: $status_pulling_file" &&
rm -vf $status_pulling_file
|| ((errors+=1));
fi fi
done done
exit $errors; exit $errors;

View File

@ -1,4 +1,4 @@
# role server_native-user-backup # role server_native-backups-provider-user
User for backups User for backups
## todo ## todo

View File

@ -1,4 +1,4 @@
# role server_native-primary-backup-host # role server_native-backups-provider-host
## todo ## todo
- add full system backup - add full system backup

View File

@ -1,3 +1,3 @@
dependencies: dependencies:
- server_native-user-backup - server_native-backups-provider-user
- server_native-backups-cleanup - server_native-backups-cleanup

View File

@ -1,4 +1,4 @@
dependencies: dependencies:
- server_native-git - server_native-git
- server_native-primary-backup - server_native-backups-provider
- server_native-systemd-email - server_native-systemd-email