Implemented deletion of not fully pulled backups

This commit is contained in:
Kevin Veen-Birkenbach 2023-04-25 21:39:44 +02:00
parent 04671e283b
commit 36e41b8c99
22 changed files with 56 additions and 29 deletions

View File

@ -35,7 +35,7 @@ This software shipts the following tools which are natively setup on the server:
- [Docker Health Check](./roles/server_native-docker-health-check/) - Checks the health of docker containers
- [Docker Reverse Proxy](./roles/server_native-docker-reverse-proxy/README.md) - Docker Reverse Proxy Solution
- [Docker Volume Backup](./roles/server_native-docker-volume-backup/) - Backup Solution for Docker Volumes
- [Pull Primary Backups](./roles/server_native-pull-primary-backups/README.md) - Pulls the backups from another server and stores them
- [Pull Primary Backups](./roles/server_native-backups-consumer/README.md) - Pulls the backups from another server and stores them
- [Wireguard](./roles/server_native-wireguard/README.md) - Integrates the server in an wireguard vpn
### Server Administration

View File

@ -184,7 +184,7 @@
hosts: replica_backup
become: true
roles:
- role: server_native-pull-primary-backups
- role: server_native-backups-consumer
## PC services
- name: general host setup

View File

@ -2,6 +2,7 @@ import psutil
import shutil
import os
import argparse
import sys
# Validating arguments
parser = argparse.ArgumentParser()
@ -9,10 +10,11 @@ parser.add_argument('--maximum-backup-size-percent', type=int, dest='maximum_bac
parser.add_argument('--backups-folder-path',type=str,dest='backups_folder_path',required=True, help="The folder in which the backups are stored")
args = parser.parse_args()
deleted = True
while psutil.disk_usage(args.backups_folder_path).percent > args.maximum_backup_size_percent and deleted:
deleted = False
print("%d %% of disk %s are used. Freeing space..." % (psutil.disk_usage(args.backups_folder_path).percent,args.backups_folder_path))
def print_used_disc_space():
print("%d %% of disk %s are used" % (psutil.disk_usage(args.backups_folder_path).percent,args.backups_folder_path))
warning_counter=0
for host_backup_directory_name in os.listdir(args.backups_folder_path):
host_backup_directory_path = os.path.join(args.backups_folder_path, host_backup_directory_name)
for application_directory in os.listdir(host_backup_directory_path):
@ -22,11 +24,27 @@ while psutil.disk_usage(args.backups_folder_path).percent > args.maximum_backup_
versions = os.listdir(versions_directory)
versions.sort(reverse=False)
if len(versions) >= 1:
delete_diff = versions_directory + versions[0]
print("Deleting %s..." % (delete_diff))
shutil.rmtree(delete_diff)
deleted = True
if not deleted:
print("All versions had been deleted!")
print("Cleaning up finished: %d %% of disk %s are used." % (psutil.disk_usage(args.backups_folder_path).percent,args.backups_folder_path))
versions_counter=len(versions)
print_used_disc_space()
for version in versions:
version_path=os.path.join(versions_directory, version)
version_status_pulling_path=os.path.join(versions_directory, version, ".pulling")
print("Checking directory %s ..." % (version_path))
if psutil.disk_usage(args.backups_folder_path).percent > args.maximum_backup_size_percent:
if versions_counter >= 1:
print("Deleting %s to free space" % (version_path))
shutil.rmtree(version_path)
versions_counter-=1
else:
print("Deletion not possible. There needs to be at least one backup version")
warning_counter+=1
elif os.path.exists(version_status_pulling_path):
last_version=versions[-1]
if last_version != version:
print("Deleting %s due to unfinished pull" % (version_path))
shutil.rmtree(version_path)
versions_counter-=1
print_used_disc_space()
print("Cleaning up finished.")
sys.exit(warning_counter)

View File

@ -1,4 +1,4 @@
# role server_native-pull-primary-backups
# role server_native-backups-consumer
## goal
This script allows to pull backups from a remote server.

View File

@ -45,10 +45,19 @@ for backup_type in $remote_backup_types; do
echo "creating local backup destination folder..." &&
mkdir -vp "$local_backup_destination_path" &&
status_pulling_file="$local_backup_destination_path/.pulling" &&
echo "creating: $status_pulling_file" &&
echo "pulling backup since $(date)" > $status_pulling_file &&
echo "starting backup..." &&
rsync_command='rsync -abP --delete --delete-excluded --rsync-path="sudo rsync" --link-dest="'$local_previous_version_dir'" "'$remote_source_path'" "'$local_backup_destination_path'"' &&
echo "executing: $rsync_command" &&
eval "$rsync_command" || ((errors+=1));
eval "$rsync_command" &&
echo "removing: $status_pulling_file" &&
rm -vf $status_pulling_file
|| ((errors+=1));
fi
done
exit $errors;

View File

@ -1,4 +1,4 @@
# role server_native-user-backup
# role server_native-backups-provider-user
User for backups
## todo

View File

@ -1,4 +1,4 @@
# role server_native-primary-backup-host
# role server_native-backups-provider-host
## todo
- add full system backup

View File

@ -1,3 +1,3 @@
dependencies:
- server_native-user-backup
- server_native-backups-provider-user
- server_native-backups-cleanup

View File

@ -1,4 +1,4 @@
dependencies:
- server_native-git
- server_native-primary-backup
- server_native-backups-provider
- server_native-systemd-email