Optimized freezer

This commit is contained in:
Kevin Veen-Birkenbach 2023-12-14 16:29:11 +01:00
parent 514cac4a04
commit 04b8565108
10 changed files with 48 additions and 30 deletions

View File

@ -47,27 +47,36 @@ execute_updates: true # Executes updates
force_backup_before_update: true # Activates the backup before the update procedure force_backup_before_update: true # Activates the backup before the update procedure
# System Maintanance Services # System maintenance Services
## Defined Services for Backup Tasks ## Timeouts to wait for other services to stop
system_maintanance_backup_services: system_maintenance_timeout_cleanup_services: "15min"
system_maintenance_timeout_backup_services: "1h"
system_maintenance_timeout_heal_docker: "30min"
system_maintenance_timeout_update_docker: "5min"
## Services
### Defined Services for Backup Tasks
system_maintenance_backup_services:
- "backup-docker-to-local" - "backup-docker-to-local"
- "backup-remote-to-local" - "backup-remote-to-local"
- "backup-data-to-usb" - "backup-data-to-usb"
## Defined Services for System Cleanup ### Defined Services for System Cleanup
system_maintanance_cleanup_services: system_maintenance_cleanup_services:
- "cleanup-backups" - "cleanup-backups"
- "cleanup-disc-space" - "cleanup-disc-space"
- "cleanup-failed-docker-backups" - "cleanup-failed-docker-backups"
## Services that Manipulate the System ### Services that Manipulate the System
system_maintanance_manipulation_services: system_maintenance_manipulation_services:
- "heal-docker" - "heal-docker"
- "update-docker" - "update-docker"
## Total System Maintenance Services ## Total System Maintenance Services
system_maintenance_services: "{{ system_maintanance_backup_services + system_maintanance_cleanup_services + system_maintanance_manipulation_services }}" system_maintenance_services: "{{ system_maintenance_backup_services + system_maintenance_cleanup_services + system_maintenance_manipulation_services }}"
## First default freezer action to apply when freezer service get triggered during play ## First default freezer action to apply when freezer service get triggered during play
system_maintenance_service_freeze_action: 'freeze' # Valid Values: freeze, defrost system_maintenance_service_freeze_action: 'freeze' # Valid Values: freeze, defrost

View File

@ -4,6 +4,6 @@ OnFailure=systemd-notifier@%n.service cleanup-failed-docker-backups.service
[Service] [Service]
Type=oneshot Type=oneshot
ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "backup-docker-to-local,backup-remote-to-local,backup-data-to-usb" --max_attempts 600' ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "{{system_maintenance_backup_services}}" --timeout "{{system_maintenance_timeout_backup_services}}"'
ExecStart=/usr/bin/python {{backup_docker_to_local_folder}}backup-docker-to-local.py ExecStart=/usr/bin/python {{backup_docker_to_local_folder}}backup-docker-to-local.py
ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service' ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service'

View File

@ -4,6 +4,6 @@ OnFailure=systemd-notifier@%n.service cleanup-failed-docker-backups.service
[Service] [Service]
Type=oneshot Type=oneshot
ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "backup-docker-to-local,backup-remote-to-local,backup-data-to-usb" --max_attempts 600' ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "{{system_maintenance_backup_services}}" --timeout "{{system_maintenance_timeout_backup_services}}"'
ExecStart=/usr/bin/bash {{docker_backup_remote_to_local_folder}}backups-remote-to-local.sh ExecStart=/usr/bin/bash {{docker_backup_remote_to_local_folder}}backups-remote-to-local.sh
ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service' ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service'

View File

@ -4,6 +4,6 @@ OnFailure=systemd-notifier@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | reject('equalto', "cleanup-backups") | join(',') }}"' ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "{{system_maintenance_cleanup_services}}" --timeout "{{system_maintenance_timeout_backup_services}}"'
ExecStart=/usr/bin/python {{docker_cleanup_backups}}cleanup-backups.py --backups-folder-path {{backups_folder_path}} --maximum-backup-size-percent {{size_percent_maximum_backup}} ExecStart=/usr/bin/python {{docker_cleanup_backups}}cleanup-backups.py --backups-folder-path {{backups_folder_path}} --maximum-backup-size-percent {{size_percent_maximum_backup}}
ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service' ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service'

View File

@ -4,6 +4,6 @@ OnFailure=systemd-notifier@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | reject('equalto', "cleanup-disc-space") | join(',') }}"' ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "{{system_maintenance_cleanup_services}}" --timeout "{{system_maintenance_timeout_backup_services}}"'
ExecStart=/bin/bash {{cleanup_disc_space_folder}}cleanup-disc-space.sh {{size_percent_cleanup_disc_space}} ExecStart=/bin/bash {{cleanup_disc_space_folder}}cleanup-disc-space.sh {{size_percent_cleanup_disc_space}}
ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service' ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service'

View File

@ -4,6 +4,6 @@ OnFailure=systemd-notifier@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | reject('equalto', "cleanup-failed-docker-backups") | join(',') }}"' ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "{{system_maintenance_cleanup_services}}" --timeout "{{system_maintenance_timeout_backup_services}}"'
ExecStart=/bin/sh -c '/usr/bin/yes | /usr/bin/bash {{backup_docker_to_local_cleanup_folder}}cleanup.sh {{backup_docker_to_local_cleanup_machine_id}} {{backup_docker_to_local_cleanup_trigger_directory}}' ExecStart=/bin/sh -c '/usr/bin/yes | /usr/bin/bash {{backup_docker_to_local_cleanup_folder}}cleanup.sh {{backup_docker_to_local_cleanup_machine_id}} {{backup_docker_to_local_cleanup_trigger_directory}}'
ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service' ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service'

View File

@ -1,6 +1,6 @@
# role docker # role docker
## maintanance ## maintenance
### list unused volumes ### list unused volumes
```bash ```bash

View File

@ -4,6 +4,6 @@ OnFailure=systemd-notifier@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | reject('equalto', "heal-docker") | join(',') }}"' ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "{{system_maintenance_cleanup_services}}" --timeout "{{system_maintenance_heal_docker}}"'
ExecStart=/bin/python {{heal_docker}}heal-docker.py ExecStart=/bin/python {{heal_docker}}heal-docker.py
ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service' ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service'

View File

@ -3,6 +3,14 @@ import subprocess
import time import time
import os import os
def parse_time_to_seconds(time_str):
"""Convert time string to seconds."""
units = {"s": 1, "min": 60, "h": 3600}
number, unit = time_str[:-1], time_str[-1]
if unit not in units:
raise ValueError("Invalid time unit")
return int(number) * units[unit]
def service_file_exists(service_name, service_type="service"): def service_file_exists(service_name, service_type="service"):
"""Check if a systemd service file exists.""" """Check if a systemd service file exists."""
# Paths where service files can be stored # Paths where service files can be stored
@ -22,18 +30,20 @@ def check_service_active(service_name):
service_status = result.stdout.decode('utf-8').strip() service_status = result.stdout.decode('utf-8').strip()
return service_status in ['active', 'activating'] return service_status in ['active', 'activating']
def freeze(services_to_wait_for, ignored_services, max_attempts): def freeze(services_to_wait_for, ignored_services, timeout_sec):
break_time_sec=5
attempt=0
max_attempts=timeout_sec/break_time_sec
# Filter services that exist and are not in the ignored list # Filter services that exist and are not in the ignored list
for service in services_to_wait_for: for service in services_to_wait_for:
print(f"\nFreezing: {service}") print(f"\nFreezing: {service}")
if service in ignored_services: if service in ignored_services:
print(f"{service} will be ignored.") print(f"{service} will be ignored.")
else: else:
attempt=0
break_time_sec=5
while check_service_active(service): while check_service_active(service):
current_time_iso = datetime.now().isoformat()
attempt += 1 attempt += 1
print(f"({attempt}/{max_attempts}) Waiting for {break_time_sec} seconds for {service} to stop...") print(f"{current_time_iso}#{attempt}/{max_attempts}: Waiting for {break_time_sec} seconds for {service} to stop...")
time.sleep(break_time_sec) time.sleep(break_time_sec)
if attempt > max_attempts: if attempt > max_attempts:
raise Exception(f"Error: Maximum attempts ({max_attempts}) reached. Exit.") raise Exception(f"Error: Maximum attempts ({max_attempts}) reached. Exit.")
@ -46,7 +56,6 @@ def freeze(services_to_wait_for, ignored_services, max_attempts):
print(f"{timer_name} stopped and disabled.") print(f"{timer_name} stopped and disabled.")
else: else:
print(f"Skipped.") print(f"Skipped.")
print("\nAll required services have stopped.") print("\nAll required services have stopped.")
def defrost(services_to_wait_for, ignored_services): def defrost(services_to_wait_for, ignored_services):
@ -64,12 +73,12 @@ def defrost(services_to_wait_for, ignored_services):
print(f"Skipped.") print(f"Skipped.")
print("\nAll required services are started.") print("\nAll required services are started.")
def main(services_to_wait_for, ignored_services, action, max_attempts): def main(services_to_wait_for, ignored_services, action, timeout_sec):
print(f"Services to wait for: {services_to_wait_for}") print(f"Services to wait for: {services_to_wait_for}")
print(f"Services to ignore: {ignored_services}") print(f"Services to ignore: {ignored_services}")
if action == 'freeze': if action == 'freeze':
print("Freezing services."); print("Freezing services.");
freeze(services_to_wait_for, ignored_services, max_attempts) freeze(services_to_wait_for, ignored_services, timeout_sec)
elif action == 'defrost': elif action == 'defrost':
print("Unfreezing services."); print("Unfreezing services.");
defrost(services_to_wait_for, ignored_services) defrost(services_to_wait_for, ignored_services)
@ -79,12 +88,12 @@ def main(services_to_wait_for, ignored_services, action, max_attempts):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description='freezes and defrost systemctl services and timers') parser = argparse.ArgumentParser(description='freezes and defrost systemctl services and timers')
parser.add_argument('action', choices=['freeze', 'defrost'], help='Action to perform: freeze or defrost services.') parser.add_argument('action', choices=['freeze', 'defrost'], help='Action to perform: freeze or defrost services.')
parser.add_argument('services', help='Comma-separated list of services to apply the action to') parser.add_argument('services', nargs='+', help='List of services to apply the action to')
parser.add_argument('--ignore', help='Comma-separated list of services to ignore in the action', default='') parser.add_argument('--ignore', nargs='*', help='List of services to ignore in the action', default=[])
parser.add_argument('--max_attempts', type=int, default=60, help='Maximum number of attempts for freezing services') parser.add_argument('--timeout', help='Timeout for freezing services (e.g., 1h, 30min, 45s)', default='1h')
args = parser.parse_args() args = parser.parse_args()
services_to_wait_for = args.services.split(',') services_to_wait_for = args.services
ignored_services = args.ignore.split(',') if args.ignore else [] ignored_services = args.ignore if args.ignore else []
max_attempts = args.max_attempts timeout_seconds = parse_time_to_seconds(args.timeout)
main(services_to_wait_for, ignored_services,args.action,max_attempts) main(services_to_wait_for, ignored_services, args.action, timeout_seconds)

View File

@ -5,6 +5,6 @@ OnFailure=systemd-notifier@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
{% if force_backup_before_update | bool %}ExecStartPre=/bin/sh -c 'systemctl start backup-docker-to-local.service'{% endif %} {% if force_backup_before_update | bool %}ExecStartPre=/bin/sh -c 'systemctl start backup-docker-to-local.service'{% endif %}
ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | reject('equalto', "update-docker") | join(',') }}"' ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "{{system_maintenance_cleanup_services + ['update-docker'] }}" --timeout "{{system_maintenance_heal_docker}}"'
ExecStart=/bin/sh -c '/usr/bin/python {{update_docker_script}} {{path_docker_compose_instances}}' ExecStart=/bin/sh -c '/usr/bin/python {{update_docker_script}} {{path_docker_compose_instances}}'
ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service' ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service'