From 21e91e5249fd86daff251ff50b0e7fff8cc6f43c Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 14 Dec 2023 16:29:11 +0100 Subject: [PATCH] Optimized freezer --- group_vars/all | 25 +++++++++---- .../backup-docker-to-local.service.j2 | 2 +- .../backup-remote-to-local.service.j2 | 2 +- .../templates/cleanup-backups.service.j2 | 2 +- .../templates/cleanup-disc-space.service.j2 | 2 +- .../cleanup-failed-docker-backups.service.j2 | 2 +- roles/docker/README.md | 2 +- .../templates/heal-docker.service.j2 | 2 +- .../system-maintenance-service-freezer.py | 37 ++++++++++++------- .../templates/update-docker.service.j2 | 2 +- 10 files changed, 48 insertions(+), 30 deletions(-) diff --git a/group_vars/all b/group_vars/all index 888c3d43..765b525f 100644 --- a/group_vars/all +++ b/group_vars/all @@ -47,27 +47,36 @@ execute_updates: true # Executes updates force_backup_before_update: true # Activates the backup before the update procedure -# System Maintanance Services +# System maintenance Services -## Defined Services for Backup Tasks -system_maintanance_backup_services: +## Timeouts to wait for other services to stop +system_maintenance_timeout_cleanup_services: "15min" +system_maintenance_timeout_backup_services: "1h" +system_maintenance_timeout_heal_docker: "30min" +system_maintenance_timeout_update_docker: "5min" + + +## Services + +### Defined Services for Backup Tasks +system_maintenance_backup_services: - "backup-docker-to-local" - "backup-remote-to-local" - "backup-data-to-usb" -## Defined Services for System Cleanup -system_maintanance_cleanup_services: +### Defined Services for System Cleanup +system_maintenance_cleanup_services: - "cleanup-backups" - "cleanup-disc-space" - "cleanup-failed-docker-backups" -## Services that Manipulate the System -system_maintanance_manipulation_services: +### Services that Manipulate the System +system_maintenance_manipulation_services: - "heal-docker" - "update-docker" ## Total System Maintenance Services -system_maintenance_services: "{{ system_maintanance_backup_services + system_maintanance_cleanup_services + system_maintanance_manipulation_services }}" +system_maintenance_services: "{{ system_maintenance_backup_services + system_maintenance_cleanup_services + system_maintenance_manipulation_services }}" ## First default freezer action to apply when freezer service get triggered during play system_maintenance_service_freeze_action: 'freeze' # Valid Values: freeze, defrost diff --git a/roles/backup-docker-to-local/templates/backup-docker-to-local.service.j2 b/roles/backup-docker-to-local/templates/backup-docker-to-local.service.j2 index 0919ee19..4246b917 100644 --- a/roles/backup-docker-to-local/templates/backup-docker-to-local.service.j2 +++ b/roles/backup-docker-to-local/templates/backup-docker-to-local.service.j2 @@ -4,6 +4,6 @@ OnFailure=systemd-notifier@%n.service cleanup-failed-docker-backups.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "backup-docker-to-local,backup-remote-to-local,backup-data-to-usb" --max_attempts 600' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "{{system_maintenance_backup_services}}" --timeout "{{system_maintenance_timeout_backup_services}}"' ExecStart=/usr/bin/python {{backup_docker_to_local_folder}}backup-docker-to-local.py ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service' \ No newline at end of file diff --git a/roles/backup-remote-to-local/templates/backup-remote-to-local.service.j2 b/roles/backup-remote-to-local/templates/backup-remote-to-local.service.j2 index 303d078e..8de8fe35 100644 --- a/roles/backup-remote-to-local/templates/backup-remote-to-local.service.j2 +++ b/roles/backup-remote-to-local/templates/backup-remote-to-local.service.j2 @@ -4,6 +4,6 @@ OnFailure=systemd-notifier@%n.service cleanup-failed-docker-backups.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "backup-docker-to-local,backup-remote-to-local,backup-data-to-usb" --max_attempts 600' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "{{system_maintenance_backup_services}}" --timeout "{{system_maintenance_timeout_backup_services}}"' ExecStart=/usr/bin/bash {{docker_backup_remote_to_local_folder}}backups-remote-to-local.sh ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service' diff --git a/roles/cleanup-backups-service/templates/cleanup-backups.service.j2 b/roles/cleanup-backups-service/templates/cleanup-backups.service.j2 index c9d979b9..072f9dec 100644 --- a/roles/cleanup-backups-service/templates/cleanup-backups.service.j2 +++ b/roles/cleanup-backups-service/templates/cleanup-backups.service.j2 @@ -4,6 +4,6 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | reject('equalto', "cleanup-backups") | join(',') }}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "{{system_maintenance_cleanup_services}}" --timeout "{{system_maintenance_timeout_backup_services}}"' ExecStart=/usr/bin/python {{docker_cleanup_backups}}cleanup-backups.py --backups-folder-path {{backups_folder_path}} --maximum-backup-size-percent {{size_percent_maximum_backup}} ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service' \ No newline at end of file diff --git a/roles/cleanup-disc-space/templates/cleanup-disc-space.service.j2 b/roles/cleanup-disc-space/templates/cleanup-disc-space.service.j2 index 90536b3b..3a6f37bf 100644 --- a/roles/cleanup-disc-space/templates/cleanup-disc-space.service.j2 +++ b/roles/cleanup-disc-space/templates/cleanup-disc-space.service.j2 @@ -4,6 +4,6 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | reject('equalto', "cleanup-disc-space") | join(',') }}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "{{system_maintenance_cleanup_services}}" --timeout "{{system_maintenance_timeout_backup_services}}"' ExecStart=/bin/bash {{cleanup_disc_space_folder}}cleanup-disc-space.sh {{size_percent_cleanup_disc_space}} ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service' \ No newline at end of file diff --git a/roles/cleanup-failed-docker-backups/templates/cleanup-failed-docker-backups.service.j2 b/roles/cleanup-failed-docker-backups/templates/cleanup-failed-docker-backups.service.j2 index 0c3f68de..a62b6e9e 100644 --- a/roles/cleanup-failed-docker-backups/templates/cleanup-failed-docker-backups.service.j2 +++ b/roles/cleanup-failed-docker-backups/templates/cleanup-failed-docker-backups.service.j2 @@ -4,6 +4,6 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | reject('equalto', "cleanup-failed-docker-backups") | join(',') }}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "{{system_maintenance_cleanup_services}}" --timeout "{{system_maintenance_timeout_backup_services}}"' ExecStart=/bin/sh -c '/usr/bin/yes | /usr/bin/bash {{backup_docker_to_local_cleanup_folder}}cleanup.sh {{backup_docker_to_local_cleanup_machine_id}} {{backup_docker_to_local_cleanup_trigger_directory}}' ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service' \ No newline at end of file diff --git a/roles/docker/README.md b/roles/docker/README.md index ccc94bc0..68c8a580 100644 --- a/roles/docker/README.md +++ b/roles/docker/README.md @@ -1,6 +1,6 @@ # role docker -## maintanance +## maintenance ### list unused volumes ```bash diff --git a/roles/heal-docker/templates/heal-docker.service.j2 b/roles/heal-docker/templates/heal-docker.service.j2 index 21b0e5dc..4fc17d67 100644 --- a/roles/heal-docker/templates/heal-docker.service.j2 +++ b/roles/heal-docker/templates/heal-docker.service.j2 @@ -4,6 +4,6 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | reject('equalto', "heal-docker") | join(',') }}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "{{system_maintenance_cleanup_services}}" --timeout "{{system_maintenance_heal_docker}}"' ExecStart=/bin/python {{heal_docker}}heal-docker.py ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service' \ No newline at end of file diff --git a/roles/system-maintenance-service-freezer/files/system-maintenance-service-freezer.py b/roles/system-maintenance-service-freezer/files/system-maintenance-service-freezer.py index a5edd633..67de69c7 100644 --- a/roles/system-maintenance-service-freezer/files/system-maintenance-service-freezer.py +++ b/roles/system-maintenance-service-freezer/files/system-maintenance-service-freezer.py @@ -3,6 +3,14 @@ import subprocess import time import os +def parse_time_to_seconds(time_str): + """Convert time string to seconds.""" + units = {"s": 1, "min": 60, "h": 3600} + number, unit = time_str[:-1], time_str[-1] + if unit not in units: + raise ValueError("Invalid time unit") + return int(number) * units[unit] + def service_file_exists(service_name, service_type="service"): """Check if a systemd service file exists.""" # Paths where service files can be stored @@ -22,18 +30,20 @@ def check_service_active(service_name): service_status = result.stdout.decode('utf-8').strip() return service_status in ['active', 'activating'] -def freeze(services_to_wait_for, ignored_services, max_attempts): +def freeze(services_to_wait_for, ignored_services, timeout_sec): + break_time_sec=5 + attempt=0 + max_attempts=timeout_sec/break_time_sec # Filter services that exist and are not in the ignored list for service in services_to_wait_for: print(f"\nFreezing: {service}") if service in ignored_services: print(f"{service} will be ignored.") else: - attempt=0 - break_time_sec=5 while check_service_active(service): + current_time_iso = datetime.now().isoformat() attempt += 1 - print(f"({attempt}/{max_attempts}) Waiting for {break_time_sec} seconds for {service} to stop...") + print(f"{current_time_iso}#{attempt}/{max_attempts}: Waiting for {break_time_sec} seconds for {service} to stop...") time.sleep(break_time_sec) if attempt > max_attempts: raise Exception(f"Error: Maximum attempts ({max_attempts}) reached. Exit.") @@ -46,7 +56,6 @@ def freeze(services_to_wait_for, ignored_services, max_attempts): print(f"{timer_name} stopped and disabled.") else: print(f"Skipped.") - print("\nAll required services have stopped.") def defrost(services_to_wait_for, ignored_services): @@ -64,12 +73,12 @@ def defrost(services_to_wait_for, ignored_services): print(f"Skipped.") print("\nAll required services are started.") -def main(services_to_wait_for, ignored_services, action, max_attempts): +def main(services_to_wait_for, ignored_services, action, timeout_sec): print(f"Services to wait for: {services_to_wait_for}") print(f"Services to ignore: {ignored_services}") if action == 'freeze': print("Freezing services."); - freeze(services_to_wait_for, ignored_services, max_attempts) + freeze(services_to_wait_for, ignored_services, timeout_sec) elif action == 'defrost': print("Unfreezing services."); defrost(services_to_wait_for, ignored_services) @@ -79,12 +88,12 @@ def main(services_to_wait_for, ignored_services, action, max_attempts): if __name__ == "__main__": parser = argparse.ArgumentParser(description='freezes and defrost systemctl services and timers') parser.add_argument('action', choices=['freeze', 'defrost'], help='Action to perform: freeze or defrost services.') - parser.add_argument('services', help='Comma-separated list of services to apply the action to') - parser.add_argument('--ignore', help='Comma-separated list of services to ignore in the action', default='') - parser.add_argument('--max_attempts', type=int, default=60, help='Maximum number of attempts for freezing services') + parser.add_argument('services', nargs='+', help='List of services to apply the action to') + parser.add_argument('--ignore', nargs='*', help='List of services to ignore in the action', default=[]) + parser.add_argument('--timeout', help='Timeout for freezing services (e.g., 1h, 30min, 45s)', default='1h') args = parser.parse_args() - services_to_wait_for = args.services.split(',') - ignored_services = args.ignore.split(',') if args.ignore else [] - max_attempts = args.max_attempts - main(services_to_wait_for, ignored_services,args.action,max_attempts) + services_to_wait_for = args.services + ignored_services = args.ignore if args.ignore else [] + timeout_seconds = parse_time_to_seconds(args.timeout) + main(services_to_wait_for, ignored_services, args.action, timeout_seconds) diff --git a/roles/update-docker/templates/update-docker.service.j2 b/roles/update-docker/templates/update-docker.service.j2 index dd189b0d..21d8eba1 100644 --- a/roles/update-docker/templates/update-docker.service.j2 +++ b/roles/update-docker/templates/update-docker.service.j2 @@ -5,6 +5,6 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot {% if force_backup_before_update | bool %}ExecStartPre=/bin/sh -c 'systemctl start backup-docker-to-local.service'{% endif %} -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | reject('equalto', "update-docker") | join(',') }}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services }}" --ignore "{{system_maintenance_cleanup_services + ['update-docker'] }}" --timeout "{{system_maintenance_heal_docker}}"' ExecStart=/bin/sh -c '/usr/bin/python {{update_docker_script}} {{path_docker_compose_instances}}' ExecStartPost=/bin/sh -c 'systemctl start system-maintenance-service-defrost.service' \ No newline at end of file