From 0b15b9c581916308310c4420f201a1da1558f9a0 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 14 Dec 2023 21:12:25 +0100 Subject: [PATCH] Solved more freezer bugs --- .../backup-docker-to-local.service.j2 | 2 +- .../backup-remote-to-local.service.j2 | 2 +- .../templates/cleanup-backups.service.j2 | 2 +- .../templates/cleanup-disc-space.service.j2 | 2 +- .../cleanup-failed-docker-backups.service.j2 | 2 +- .../templates/heal-docker.service.j2 | 2 +- .../system-maintenance-service-freezer.py | 10 +++++-- .../tasks/main.yml | 28 +++++++++++++------ ...tem-maintenance-service-freezer.service.j2 | 4 +-- .../templates/update-docker.service.j2 | 2 +- 10 files changed, 36 insertions(+), 20 deletions(-) diff --git a/roles/backup-docker-to-local/templates/backup-docker-to-local.service.j2 b/roles/backup-docker-to-local/templates/backup-docker-to-local.service.j2 index b4dc55c2..115693fc 100644 --- a/roles/backup-docker-to-local/templates/backup-docker-to-local.service.j2 +++ b/roles/backup-docker-to-local/templates/backup-docker-to-local.service.j2 @@ -4,5 +4,5 @@ OnFailure=systemd-notifier@%n.service cleanup-failed-docker-backups.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | join(' ') }}" --ignore "{{system_maintenance_backup_services| join(' ') }}" --timeout "{{system_maintenance_timeout_backup_services}}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_backup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"' ExecStart=/bin/sh -c '/usr/bin/python {{backup_docker_to_local_folder}}backup-docker-to-local.py && systemctl start system-maintenance-service-defrost.service' \ No newline at end of file diff --git a/roles/backup-remote-to-local/templates/backup-remote-to-local.service.j2 b/roles/backup-remote-to-local/templates/backup-remote-to-local.service.j2 index f7400e00..260308b7 100644 --- a/roles/backup-remote-to-local/templates/backup-remote-to-local.service.j2 +++ b/roles/backup-remote-to-local/templates/backup-remote-to-local.service.j2 @@ -4,5 +4,5 @@ OnFailure=systemd-notifier@%n.service cleanup-failed-docker-backups.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | join(' ') }}" --ignore "{{system_maintenance_backup_services| join(' ') }}" --timeout "{{system_maintenance_timeout_backup_services}}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_backup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"' ExecStart=/bin/sh -c '/usr/bin/bash {{docker_backup_remote_to_local_folder}}backups-remote-to-local.sh && systemctl start system-maintenance-service-defrost.service' diff --git a/roles/cleanup-backups-service/templates/cleanup-backups.service.j2 b/roles/cleanup-backups-service/templates/cleanup-backups.service.j2 index 5d29e634..30a3dc07 100644 --- a/roles/cleanup-backups-service/templates/cleanup-backups.service.j2 +++ b/roles/cleanup-backups-service/templates/cleanup-backups.service.j2 @@ -4,5 +4,5 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | join(' ') }}" --ignore "{{system_maintenance_cleanup_services| join(' ') }}" --timeout "{{system_maintenance_timeout_backup_services}}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"' ExecStart=/bin/sh -c '/usr/bin/python {{docker_cleanup_backups}}cleanup-backups.py --backups-folder-path {{backups_folder_path}} --maximum-backup-size-percent {{size_percent_maximum_backup}} && systemctl start system-maintenance-service-defrost.service' \ No newline at end of file diff --git a/roles/cleanup-disc-space/templates/cleanup-disc-space.service.j2 b/roles/cleanup-disc-space/templates/cleanup-disc-space.service.j2 index 4445bbb3..e243e92f 100644 --- a/roles/cleanup-disc-space/templates/cleanup-disc-space.service.j2 +++ b/roles/cleanup-disc-space/templates/cleanup-disc-space.service.j2 @@ -4,5 +4,5 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | join(' ') }}" --ignore "{{system_maintenance_cleanup_services| join(' ') }}" --timeout "{{system_maintenance_timeout_backup_services}}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"' ExecStart=/bin/sh -c '/bin/bash {{cleanup_disc_space_folder}}cleanup-disc-space.sh {{size_percent_cleanup_disc_space}} && systemctl start system-maintenance-service-defrost.service' \ No newline at end of file diff --git a/roles/cleanup-failed-docker-backups/templates/cleanup-failed-docker-backups.service.j2 b/roles/cleanup-failed-docker-backups/templates/cleanup-failed-docker-backups.service.j2 index 8c7245d8..40fa655a 100644 --- a/roles/cleanup-failed-docker-backups/templates/cleanup-failed-docker-backups.service.j2 +++ b/roles/cleanup-failed-docker-backups/templates/cleanup-failed-docker-backups.service.j2 @@ -4,5 +4,5 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | join(' ') }}" --ignore "{{system_maintenance_cleanup_services| join(' ') }}" --timeout "{{system_maintenance_timeout_backup_services}}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"' ExecStart=/bin/sh -c '/usr/bin/yes | /usr/bin/bash {{backup_docker_to_local_cleanup_folder}}cleanup.sh {{backup_docker_to_local_cleanup_machine_id}} {{backup_docker_to_local_cleanup_trigger_directory}} && systemctl start system-maintenance-service-defrost.service' \ No newline at end of file diff --git a/roles/heal-docker/templates/heal-docker.service.j2 b/roles/heal-docker/templates/heal-docker.service.j2 index 447ccf90..9be3f249 100644 --- a/roles/heal-docker/templates/heal-docker.service.j2 +++ b/roles/heal-docker/templates/heal-docker.service.j2 @@ -4,5 +4,5 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | join(' ') }}" --ignore "{{system_maintenance_cleanup_services| join(' ') }}" --timeout "{{system_maintenance_timeout_heal_docker}}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{system_maintenance_timeout_heal_docker}}"' ExecStart=/bin/sh -c '/bin/python {{heal_docker}}heal-docker.py && systemctl start system-maintenance-service-defrost.service' \ No newline at end of file diff --git a/roles/system-maintenance-service-freezer/files/system-maintenance-service-freezer.py b/roles/system-maintenance-service-freezer/files/system-maintenance-service-freezer.py index 746e8cc3..7bca9290 100644 --- a/roles/system-maintenance-service-freezer/files/system-maintenance-service-freezer.py +++ b/roles/system-maintenance-service-freezer/files/system-maintenance-service-freezer.py @@ -51,9 +51,7 @@ def stop_timer(service): """ Stop and disable a systemd timer for a service if it exists. """ - if service == "system-maintenance-service-defrost": - print(f"Ignore: {service}") - elif service_file_exists(service, "timer"): + if service_file_exists(service, "timer"): timer_name = f"{service}.timer" subprocess.run(['systemctl', 'stop', timer_name]) subprocess.run(['systemctl', 'disable', timer_name]) @@ -119,6 +117,12 @@ def main(services, ignored_services, action, timeout_sec): """ Main function to process the command-line arguments and perform actions. """ + + # Ignoring the current running service + running_service=f"system-maintenance-service-{action}" + if running_service not in ignored_services: + ignored_services.append(running_service) + filtered_services = filter_services(services, ignored_services) print(f"Services to handle: {services}") print(f"Services to ignore: {ignored_services}") diff --git a/roles/system-maintenance-service-freezer/tasks/main.yml b/roles/system-maintenance-service-freezer/tasks/main.yml index 4e24ca99..dc15e03d 100644 --- a/roles/system-maintenance-service-freezer/tasks/main.yml +++ b/roles/system-maintenance-service-freezer/tasks/main.yml @@ -5,9 +5,14 @@ dest: "{{path_system_maintenance_service_freezer_script}}" when: run_once_system_maintenance_service_freeze is not defined -- name: run the system_maintenance_service_freezer tasks once - set_fact: - run_once_system_maintenance_service_freeze: true +- name: Configure system-maintenance-service for each action + loop: + - freeze + - defrost + template: + src: system-maintenance-service-freezer.service.j2 + dest: "/etc/systemd/system/system-maintenance-service-{{ item }}.service" + notify: "reload system-maintenance-service-{{ item }}.service" when: run_once_system_maintenance_service_freeze is not defined - name: "restart system-maintenance-service.service" @@ -18,10 +23,6 @@ daemon_reload: yes when: maintenance_service_freeze_action_last is not defined or maintenance_service_freeze_action_last != system_maintenance_service_freeze_action -- name: "set variable to prevent loading when action status didn't change" - set_fact: - maintenance_service_freeze_action_last: "{{system_maintenance_service_freeze_action}}" - - name: create system-maintenance-service-defrost.timer template: src: system-maintenance-service-defrost.timer.j2 @@ -29,4 +30,15 @@ register: system_maintenance_service_defrost_timer changed_when: system_maintenance_service_defrost_timer.changed or activate_all_timers | bool notify: restart system-maintenance-service-defrost.timer - when: run_once_system_maintenance_service_freeze is not defined \ No newline at end of file + when: run_once_system_maintenance_service_freeze is not defined + +## Runtime Variable Setting + +- name: run the system_maintenance_service_freezer tasks once + set_fact: + run_once_system_maintenance_service_freeze: true + when: run_once_system_maintenance_service_freeze is not defined + +- name: "set variable to prevent loading when action status didn't change" + set_fact: + maintenance_service_freeze_action_last: "{{system_maintenance_service_freeze_action}}" diff --git a/roles/system-maintenance-service-freezer/templates/system-maintenance-service-freezer.service.j2 b/roles/system-maintenance-service-freezer/templates/system-maintenance-service-freezer.service.j2 index 38b668d7..6cf18b92 100644 --- a/roles/system-maintenance-service-freezer/templates/system-maintenance-service-freezer.service.j2 +++ b/roles/system-maintenance-service-freezer/templates/system-maintenance-service-freezer.service.j2 @@ -4,5 +4,5 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot -ExecPreStart=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} {{item}} "{{ system_maintenance_services + [item] | join(' ') }}" --ignore "{{item}}" --timeout "{{system_maintenance_timeout_defroster}}"' -ExecStart=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} {{item}} "{{ system_maintenance_services }}"' \ No newline at end of file +ExecPreStart=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} {{item}} {{ system_maintenance_services | join(' ') }} --timeout "{{system_maintenance_timeout_defroster}}"' +ExecStart=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} {{item}} {{ system_maintenance_services | join(' ') }}' \ No newline at end of file diff --git a/roles/update-docker/templates/update-docker.service.j2 b/roles/update-docker/templates/update-docker.service.j2 index c3b7a8c6..528a8716 100644 --- a/roles/update-docker/templates/update-docker.service.j2 +++ b/roles/update-docker/templates/update-docker.service.j2 @@ -5,5 +5,5 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot {% if force_backup_before_update | bool %}ExecStartPre=/bin/sh -c 'systemctl start backup-docker-to-local.service'{% endif %} -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze "{{ system_maintenance_services | join(' ') }}" --ignore "{{system_maintenance_cleanup_services + ['update-docker'] | join(' ') }}" --timeout "{{system_maintenance_timeout_heal_docker}}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services + ['update-docker'] | join(' ') }} --timeout "{{system_maintenance_timeout_heal_docker}}"' ExecStart=/bin/sh -c '/usr/bin/python {{update_docker_script}} {{path_docker_compose_instances}} && systemctl start system-maintenance-service-defrost.service' \ No newline at end of file