Compare commits

...

5 Commits

15 changed files with 66 additions and 36 deletions

View File

@ -58,7 +58,7 @@ system_maintenance_timeout_cleanup_services: "15min"
system_maintenance_timeout_backup_services: "1h" system_maintenance_timeout_backup_services: "1h"
system_maintenance_timeout_heal_docker: "30min" system_maintenance_timeout_heal_docker: "30min"
system_maintenance_timeout_update_docker: "5min" system_maintenance_timeout_update_docker: "5min"
system_maintenance_timeout_defroster: "10min" system_maintenance_timeout_defroster: "2min"
## Services ## Services

View File

@ -30,7 +30,7 @@
- name: configure backup-docker-to-local.timer.tpl - name: configure backup-docker-to-local.timer.tpl
template: src=backup-docker-to-local.timer.j2 dest=/etc/systemd/system/backup-docker-to-local.timer template: src=backup-docker-to-local.timer.j2 dest=/etc/systemd/system/backup-docker-to-local.timer
register: backup_docker_to_local_timer register: backup_docker_to_local_timer
changed_when: backup_docker_to_local_timer.changed or activate_all_timers | default(false) | bool changed_when: backup_docker_to_local_timer.changed or activate_all_timers | bool
notify: restart backup-docker-to-local.timer notify: restart backup-docker-to-local.timer
when: run_once_backup_docker_to_local is not defined when: run_once_backup_docker_to_local is not defined

View File

@ -17,7 +17,7 @@
- name: create backup-remote-to-local.timer - name: create backup-remote-to-local.timer
template: src=backup-remote-to-local.timer.j2 dest=/etc/systemd/system/backup-remote-to-local.timer template: src=backup-remote-to-local.timer.j2 dest=/etc/systemd/system/backup-remote-to-local.timer
register: backup_backup_remote_to_local_timer register: backup_backup_remote_to_local_timer
changed_when: backup_backup_remote_to_local_timer.changed or activate_all_timers | default(false) | bool changed_when: backup_backup_remote_to_local_timer.changed or activate_all_timers | bool
notify: restart backup-remote-to-local timer notify: restart backup-remote-to-local timer
- name: create backups-remote-to-local.sh - name: create backups-remote-to-local.sh

View File

@ -3,7 +3,7 @@
src: "cleanup-backups.timer.j2" src: "cleanup-backups.timer.j2"
dest: "/etc/systemd/system/cleanup-backups.timer" dest: "/etc/systemd/system/cleanup-backups.timer"
register: cleanup_backups_timer register: cleanup_backups_timer
changed_when: cleanup_backups_timer.changed or activate_all_timers | default(false) | bool changed_when: cleanup_backups_timer.changed or activate_all_timers | bool
notify: restart cleanup-backups.timer notify: restart cleanup-backups.timer
when: run_once_cleanup_backups_timer is not defined when: run_once_cleanup_backups_timer is not defined

View File

@ -20,5 +20,5 @@
src: cleanup-disc-space.timer.j2 src: cleanup-disc-space.timer.j2
dest: /etc/systemd/system/cleanup-disc-space.timer dest: /etc/systemd/system/cleanup-disc-space.timer
register: cleanup_disc_space_timer register: cleanup_disc_space_timer
changed_when: cleanup_disc_space_timer.changed or activate_all_timers | default(false) | bool changed_when: cleanup_disc_space_timer.changed or activate_all_timers | bool
notify: restart cleanup-disc-space.timer notify: restart cleanup-disc-space.timer

View File

@ -56,5 +56,5 @@
src: "deploy-letsencrypt-mailu.timer.j2" src: "deploy-letsencrypt-mailu.timer.j2"
dest: "/etc/systemd/system/deploy-letsencrypt-mailu.timer" dest: "/etc/systemd/system/deploy-letsencrypt-mailu.timer"
register: deploy_letsencrypt_mailu_timer register: deploy_letsencrypt_mailu_timer
changed_when: deploy_letsencrypt_mailu_timer.changed or activate_all_timers | default(false) | bool changed_when: deploy_letsencrypt_mailu_timer.changed or activate_all_timers | bool
notify: restart deploy-letsencrypt-mailu.timer notify: restart deploy-letsencrypt-mailu.timer

View File

@ -23,7 +23,7 @@
src: heal-docker.timer.j2 src: heal-docker.timer.j2
dest: "/etc/systemd/system/heal-docker.timer" dest: "/etc/systemd/system/heal-docker.timer"
register: heal_docker_timer register: heal_docker_timer
changed_when: heal_docker_timer.changed or activate_all_timers | default(false) | bool changed_when: heal_docker_timer.changed or activate_all_timers | bool
notify: restart heal-docker.timer notify: restart heal-docker.timer
when: run_once_heal_docker is not defined when: run_once_heal_docker is not defined

View File

@ -20,5 +20,5 @@
src: health-btrfs.timer.j2 src: health-btrfs.timer.j2
dest: "/etc/systemd/system/health-btrfs.timer" dest: "/etc/systemd/system/health-btrfs.timer"
register: health_btrs_timer register: health_btrs_timer
changed_when: health_btrs_timer.changed or activate_all_timers | default(false) | bool changed_when: health_btrs_timer.changed or activate_all_timers | bool
notify: restart health-btrfs.timer notify: restart health-btrfs.timer

View File

@ -20,5 +20,5 @@
src: health-disc-space.timer.j2 src: health-disc-space.timer.j2
dest: /etc/systemd/system/health-disc-space.timer dest: /etc/systemd/system/health-disc-space.timer
register: health_disc_space_timer register: health_disc_space_timer
changed_when: health_disc_space_timer.changed or activate_all_timers | default(false) | bool changed_when: health_disc_space_timer.changed or activate_all_timers | bool
notify: restart health-disc-space.timer notify: restart health-disc-space.timer

View File

@ -21,7 +21,7 @@
src: health-docker-container.timer.j2 src: health-docker-container.timer.j2
dest: "/etc/systemd/system/health-docker-container.timer" dest: "/etc/systemd/system/health-docker-container.timer"
register: health_docker_container_timer register: health_docker_container_timer
changed_when: health_docker_container_timer.changed or activate_all_timers | default(false) | bool changed_when: health_docker_container_timer.changed or activate_all_timers | bool
notify: restart health-docker-container.timer notify: restart health-docker-container.timer
when: run_once_health_docker_container is not defined when: run_once_health_docker_container is not defined

View File

@ -21,7 +21,7 @@
src: health-docker-volumes.timer.j2 src: health-docker-volumes.timer.j2
dest: "/etc/systemd/system/health-docker-volumes.timer" dest: "/etc/systemd/system/health-docker-volumes.timer"
register: health_docker_volumes_timer register: health_docker_volumes_timer
changed_when: health_docker_volumes_timer.changed or activate_all_timers | default(false) | bool changed_when: health_docker_volumes_timer.changed or activate_all_timers | bool
notify: restart health-docker-volumes.timer notify: restart health-docker-volumes.timer
when: run_once_health_docker_volumes is not defined when: run_once_health_docker_volumes is not defined

View File

@ -20,5 +20,5 @@
src: health-journalctl.timer.j2 src: health-journalctl.timer.j2
dest: /etc/systemd/system/health-journalctl.timer dest: /etc/systemd/system/health-journalctl.timer
register: health_journalctl_timer register: health_journalctl_timer
changed_when: health_journalctl_timer.changed or activate_all_timers | default(false) | bool changed_when: health_journalctl_timer.changed or activate_all_timers | bool
notify: restart health-journalctl.timer notify: restart health-journalctl.timer

View File

@ -29,7 +29,7 @@
src: health-nginx.timer.j2 src: health-nginx.timer.j2
dest: "/etc/systemd/system/health-nginx.timer" dest: "/etc/systemd/system/health-nginx.timer"
register: health_nginx_timer register: health_nginx_timer
changed_when: health_nginx_timer.changed or activate_all_timers | default(false) | bool changed_when: health_nginx_timer.changed or activate_all_timers | bool
notify: restart health-nginx.timer notify: restart health-nginx.timer
when: run_once_health_nginx is not defined when: run_once_health_nginx is not defined

View File

@ -16,7 +16,7 @@
src: certbot.timer.j2 src: certbot.timer.j2
dest: /etc/systemd/system/certbot.timer dest: /etc/systemd/system/certbot.timer
register: certbot_timer register: certbot_timer
changed_when: certbot_timer.changed or activate_all_timers | default(false) | bool changed_when: certbot_timer.changed or activate_all_timers | bool
notify: restart certbot timer notify: restart certbot timer
when: run_once_nginx_certbot is not defined when: run_once_nginx_certbot is not defined

View File

@ -4,6 +4,14 @@ import time
import os import os
from datetime import datetime from datetime import datetime
# Global variable definition
BREAK_TIME_SECONDS = 5
FREEZER_SERVICES_PREFIX="system-maintenance-service-"
class AttemptException(Exception):
"""A custom exception for maximum number of attempts."""
pass
def parse_time_to_seconds(time_str): def parse_time_to_seconds(time_str):
""" """
Convert a time string (e.g., '1h', '30min', '45s') to seconds. Convert a time string (e.g., '1h', '30min', '45s') to seconds.
@ -47,19 +55,36 @@ def check_any_service_active(services):
""" """
return any(check_service_active(service) for service in services) return any(check_service_active(service) for service in services)
def manage_timer(service, action):
"""
Manage a systemd timer for a service.
action can be 'start' or 'stop'.
"""
if action not in ['start', 'stop']:
raise ValueError("Invalid action specified for manage_timer")
timer_name = f"{service}.timer"
try:
subprocess.run(['systemctl', action, timer_name], check=True)
if action == 'start':
subprocess.run(['systemctl', 'enable', timer_name], check=True)
elif action == 'stop':
subprocess.run(['systemctl', 'disable', timer_name], check=True)
print(f"{timer_name} {action}ed and {'enabled' if action == 'start' else 'disabled'}.")
except subprocess.CalledProcessError as e:
print(f"Error managing timer {timer_name}: {e}")
exit(1)
def stop_timer(service): def stop_timer(service):
""" """
Stop and disable a systemd timer for a service if it exists. Stop and disable a systemd timer for a service if it exists.
""" """
if service == "system-maintenance-service-defrost": if service == f"{FREEZER_SERVICES_PREFIX}defrost":
print(f"Ignoring {service}. It's the initializer of freezer.") print(f"Ignoring {service}. It's the initializer of freezer.")
if service_file_exists(service, "timer"): if service_file_exists(service, "timer"):
timer_name = f"{service}.timer" manage_timer(service, 'stop')
subprocess.run(['systemctl', 'stop', timer_name])
subprocess.run(['systemctl', 'disable', timer_name])
print(f"{timer_name} stopped and disabled.")
else: else:
print("No timer to stop for service.") print(f"Timer {service}.timer does not exist.")
def filter_services(services, ignored_services): def filter_services(services, ignored_services):
""" """
@ -74,7 +99,7 @@ def stop_all_timers(services):
for service in services: for service in services:
stop_timer(service) stop_timer(service)
def wait_for_all_services_to_stop(filtered_services, max_attempts, attempt, break_time_sec): def wait_for_all_services_to_stop(filtered_services, max_attempts, attempt):
""" """
Wait until all services in the list have stopped, with a maximum number of attempts. Wait until all services in the list have stopped, with a maximum number of attempts.
""" """
@ -82,40 +107,45 @@ def wait_for_all_services_to_stop(filtered_services, max_attempts, attempt, brea
while check_service_active(service): while check_service_active(service):
attempt += 1 attempt += 1
if attempt > max_attempts: if attempt > max_attempts:
raise Exception(f"Maximum attempts ({max_attempts}) reached. Exiting.") raise AttemptException(f"Maximum attempts ({max_attempts}) reached. Exiting.")
print(f"{datetime.now().isoformat()}#{attempt}/{max_attempts}: Waiting for {break_time_sec} seconds for {service} to stop...") print(f"{datetime.now().isoformat()}#{attempt}/{max_attempts}: Waiting for {BREAK_TIME_SECONDS} seconds for {service} to stop...")
time.sleep(break_time_sec) time.sleep(BREAK_TIME_SECONDS)
return attempt return attempt
def freeze(filtered_services, timeout_sec): def freeze(filtered_services, timeout_sec):
""" """
Freeze services by stopping them and their timers, waiting up to a timeout. Freeze services by stopping them and their timers, waiting up to a timeout.
""" """
break_time_sec = 5
attempt = 0 attempt = 0
max_attempts = timeout_sec / break_time_sec max_attempts = get_max_attempts(timeout_sec)
while check_any_service_active(filtered_services): while check_any_service_active(filtered_services):
stop_all_timers(filtered_services) stop_all_timers(filtered_services)
attempt = wait_for_all_services_to_stop(filtered_services, max_attempts, attempt, break_time_sec) attempt = wait_for_all_services_to_stop(filtered_services, max_attempts, attempt)
print("All required services have stopped.") print("All required services have stopped.")
def get_max_attempts(timeout_sec):
return timeout_sec // BREAK_TIME_SECONDS
def defrost(filtered_services,timeout_sec): def defrost(filtered_services,timeout_sec):
""" """
Defrost services by starting and enabling their timers. Defrost services by starting and enabling their timers.
""" """
break_time_sec = 5 running_service = f"{FREEZER_SERVICES_PREFIX}defrost"
attempt = 0 attempt = 0
max_attempts = timeout_sec / break_time_sec max_attempts = get_max_attempts(timeout_sec)
wait_for_all_services_to_stop(filtered_services, max_attempts, attempt, break_time_sec) try:
wait_for_all_services_to_stop(filtered_services, max_attempts, attempt)
except AttemptException as e:
print(e)
print("Defrosting was not possible. The execution of other services took to long.")
manage_timer(running_service, "stop")
exit(0)
for service in filtered_services + [f"system-maintenance-service-{action}"]: for service in filtered_services + [running_service]:
print(f"Unfreezing: {service}") print(f"Unfreezing: {service}")
if service_file_exists(service, "timer"): if service_file_exists(service, "timer"):
timer_name = f"{service}.timer" manage_timer(service, "start")
subprocess.run(['systemctl', 'start', timer_name])
subprocess.run(['systemctl', 'enable', timer_name])
print(f"{timer_name} started and enabled.")
else: else:
print("No timer to activate for service.") print("No timer to activate for service.")
print("All required services are started.") print("All required services are started.")
@ -126,7 +156,7 @@ def main(services, ignored_services, action, timeout_sec):
""" """
# Ignoring the current running service # Ignoring the current running service
running_service=f"system-maintenance-service-{action}" running_service=f"{FREEZER_SERVICES_PREFIX}{action}"
if running_service not in ignored_services: if running_service not in ignored_services:
ignored_services.append(running_service) ignored_services.append(running_service)