From 9c21d052c42b3328ec2aa6b7e6bc45267769edfa Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 16 Dec 2023 20:37:40 +0100 Subject: [PATCH] Implemented system-maintenance-lock and reduced unnecessary complexity --- COMMON_APPLICATIONS.md | 2 +- README.md | 2 +- destructor.yml | 8 - group_vars/all | 28 +-- roles/backup-data-to-usb/meta/main.yml | 2 +- roles/backup-docker-to-local/meta/main.yml | 2 +- .../backup-docker-to-local.service.j2 | 2 +- roles/backup-remote-to-local/meta/main.yml | 2 +- .../backup-remote-to-local.service.j2 | 2 +- roles/cleanup-backups-service/meta/main.yml | 2 +- .../templates/cleanup-backups.service.j2 | 2 +- roles/cleanup-disc-space/meta/main.yml | 2 +- .../templates/cleanup-disc-space.service.j2 | 2 +- .../meta/main.yml | 2 +- .../cleanup-failed-docker-backups.service.j2 | 2 +- roles/heal-docker/meta/main.yml | 2 +- .../templates/heal-docker.service.j2 | 2 +- roles/system-maintenance-lock/README.md | 10 + .../files/system-maintenance-lock.py | 96 +++++++++ .../system-maintenance-lock/handlers/main.yml | 0 roles/system-maintenance-lock/tasks/main.yml | 20 ++ .../README.md | 22 --- .../system-maintenance-service-freezer.py | 187 ------------------ .../handlers/main.yml | 16 -- .../meta/main.yml | 3 - .../tasks/main.yml | 44 ----- ...ystem-maintenance-service-defrost.timer.j2 | 10 - ...tem-maintenance-service-freezer.service.j2 | 7 - roles/update-docker/meta/main.yml | 2 +- .../templates/update-docker.service.j2 | 2 +- 30 files changed, 152 insertions(+), 333 deletions(-) create mode 100644 roles/system-maintenance-lock/README.md create mode 100644 roles/system-maintenance-lock/files/system-maintenance-lock.py create mode 100644 roles/system-maintenance-lock/handlers/main.yml create mode 100644 roles/system-maintenance-lock/tasks/main.yml delete mode 100644 roles/system-maintenance-service-freezer/README.md delete mode 100644 roles/system-maintenance-service-freezer/files/system-maintenance-service-freezer.py delete mode 100644 roles/system-maintenance-service-freezer/handlers/main.yml delete mode 100644 roles/system-maintenance-service-freezer/meta/main.yml delete mode 100644 roles/system-maintenance-service-freezer/tasks/main.yml delete mode 100644 roles/system-maintenance-service-freezer/templates/system-maintenance-service-defrost.timer.j2 delete mode 100644 roles/system-maintenance-service-freezer/templates/system-maintenance-service-freezer.service.j2 diff --git a/COMMON_APPLICATIONS.md b/COMMON_APPLICATIONS.md index 4a7767c3..cdd331db 100644 --- a/COMMON_APPLICATIONS.md +++ b/COMMON_APPLICATIONS.md @@ -40,7 +40,7 @@ Enhances system security with roles focused on security measures, user configura - **[User Alarm](./roles/user-alarm/)**: Manages the alarm user. - **[PC SSH](./roles/pc-ssh/)**: Configuration of SSH for secure remote access. - **[SSHD](./roles/sshd/)**: Configures SSH daemon settings. -- **[System Maintenance Service Freezer](./roles/system-maintenance-service-freezer)**: Freezes and defrost maintenance services to prevent dangerous inteactions between services +- **[System Maintanance Lock](./roles/system-maintenance-lock)**: Locks maintenance services to prevent dangerous inteactions between services ## Virtual Private Network (VPN) Centers on VPN configurations for secure and efficient network connectivity, particularly crucial for remote server access and end-users needing secure connections. diff --git a/README.md b/README.md index 54a9a44d..abd56a92 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ Contact me for more details: ## Showcases The following list showcases the extensive range of solutions that CyMaIS incorporates, each playing a vital role in providing a comprehensive, efficient, and secure IT infrastructure setup: -[ELK Stack](./roles/docker-elk), [Intel Driver](./roles/driver-intel), [Nginx Docker Reverse Proxy](./roles/nginx-docker-reverse-proxy), [Sudo](./roles/sudo), [Funkwhale](./roles/docker-funkwhale), [MSI Keyboard Color Driver](./roles/driver-msi-keyboard-color), [Nginx Domain Redirect](./roles/nginx-domain-redirect), [GnuCash](./roles/pc-gnucash), [Backup Data to USB](./roles/backup-data-to-usb), [Gitea](./roles/docker-gitea), [Non-Free Driver](./roles/driver-non-free), [Nginx Homepage](./roles/nginx-homepage), [Jrnl](./roles/pc-jrnl), [Systemd Notifier](./roles/systemd-notifier), [Backup Docker to Local](./roles/backup-docker-to-local), [Jenkins](./roles/docker-jenkins), [Git](./roles/git), [Nginx HTTPS](./roles/nginx-https), [Latex](./roles/pc-latex), [Email Notifier](./roles/systemd-notifier-email), [Remote to Local Backup Solution](./roles/backup-remote-to-local), [Joomla](./roles/docker-joomla), [Heal Defect Docker Installations](./roles/heal-docker), [Nginx Matomo Tracking](./roles/nginx-matomo-tracking), [LibreOffice](./roles/pc-libreoffice), [Telegram Notifier](./roles/systemd-notifier-telegram), [Listmonk](./roles/docker-listmonk), [Btrfs Health Check](./roles/health-btrfs), [Nginx WWW Redirect](./roles/nginx-www-redirect), [Network Analyze Tools](./roles/pc-network-analyze-tools), [System Security](./roles/system-security), [Mailu](./roles/docker-mailu), [Disc Space Health Check](./roles/health-disc-space), [Administrator Tools](./roles/pc-administrator-tools), [Nextcloud](./roles/pc-nextcloud), [Swapfile Setup](./roles/system-swapfile), [Backups Cleanup](./roles/cleanup-backups-service), [Mastodon](./roles/docker-mastodon), [Docker Container Health Checker](./roles/health-docker-container), [Blu-ray Player Tools](./roles/pc-bluray-player-tools), [Office](./roles/pc-office), [Update Solutions](./roles/update), [Matomo](./roles/docker-matomo), [Docker Volumes Health Checker](./roles/health-docker-volumes), [Caffeine](./roles/pc-caffeine), [Qbittorrent](./roles/pc-qbittorrent), [Update Apt](./roles/update-apt), [Disc Space Cleanup](./roles/cleanup-disc-space), [Matrix](./roles/docker-matrix), [Health Journalctl](./roles/health-journalctl), [Designer Tools](./roles/pc-designer-tools), [Security Tools](./roles/pc-security-tools), [Update Docker](./roles/update-docker), [Failed Docker Backups Cleanup](./roles/cleanup-failed-docker-backups), [MediaWiki](./roles/docker-mediawiki), [Nginx Health Checker](./roles/health-nginx), [Developer Tools](./roles/pc-developer-tools), [Spotify](./roles/pc-spotify), [Update Pacman](./roles/update-pacman), [Client Wireguard](./roles/client-wireguard), [MyBB](./roles/docker-mybb), [Developer Tools for Arduino](./roles/pc-developer-tools-arduino), [SSH](./roles/pc-ssh), [Update Yay](./roles/update-yay), [Client Setup for Wireguard Behind Firewall](./roles/client-wireguard-behind-firewall), [Nextcloud](./roles/docker-nextcloud), [Hunspell](./roles/hunspell), [Developer Tools for Bash](./roles/pc-developer-tools-bash), [Streaming Tools](./roles/pc-streaming-tools), [Administrator](./roles/user-administrator), [Docker](./roles/docker), [Peertube](./roles/docker-peertube), [Java](./roles/java), [Developer Tools for Java](./roles/pc-developer-tools-java), [Tor Browser](./roles/pc-torbrowser), [Video Conference](./roles/pc-video-conference), [Wireguard](./roles/wireguard), [Akaunting](./roles/docker-akaunting), [Pixelfed](./roles/docker-pixelfed), [Journalctl](./roles/journalctl), [Developer Tools for PHP](./roles/pc-developer-tools-php), [Virtual Box](./roles/pc-virtual-box), [Postfix](./roles/postfix), [Attendize](./roles/docker-attendize), [Wordpress](./roles/docker-wordpress), [Locales](./roles/locales), [Docker for End Users](./roles/pc-docker), [Games](./roles/pc-games), [Python Pip](./roles/python-pip), [Discourse](./roles/docker-discourse), [Epson Multiprinter Driver](./roles/driver-epson-multiprinter), [Nginx Certbot](./roles/nginx-certbot), [Git](./roles/pc-git), [SSHD](./roles/sshd), [YOURLS](./roles/docker-yourls), [BigBlueButton](./roles/docker-bigbluebutton),[System Maintenance Service Freezer](./roles/system-maintenance-service-freezer)... +[ELK Stack](./roles/docker-elk), [Intel Driver](./roles/driver-intel), [Nginx Docker Reverse Proxy](./roles/nginx-docker-reverse-proxy), [Sudo](./roles/sudo), [Funkwhale](./roles/docker-funkwhale), [MSI Keyboard Color Driver](./roles/driver-msi-keyboard-color), [Nginx Domain Redirect](./roles/nginx-domain-redirect), [GnuCash](./roles/pc-gnucash), [Backup Data to USB](./roles/backup-data-to-usb), [Gitea](./roles/docker-gitea), [Non-Free Driver](./roles/driver-non-free), [Nginx Homepage](./roles/nginx-homepage), [Jrnl](./roles/pc-jrnl), [Systemd Notifier](./roles/systemd-notifier), [Backup Docker to Local](./roles/backup-docker-to-local), [Jenkins](./roles/docker-jenkins), [Git](./roles/git), [Nginx HTTPS](./roles/nginx-https), [Latex](./roles/pc-latex), [Email Notifier](./roles/systemd-notifier-email), [Remote to Local Backup Solution](./roles/backup-remote-to-local), [Joomla](./roles/docker-joomla), [Heal Defect Docker Installations](./roles/heal-docker), [Nginx Matomo Tracking](./roles/nginx-matomo-tracking), [LibreOffice](./roles/pc-libreoffice), [Telegram Notifier](./roles/systemd-notifier-telegram), [Listmonk](./roles/docker-listmonk), [Btrfs Health Check](./roles/health-btrfs), [Nginx WWW Redirect](./roles/nginx-www-redirect), [Network Analyze Tools](./roles/pc-network-analyze-tools), [System Security](./roles/system-security), [Mailu](./roles/docker-mailu), [Disc Space Health Check](./roles/health-disc-space), [Administrator Tools](./roles/pc-administrator-tools), [Nextcloud Client](./roles/pc-nextcloud), [Swapfile Setup](./roles/system-swapfile), [Backups Cleanup](./roles/cleanup-backups-service), [Mastodon](./roles/docker-mastodon), [Docker Container Health Checker](./roles/health-docker-container), [Blu-ray Player Tools](./roles/pc-bluray-player-tools), [Office](./roles/pc-office), [Update Solutions](./roles/update), [Matomo](./roles/docker-matomo), [Docker Volumes Health Checker](./roles/health-docker-volumes), [Caffeine](./roles/pc-caffeine), [Qbittorrent](./roles/pc-qbittorrent), [Update Apt](./roles/update-apt), [Disc Space Cleanup](./roles/cleanup-disc-space), [Matrix](./roles/docker-matrix), [Health Journalctl](./roles/health-journalctl), [Designer Tools](./roles/pc-designer-tools), [Security Tools](./roles/pc-security-tools), [Update Docker](./roles/update-docker), [Failed Docker Backups Cleanup](./roles/cleanup-failed-docker-backups), [MediaWiki](./roles/docker-mediawiki), [Nginx Health Checker](./roles/health-nginx), [Developer Tools](./roles/pc-developer-tools), [Spotify](./roles/pc-spotify), [Update Pacman](./roles/update-pacman), [Client Wireguard](./roles/client-wireguard), [MyBB](./roles/docker-mybb), [Developer Tools for Arduino](./roles/pc-developer-tools-arduino), [SSH](./roles/pc-ssh), [Update Yay](./roles/update-yay), [Client Setup for Wireguard Behind Firewall](./roles/client-wireguard-behind-firewall), [Nextcloud Server](./roles/docker-nextcloud), [Hunspell](./roles/hunspell), [Developer Tools for Bash](./roles/pc-developer-tools-bash), [Streaming Tools](./roles/pc-streaming-tools), [Administrator](./roles/user-administrator), [Docker](./roles/docker), [Peertube](./roles/docker-peertube), [Java](./roles/java), [Developer Tools for Java](./roles/pc-developer-tools-java), [Tor Browser](./roles/pc-torbrowser), [Video Conference](./roles/pc-video-conference), [Wireguard](./roles/wireguard), [Akaunting](./roles/docker-akaunting), [Pixelfed](./roles/docker-pixelfed), [Journalctl](./roles/journalctl), [Developer Tools for PHP](./roles/pc-developer-tools-php), [Virtual Box](./roles/pc-virtual-box), [Postfix](./roles/postfix), [Attendize](./roles/docker-attendize), [Wordpress](./roles/docker-wordpress), [Locales](./roles/locales), [Docker for End Users](./roles/pc-docker), [Games](./roles/pc-games), [Python Pip](./roles/python-pip), [Discourse](./roles/docker-discourse), [Epson Multiprinter Driver](./roles/driver-epson-multiprinter), [Nginx Certbot](./roles/nginx-certbot), [Git](./roles/pc-git), [SSHD](./roles/sshd), [YOURLS](./roles/docker-yourls), [BigBlueButton](./roles/docker-bigbluebutton),[System Maintanance Lock](./roles/system-maintenance-lock)... ## License diff --git a/destructor.yml b/destructor.yml index 1513f8ee..e69de29b 100644 --- a/destructor.yml +++ b/destructor.yml @@ -1,8 +0,0 @@ ---- -- name: call destructor method - hosts: all - become: true - roles: - - role: system-maintenance-service-freezer - vars: - system_maintenance_service_freeze_action: "defrost" \ No newline at end of file diff --git a/group_vars/all b/group_vars/all index 893e5b8f..d6899768 100644 --- a/group_vars/all +++ b/group_vars/all @@ -24,7 +24,6 @@ on_calendar_backup_remote_to_local: "*-*-* 21:30:00" ## Schedule for Maintenance Tasks on_calendar_heal_docker: "*-*-* {{ hours_server_awake }}:30:00" # Heal unhealthy docker instances once per hour -on_calendar_defrost: "*:0/5" # Defrost every 5min on_calendar_renew_lets_encrypt_certificates: "*-*-* 12,00:30:00" # Renew Mailu certificates twice per day on_calendar_deploy_mailu_certificates: "*-*-* 13,01:30:00" # Deploy Mailu certificates twice per day on_calendar_msi_keyboard_color: "*-*-* *:*:00" # Change the keyboard color every minute @@ -37,11 +36,11 @@ size_percent_cleanup_disc_space: 90 # Threshold for triggering cle # Path Variables for Key Directories and Scripts -path_administrator_home: "/home/administrator/" -path_administrator_scripts: "{{path_administrator_home}}scripts/" -path_docker_volumes: "{{path_administrator_home}}volumes/docker/" -path_docker_compose_instances: "{{path_administrator_home}}docker-compose/" -path_system_maintenance_service_freezer_script: "{{path_administrator_scripts}}system-maintenance-service-freezer.py" +path_administrator_home: "/home/administrator/" +path_administrator_scripts: "{{path_administrator_home}}scripts/" +path_docker_volumes: "{{path_administrator_home}}volumes/docker/" +path_docker_compose_instances: "{{path_administrator_home}}docker-compose/" +path_system_lock_script: "{{path_administrator_scripts}}system-maintenance-lock.py" # Runtime Variables for Process Control @@ -54,11 +53,10 @@ force_backup_before_update: true # Activates the backup before the update p # System maintenance Services ## Timeouts to wait for other services to stop -system_maintenance_timeout_cleanup_services: "15min" -system_maintenance_timeout_backup_services: "1h" -system_maintenance_timeout_heal_docker: "30min" -system_maintenance_timeout_update_docker: "5min" -system_maintenance_timeout_freezer_action: "2min" +sytem_maintenance_lock_timeoutcleanup_services: "15min" +sytem_maintenance_lock_timeoutbackup_services: "1h" +sytem_maintenance_lock_timeoutheal_docker: "30min" +sytem_maintenance_lock_timeoutupdate_docker: "2min" ## Services @@ -74,10 +72,6 @@ system_maintenance_cleanup_services: - "cleanup-disc-space" - "cleanup-failed-docker-backups" -### Freeze services (wait until they are finished to be sure that nobody else is doing stuff in the fridge) - - "system-maintenance-service-freeze" - - "system-maintenance-service-defrost" - ### Services that Manipulate the System system_maintenance_manipulation_services: - "heal-docker" @@ -86,10 +80,6 @@ system_maintenance_manipulation_services: ## Total System Maintenance Services system_maintenance_services: "{{ system_maintenance_backup_services + system_maintenance_cleanup_services + system_maintenance_manipulation_services }}" -## First default freezer action to apply when freezer service get triggered during play -system_maintenance_service_freeze_action: 'freeze' # Valid Values: freeze, defrost - - # Webserver Configuration ## Nginx-Specific Path Configurations diff --git a/roles/backup-data-to-usb/meta/main.yml b/roles/backup-data-to-usb/meta/main.yml index add533e5..cd4891be 100644 --- a/roles/backup-data-to-usb/meta/main.yml +++ b/roles/backup-data-to-usb/meta/main.yml @@ -1,4 +1,4 @@ --- dependencies: - role: cleanup-backups-service - - role: system-maintenance-service-freezer + - role: system-maintenance-lock diff --git a/roles/backup-docker-to-local/meta/main.yml b/roles/backup-docker-to-local/meta/main.yml index 8bb53c96..09266425 100644 --- a/roles/backup-docker-to-local/meta/main.yml +++ b/roles/backup-docker-to-local/meta/main.yml @@ -3,4 +3,4 @@ dependencies: - backups-provider - systemd-notifier - cleanup-failed-docker-backups - - system-maintenance-service-freezer \ No newline at end of file + - system-maintenance-lock \ No newline at end of file diff --git a/roles/backup-docker-to-local/templates/backup-docker-to-local.service.j2 b/roles/backup-docker-to-local/templates/backup-docker-to-local.service.j2 index 63c3ffae..ec4caa59 100644 --- a/roles/backup-docker-to-local/templates/backup-docker-to-local.service.j2 +++ b/roles/backup-docker-to-local/templates/backup-docker-to-local.service.j2 @@ -4,5 +4,5 @@ OnFailure=systemd-notifier@%n.service cleanup-failed-docker-backups.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_backup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_lock_script }} {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_backup_services| join(' ') }} --timeout "{{sytem_maintenance_lock_timeoutbackup_services}}"' ExecStart=/bin/sh -c '/usr/bin/python {{backup_docker_to_local_folder}}backup-docker-to-local.py' \ No newline at end of file diff --git a/roles/backup-remote-to-local/meta/main.yml b/roles/backup-remote-to-local/meta/main.yml index 634f4b04..a734813d 100644 --- a/roles/backup-remote-to-local/meta/main.yml +++ b/roles/backup-remote-to-local/meta/main.yml @@ -3,4 +3,4 @@ dependencies: - systemd-notifier - cleanup-backups-timer - cleanup-failed-docker-backups - - system-maintenance-service-freezer + - system-maintenance-lock diff --git a/roles/backup-remote-to-local/templates/backup-remote-to-local.service.j2 b/roles/backup-remote-to-local/templates/backup-remote-to-local.service.j2 index 54cb98b9..4dea5cf7 100644 --- a/roles/backup-remote-to-local/templates/backup-remote-to-local.service.j2 +++ b/roles/backup-remote-to-local/templates/backup-remote-to-local.service.j2 @@ -4,5 +4,5 @@ OnFailure=systemd-notifier@%n.service cleanup-failed-docker-backups.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_backup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_lock_script }} {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_backup_services| join(' ') }} --timeout "{{sytem_maintenance_lock_timeoutbackup_services}}"' ExecStart=/bin/sh -c '/usr/bin/bash {{docker_backup_remote_to_local_folder}}backups-remote-to-local.sh' diff --git a/roles/cleanup-backups-service/meta/main.yml b/roles/cleanup-backups-service/meta/main.yml index 5553d6da..bd8f1996 100644 --- a/roles/cleanup-backups-service/meta/main.yml +++ b/roles/cleanup-backups-service/meta/main.yml @@ -1,4 +1,4 @@ dependencies: - python-pip - systemd-notifier - - system-maintenance-service-freezer + - system-maintenance-lock diff --git a/roles/cleanup-backups-service/templates/cleanup-backups.service.j2 b/roles/cleanup-backups-service/templates/cleanup-backups.service.j2 index d8a62113..d09f9849 100644 --- a/roles/cleanup-backups-service/templates/cleanup-backups.service.j2 +++ b/roles/cleanup-backups-service/templates/cleanup-backups.service.j2 @@ -4,5 +4,5 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_lock_script }} {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{sytem_maintenance_lock_timeoutbackup_services}}"' ExecStart=/bin/sh -c '/usr/bin/python {{docker_cleanup_backups}}cleanup-backups.py --backups-folder-path {{backups_folder_path}} --maximum-backup-size-percent {{size_percent_maximum_backup}}' \ No newline at end of file diff --git a/roles/cleanup-disc-space/meta/main.yml b/roles/cleanup-disc-space/meta/main.yml index 8cfa9b66..a9080d44 100644 --- a/roles/cleanup-disc-space/meta/main.yml +++ b/roles/cleanup-disc-space/meta/main.yml @@ -1,3 +1,3 @@ dependencies: - systemd-notifier - - system-maintenance-service-freezer + - system-maintenance-lock diff --git a/roles/cleanup-disc-space/templates/cleanup-disc-space.service.j2 b/roles/cleanup-disc-space/templates/cleanup-disc-space.service.j2 index d23eac8e..91375b96 100644 --- a/roles/cleanup-disc-space/templates/cleanup-disc-space.service.j2 +++ b/roles/cleanup-disc-space/templates/cleanup-disc-space.service.j2 @@ -4,5 +4,5 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_lock_script }} {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{sytem_maintenance_lock_timeoutbackup_services}}"' ExecStart=/bin/sh -c '/bin/bash {{cleanup_disc_space_folder}}cleanup-disc-space.sh {{size_percent_cleanup_disc_space}}' \ No newline at end of file diff --git a/roles/cleanup-failed-docker-backups/meta/main.yml b/roles/cleanup-failed-docker-backups/meta/main.yml index dc05a8b0..34f87494 100644 --- a/roles/cleanup-failed-docker-backups/meta/main.yml +++ b/roles/cleanup-failed-docker-backups/meta/main.yml @@ -1,4 +1,4 @@ dependencies: - git - systemd-notifier - - system-maintenance-service-freezer + - system-maintenance-lock diff --git a/roles/cleanup-failed-docker-backups/templates/cleanup-failed-docker-backups.service.j2 b/roles/cleanup-failed-docker-backups/templates/cleanup-failed-docker-backups.service.j2 index 287400a5..335fc471 100644 --- a/roles/cleanup-failed-docker-backups/templates/cleanup-failed-docker-backups.service.j2 +++ b/roles/cleanup-failed-docker-backups/templates/cleanup-failed-docker-backups.service.j2 @@ -4,5 +4,5 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_lock_script }} {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{sytem_maintenance_lock_timeoutbackup_services}}"' ExecStart=/bin/sh -c '/usr/bin/yes | /usr/bin/bash {{backup_docker_to_local_cleanup_folder}}cleanup.sh {{backup_docker_to_local_cleanup_machine_id}} {{backup_docker_to_local_cleanup_trigger_directory}}' \ No newline at end of file diff --git a/roles/heal-docker/meta/main.yml b/roles/heal-docker/meta/main.yml index 13944fb9..6f898c6d 100644 --- a/roles/heal-docker/meta/main.yml +++ b/roles/heal-docker/meta/main.yml @@ -1,2 +1,2 @@ dependencies: - - system-maintenance-service-freezer + - system-maintenance-lock diff --git a/roles/heal-docker/templates/heal-docker.service.j2 b/roles/heal-docker/templates/heal-docker.service.j2 index 85b04ba7..4c2340ff 100644 --- a/roles/heal-docker/templates/heal-docker.service.j2 +++ b/roles/heal-docker/templates/heal-docker.service.j2 @@ -4,5 +4,5 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} heal-docker --timeout "{{system_maintenance_timeout_heal_docker}}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_lock_script }} {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} heal-docker --timeout "{{sytem_maintenance_lock_timeoutheal_docker}}"' ExecStart=/bin/sh -c '/bin/python {{heal_docker}}heal-docker.py' \ No newline at end of file diff --git a/roles/system-maintenance-lock/README.md b/roles/system-maintenance-lock/README.md new file mode 100644 index 00000000..5746ff56 --- /dev/null +++ b/roles/system-maintenance-lock/README.md @@ -0,0 +1,10 @@ +# Role: System-Maintenance-Lock + +## Overview +The `system-maintenance-lock` role is a critical part of maintaining the integrity and performance of a system. It ensures that specific services are not interrupted or conflicted with by other system processes. This role is particularly vital during system updates, backups, or other maintenance activities where conflicting processes could cause issues. + +## Usage +This role is used in scenarios where system stability and integrity are paramount, such as during system upgrades, backup processes, or when applying critical patches. + +## Created with AI +Created with ChatGPT. Conversation is [here](https://chat.openai.com/share/a886b86b-8de6-4eca-9fba-e36c9f20d536) available. \ No newline at end of file diff --git a/roles/system-maintenance-lock/files/system-maintenance-lock.py b/roles/system-maintenance-lock/files/system-maintenance-lock.py new file mode 100644 index 00000000..eedcc6bc --- /dev/null +++ b/roles/system-maintenance-lock/files/system-maintenance-lock.py @@ -0,0 +1,96 @@ +import argparse +import subprocess +import time +import os +from datetime import datetime + +# Global variable definition +BREAK_TIME_SECONDS = 5 + +class AttemptException(Exception): + """A custom exception for maximum number of attempts.""" + pass + +def parse_time_to_seconds(time_str): + """ + Convert a time string (e.g., '1h', '30min', '45s') to seconds. + """ + units = {"s": 1, "min": 60, "h": 3600} + if time_str[-3:] in units: + number, unit = time_str[:-3], time_str[-3:] + elif time_str[-2:] in units: + number, unit = time_str[:-2], time_str[-2:] + elif time_str[-1:] in units: + number, unit = time_str[:-1], time_str[-1:] + else: + raise ValueError("Invalid time unit") + return int(number) * units[unit] + +def check_service_active(service_name): + """ + Check if a systemd service is currently active or activating. + """ + result = subprocess.run(['systemctl', 'is-active', service_name], stdout=subprocess.PIPE) + service_status = result.stdout.decode('utf-8').strip() + is_active = service_status in ['active', 'activating'] + print(f"Service {service_name} is {'active' if is_active else 'not active'}.") + return is_active + +def check_any_service_active(services): + """ + Check if any service in a given list is active or activating. + """ + return any(check_service_active(service) for service in services) + +def filter_services(services, ignored_services): + """ + Filter out services that are in the ignored_services list from services list. + """ + return [service for service in services if service not in ignored_services] + +def wait_for_all_services_to_stop(filtered_services, max_attempts, attempt): + """ + Wait until all services in the list have stopped, with a maximum number of attempts. + """ + for service in filtered_services: + while check_service_active(service): + attempt += 1 + if attempt > max_attempts: + raise AttemptException(f"Maximum attempts ({max_attempts}) reached. Exiting.") + print(f"{datetime.now().isoformat()}#{attempt}/{max_attempts}: Waiting for {BREAK_TIME_SECONDS} seconds for {service} to stop...") + time.sleep(BREAK_TIME_SECONDS) + return attempt + + +def get_max_attempts(timeout_sec): + return timeout_sec // BREAK_TIME_SECONDS + +def main(services, ignored_services, timeout_sec): + """ + Main function to process the command-line arguments and perform actions. + """ + + filtered_services = filter_services(services, ignored_services) + print(f"Services to handle: {services}") + print(f"Services to ignore: {ignored_services}") + print(f"Services filtered: {filtered_services}") + + print("Waiting for services to stop.") + + attempt = 0 + max_attempts = get_max_attempts(timeout_sec) + while check_any_service_active(filtered_services): + attempt = wait_for_all_services_to_stop(filtered_services, max_attempts, attempt) + print("All required services have stopped.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Blocks the code execution as long as defined services are running. Terminates with 0 when all services stopped') + parser.add_argument('services', nargs='+', help='List of services to apply the action to.') + parser.add_argument('--ignore', nargs='*', help='List of services to ignore in the action.', default=[]) + parser.add_argument('--timeout', help='Timeout for lock actions (e.g., 1h, 30min, 45s).', default='1min') + args = parser.parse_args() + services = args.services + ignored_services = args.ignore if args.ignore else [] + timeout_seconds = parse_time_to_seconds(args.timeout) + main(services, ignored_services, timeout_seconds) diff --git a/roles/system-maintenance-lock/handlers/main.yml b/roles/system-maintenance-lock/handlers/main.yml new file mode 100644 index 00000000..e69de29b diff --git a/roles/system-maintenance-lock/tasks/main.yml b/roles/system-maintenance-lock/tasks/main.yml new file mode 100644 index 00000000..180387f5 --- /dev/null +++ b/roles/system-maintenance-lock/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: create {{path_system_lock_script}} + copy: + src: system-maintenance-lock.py + dest: "{{path_system_lock_script}}" + when: run_once_system-maintenance_lock is not defined + +- name: Configure system-maintenance-service for each action + template: + src: system-maintenance-lock.service.j2 + dest: "/etc/systemd/system/system-maintenance-lock.service" + notify: "reload system-maintenance-lock.service" + when: run_once_system-maintenance_lock is not defined + +## Runtime Variable Setting + +- name: run the system_maintenance_service_freezer tasks once + set_fact: + run_once_system-maintenance_lock: true + when: run_once_system-maintenance_lock is not defined diff --git a/roles/system-maintenance-service-freezer/README.md b/roles/system-maintenance-service-freezer/README.md deleted file mode 100644 index d497fbb0..00000000 --- a/roles/system-maintenance-service-freezer/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# System Maintenance Service Freezer - -## Overview -This Ansible role is designed to manage system services through freezing (disabling) and defrosting (enabling) actions. It automates the process of managing crucial system services, especially useful for maintenance tasks like backups, cleanups, and updates. - -## Monitoring -To monitor the sucess of the script and the status of systemctl timers execute: - -```bash -watch -n 2 systemctl list-timers -``` - -## Role Variables -- `system_maintenance_services`: List of services to be managed by this role. - -## Usage -Configure the role by defining the required variables. The role creates systemd service files that control the specified services based on the `freeze` or `defrost` actions. - -For further details and usage examples, refer to the chat conversation with ChatGPT: [Link to ChatGPT Conversation](https://chat.openai.com/share/212af169-1b57-41df-bd2d-c3d32eb1331b). - -## Dependencies -- `systemd-notifier`: Ensure this role is present for handling service failures. diff --git a/roles/system-maintenance-service-freezer/files/system-maintenance-service-freezer.py b/roles/system-maintenance-service-freezer/files/system-maintenance-service-freezer.py deleted file mode 100644 index 764033d1..00000000 --- a/roles/system-maintenance-service-freezer/files/system-maintenance-service-freezer.py +++ /dev/null @@ -1,187 +0,0 @@ -import argparse -import subprocess -import time -import os -from datetime import datetime - -# Global variable definition -BREAK_TIME_SECONDS = 5 -FREEZER_SERVICES_PREFIX="system-maintenance-service-" - -class AttemptException(Exception): - """A custom exception for maximum number of attempts.""" - pass - -def parse_time_to_seconds(time_str): - """ - Convert a time string (e.g., '1h', '30min', '45s') to seconds. - """ - units = {"s": 1, "min": 60, "h": 3600} - if time_str[-3:] in units: - number, unit = time_str[:-3], time_str[-3:] - elif time_str[-2:] in units: - number, unit = time_str[:-2], time_str[-2:] - elif time_str[-1:] in units: - number, unit = time_str[:-1], time_str[-1:] - else: - raise ValueError("Invalid time unit") - return int(number) * units[unit] - -def service_file_exists(service_name, service_type="service"): - """ - Check if a systemd service file of a given type exists for a service. - """ - path = "/etc/systemd/system/" - service_file_name = f"{service_name}.{service_type}" - full_path = os.path.join(path, service_file_name) - - # Debug output for checking the service file existence - print(f"Checking {full_path}") - return os.path.isfile(full_path) - -def check_service_active(service_name): - """ - Check if a systemd service is currently active or activating. - """ - result = subprocess.run(['systemctl', 'is-active', service_name], stdout=subprocess.PIPE) - service_status = result.stdout.decode('utf-8').strip() - is_active = service_status in ['active', 'activating'] - print(f"Service {service_name} is {'active' if is_active else 'not active'}.") - return is_active - -def check_any_service_active(services): - """ - Check if any service in a given list is active or activating. - """ - return any(check_service_active(service) for service in services) - -def manage_timer(service, action): - """ - Manage a systemd timer for a service. - action can be 'start' or 'stop'. - """ - if action not in ['start', 'stop']: - raise ValueError("Invalid action specified for manage_timer") - - timer_name = f"{service}.timer" - try: - subprocess.run(['systemctl', action, timer_name], check=True) - if action == 'start': - subprocess.run(['systemctl', 'enable', timer_name], check=True) - elif action == 'stop': - subprocess.run(['systemctl', 'disable', timer_name], check=True) - print(f"{timer_name} {action}ed and {'enabled' if action == 'start' else 'disabled'}.") - except subprocess.CalledProcessError as e: - print(f"Error managing timer {timer_name}: {e}") - exit(1) - -def stop_timer(service): - """ - Stop and disable a systemd timer for a service if it exists. - """ - if service == f"{FREEZER_SERVICES_PREFIX}defrost": - print(f"Ignoring {service}. It's the initializer of freezer.") - if service_file_exists(service, "timer"): - manage_timer(service, 'stop') - else: - print(f"Timer {service}.timer does not exist.") - -def filter_services(services, ignored_services): - """ - Filter out services that are in the ignored_services list from services list. - """ - return [service for service in services if service not in ignored_services] - -def stop_all_timers(services): - """ - Stop and disable timers for all services in a given list. - """ - for service in services: - stop_timer(service) - -def wait_for_all_services_to_stop(filtered_services, max_attempts, attempt): - """ - Wait until all services in the list have stopped, with a maximum number of attempts. - """ - for service in filtered_services: - while check_service_active(service): - attempt += 1 - if attempt > max_attempts: - raise AttemptException(f"Maximum attempts ({max_attempts}) reached. Exiting.") - print(f"{datetime.now().isoformat()}#{attempt}/{max_attempts}: Waiting for {BREAK_TIME_SECONDS} seconds for {service} to stop...") - time.sleep(BREAK_TIME_SECONDS) - return attempt - -def freeze(filtered_services, timeout_sec): - """ - Freeze services by stopping them and their timers, waiting up to a timeout. - """ - attempt = 0 - max_attempts = get_max_attempts(timeout_sec) - - while check_any_service_active(filtered_services): - stop_all_timers(filtered_services) - attempt = wait_for_all_services_to_stop(filtered_services, max_attempts, attempt) - print("All required services have stopped.") - -def get_max_attempts(timeout_sec): - return timeout_sec // BREAK_TIME_SECONDS - -def defrost(filtered_services,timeout_sec): - """ - Defrost services by starting and enabling their timers. - """ - running_service = f"{FREEZER_SERVICES_PREFIX}defrost" - attempt = 0 - max_attempts = get_max_attempts(timeout_sec) - try: - wait_for_all_services_to_stop(filtered_services, max_attempts, attempt) - except AttemptException as e: - print(e) - print("Defrosting was not possible. The execution of other services took to long.") - manage_timer(running_service, "stop") - exit(0) - - for service in filtered_services + [running_service]: - print(f"Unfreezing: {service}") - if service_file_exists(service, "timer"): - manage_timer(service, "start") - else: - print("No timer to activate for service.") - print("All required services are started.") - -def main(services, ignored_services, action, timeout_sec): - """ - Main function to process the command-line arguments and perform actions. - """ - - # Ignoring the current running service - running_service=f"{FREEZER_SERVICES_PREFIX}{action}" - if running_service not in ignored_services: - ignored_services.append(running_service) - - filtered_services = filter_services(services, ignored_services) - print(f"Services to handle: {services}") - print(f"Services to ignore: {ignored_services}") - print(f"Services filtered: {filtered_services}") - - if action == 'freeze': - print("Freezing services.") - freeze(filtered_services, timeout_sec) - elif action == 'defrost': - print("Unfreezing services.") - defrost(filtered_services, timeout_sec) - print("Overview:") - subprocess.run(['systemctl', 'list-timers']) - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Freezes and defrosts systemd services and timers.') - parser.add_argument('action', choices=['freeze', 'defrost'], help='Action to perform: freeze or defrost services.') - parser.add_argument('services', nargs='+', help='List of services to apply the action to.') - parser.add_argument('--ignore', nargs='*', help='List of services to ignore in the action.', default=[]) - parser.add_argument('--timeout', help='Timeout for freezer actions (e.g., 1h, 30min, 45s).', default='1min') - args = parser.parse_args() - services = args.services - ignored_services = args.ignore if args.ignore else [] - timeout_seconds = parse_time_to_seconds(args.timeout) - main(services, ignored_services, args.action, timeout_seconds) diff --git a/roles/system-maintenance-service-freezer/handlers/main.yml b/roles/system-maintenance-service-freezer/handlers/main.yml deleted file mode 100644 index bb1a1fc9..00000000 --- a/roles/system-maintenance-service-freezer/handlers/main.yml +++ /dev/null @@ -1,16 +0,0 @@ -- name: "restart system-maintenance-service-defrost.timer" - systemd: - name: system-maintenance-service-defrost.timer - state: restarted - enabled: yes - daemon_reload: yes - -- name: "reload system-maintenance-service-freeze.service" - systemd: - name: system-maintenance-service-freeze.service - daemon_reload: yes - -- name: "reload system-maintenance-service-defrost.service" - systemd: - name: system-maintenance-service-defrost.service - daemon_reload: yes \ No newline at end of file diff --git a/roles/system-maintenance-service-freezer/meta/main.yml b/roles/system-maintenance-service-freezer/meta/main.yml deleted file mode 100644 index bebc0b50..00000000 --- a/roles/system-maintenance-service-freezer/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - role: systemd-notifier diff --git a/roles/system-maintenance-service-freezer/tasks/main.yml b/roles/system-maintenance-service-freezer/tasks/main.yml deleted file mode 100644 index dc15e03d..00000000 --- a/roles/system-maintenance-service-freezer/tasks/main.yml +++ /dev/null @@ -1,44 +0,0 @@ ---- -- name: create {{path_system_maintenance_service_freezer_script}} - copy: - src: system-maintenance-service-freezer.py - dest: "{{path_system_maintenance_service_freezer_script}}" - when: run_once_system_maintenance_service_freeze is not defined - -- name: Configure system-maintenance-service for each action - loop: - - freeze - - defrost - template: - src: system-maintenance-service-freezer.service.j2 - dest: "/etc/systemd/system/system-maintenance-service-{{ item }}.service" - notify: "reload system-maintenance-service-{{ item }}.service" - when: run_once_system_maintenance_service_freeze is not defined - -- name: "restart system-maintenance-service.service" - systemd: - name: system-maintenance-service-{{system_maintenance_service_freeze_action}}.service - state: restarted - enabled: yes - daemon_reload: yes - when: maintenance_service_freeze_action_last is not defined or maintenance_service_freeze_action_last != system_maintenance_service_freeze_action - -- name: create system-maintenance-service-defrost.timer - template: - src: system-maintenance-service-defrost.timer.j2 - dest: "/etc/systemd/system/system-maintenance-service-defrost.timer" - register: system_maintenance_service_defrost_timer - changed_when: system_maintenance_service_defrost_timer.changed or activate_all_timers | bool - notify: restart system-maintenance-service-defrost.timer - when: run_once_system_maintenance_service_freeze is not defined - -## Runtime Variable Setting - -- name: run the system_maintenance_service_freezer tasks once - set_fact: - run_once_system_maintenance_service_freeze: true - when: run_once_system_maintenance_service_freeze is not defined - -- name: "set variable to prevent loading when action status didn't change" - set_fact: - maintenance_service_freeze_action_last: "{{system_maintenance_service_freeze_action}}" diff --git a/roles/system-maintenance-service-freezer/templates/system-maintenance-service-defrost.timer.j2 b/roles/system-maintenance-service-freezer/templates/system-maintenance-service-defrost.timer.j2 deleted file mode 100644 index 3885217e..00000000 --- a/roles/system-maintenance-service-freezer/templates/system-maintenance-service-defrost.timer.j2 +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=starts system-maintenance-service-defrost.service - -[Timer] -OnCalendar={{on_calendar_defrost}} -RandomizedDelaySec={{randomized_delay_sec}} -Persistent=false - -[Install] -WantedBy=timers.target diff --git a/roles/system-maintenance-service-freezer/templates/system-maintenance-service-freezer.service.j2 b/roles/system-maintenance-service-freezer/templates/system-maintenance-service-freezer.service.j2 deleted file mode 100644 index aaa4e68b..00000000 --- a/roles/system-maintenance-service-freezer/templates/system-maintenance-service-freezer.service.j2 +++ /dev/null @@ -1,7 +0,0 @@ -[Unit] -Description={{item}} systemctl maintenance services -OnFailure=systemd-notifier@%n.service - -[Service] -Type=oneshot -ExecStart=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} {{item}} {{ system_maintenance_services | join(' ') }} --timeout "{{system_maintenance_timeout_freezer_action}}"' \ No newline at end of file diff --git a/roles/update-docker/meta/main.yml b/roles/update-docker/meta/main.yml index 13944fb9..6f898c6d 100644 --- a/roles/update-docker/meta/main.yml +++ b/roles/update-docker/meta/main.yml @@ -1,2 +1,2 @@ dependencies: - - system-maintenance-service-freezer + - system-maintenance-lock diff --git a/roles/update-docker/templates/update-docker.service.j2 b/roles/update-docker/templates/update-docker.service.j2 index aaf1cc33..7f1638a7 100644 --- a/roles/update-docker/templates/update-docker.service.j2 +++ b/roles/update-docker/templates/update-docker.service.j2 @@ -4,5 +4,5 @@ OnFailure=systemd-notifier@%n.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services | join(' ') }} update-docker --timeout "{{system_maintenance_timeout_heal_docker}}"' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_lock_script }} {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services | join(' ') }} update-docker --timeout "{{sytem_maintenance_lock_timeoutheal_docker}}"' ExecStart=/bin/sh -c '/usr/bin/python {{update_docker_script}} {{path_docker_compose_instances}}' \ No newline at end of file