Compare commits

...

3 Commits

12 changed files with 34 additions and 28 deletions

View File

@ -1,33 +1,33 @@
# Server Tact Variables # Server Tact Variables
## Ours in which the server is 100% working. Rest of the time is reserved for maintanance ## Ours in which the server is 100% working. Rest of the time is reserved for maintanance
hours_server_awake: "{{ range(9, 24) | list + range(0, 3) | list }}" hours_server_awake: "0..2,9..23"
## Random delay for systemd timers to avoid peak loads. ## Random delay for systemd timers to avoid peak loads.
randomized_delay_sec: "5min" randomized_delay_sec: "5min"
## Schedule for Health Checks ## Schedule for Health Checks
on_calendar_health_btrfs: "*-*-* 00:00:00" on_calendar_health_btrfs: "*-*-* 00:00:00" # Check once per day the btrfs for errors
on_calendar_health_journalctl: "*-*-* 00:00:00" on_calendar_health_journalctl: "*-*-* 00:00:00" # Check once per day the journalctl for errors
on_calendar_health_disc_space: "*-*-* 06,12,18,00:00:00" on_calendar_health_disc_space: "*-*-* 06,12,18,00:00:00" # Check four times per day if there is sufficient disc space
on_calendar_health_docker_container: "*-*-* {{ hours_server_awake | join(',') }}:00:00" on_calendar_health_docker_container: "*-*-* {{ hours_server_awake }}:00:00" # Check once per hour if the docker containers are healthy
on_calendar_health_docker_volumes: "*-*-* {{ hours_server_awake | join(',') }}:15:00" on_calendar_health_docker_volumes: "*-*-* {{ hours_server_awake }}:15:00" # Check once per hour if the docker volumes are healthy
on_calendar_health_nginx: "*-*-* {{ hours_server_awake | join(',') }}:45:00" on_calendar_health_nginx: "*-*-* {{ hours_server_awake }}:45:00"
## Schedule for Cleanup Tasks ## Schedule for Cleanup Tasks
on_calendar_cleanup_backups: "*-*-* 06,12,18,00:30:00" on_calendar_cleanup_backups: "*-*-* 00,06,12,18:30:00" # Cleanup backups every 6 hours, MUST be called before disc space cleanup
on_calendar_cleanup_disc_space: "*-*-* 07,13,19,01:30:00" on_calendar_cleanup_disc_space: "*-*-* 07,13,19,01:30:00" # Cleanup disc space every 6 hours
## Schedule for Backup Tasks ## Schedule for Backup Tasks
on_calendar_backup_docker_to_local: "*-*-* 03:30:00" on_calendar_backup_docker_to_local: "*-*-* 03:30:00"
on_calendar_backup_remote_to_local: "*-*-* 21:30:00" on_calendar_backup_remote_to_local: "*-*-* 21:30:00"
## Schedule for Maintenance Tasks ## Schedule for Maintenance Tasks
on_calendar_heal_docker: "*-*-* {{ hours_server_awake | join(',') }}:30:00" on_calendar_heal_docker: "*-*-* {{ hours_server_awake }}:30:00" # Heal unhealthy docker instances once per hour
on_calendar_defrost: "*-*-* *:00,15,30,45:00" on_calendar_defrost: "*:0/5" # Defrost every 5min
on_calendar_renew_lets_encrypt_certificates: "*-*-* 12,00:30:00" on_calendar_renew_lets_encrypt_certificates: "*-*-* 12,00:30:00" # Renew Mailu certificates twice per day
on_calendar_deploy_mailu_certificates: "*-*-* 13,01:30:00" on_calendar_deploy_mailu_certificates: "*-*-* 13,01:30:00" # Deploy Mailu certificates twice per day
on_calendar_msi_keyboard_color: "*-*-* *:*:00" on_calendar_msi_keyboard_color: "*-*-* *:*:00" # Change the keyboard color every minute
# Storage Space-Related Configurations # Storage Space-Related Configurations
@ -133,7 +133,6 @@ domain_matrix: "chat.{{top_domain}}"
domain_mediawiki: "wiki.{{top_domain}}" domain_mediawiki: "wiki.{{top_domain}}"
domain_nextcloud: "cloud.{{top_domain}}" domain_nextcloud: "cloud.{{top_domain}}"
domain_pixelfed: "picture.{{top_domain}}" domain_pixelfed: "picture.{{top_domain}}"
domains_pixelfed: []
domain_peertube: "video.{{top_domain}}" domain_peertube: "video.{{top_domain}}"
domains_peertube: [] domains_peertube: []
domain_roulette: "roulette.{{top_domain}}" domain_roulette: "roulette.{{top_domain}}"

View File

@ -5,4 +5,4 @@ OnFailure=systemd-notifier@%n.service cleanup-failed-docker-backups.service
[Service] [Service]
Type=oneshot Type=oneshot
ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_backup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"' ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_backup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"'
ExecStart=/bin/sh -c '/usr/bin/python {{backup_docker_to_local_folder}}backup-docker-to-local.py && systemctl start system-maintenance-service-defrost.service' ExecStart=/bin/sh -c '/usr/bin/python {{backup_docker_to_local_folder}}backup-docker-to-local.py'

View File

@ -5,4 +5,4 @@ OnFailure=systemd-notifier@%n.service cleanup-failed-docker-backups.service
[Service] [Service]
Type=oneshot Type=oneshot
ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_backup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"' ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_backup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"'
ExecStart=/bin/sh -c '/usr/bin/bash {{docker_backup_remote_to_local_folder}}backups-remote-to-local.sh && systemctl start system-maintenance-service-defrost.service' ExecStart=/bin/sh -c '/usr/bin/bash {{docker_backup_remote_to_local_folder}}backups-remote-to-local.sh'

View File

@ -5,4 +5,4 @@ OnFailure=systemd-notifier@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"' ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"'
ExecStart=/bin/sh -c '/usr/bin/python {{docker_cleanup_backups}}cleanup-backups.py --backups-folder-path {{backups_folder_path}} --maximum-backup-size-percent {{size_percent_maximum_backup}} && systemctl start system-maintenance-service-defrost.service' ExecStart=/bin/sh -c '/usr/bin/python {{docker_cleanup_backups}}cleanup-backups.py --backups-folder-path {{backups_folder_path}} --maximum-backup-size-percent {{size_percent_maximum_backup}}'

View File

@ -5,4 +5,4 @@ OnFailure=systemd-notifier@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"' ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"'
ExecStart=/bin/sh -c '/bin/bash {{cleanup_disc_space_folder}}cleanup-disc-space.sh {{size_percent_cleanup_disc_space}} && systemctl start system-maintenance-service-defrost.service' ExecStart=/bin/sh -c '/bin/bash {{cleanup_disc_space_folder}}cleanup-disc-space.sh {{size_percent_cleanup_disc_space}}'

View File

@ -5,4 +5,4 @@ OnFailure=systemd-notifier@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"' ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} --timeout "{{system_maintenance_timeout_backup_services}}"'
ExecStart=/bin/sh -c '/usr/bin/yes | /usr/bin/bash {{backup_docker_to_local_cleanup_folder}}cleanup.sh {{backup_docker_to_local_cleanup_machine_id}} {{backup_docker_to_local_cleanup_trigger_directory}} && systemctl start system-maintenance-service-defrost.service' ExecStart=/bin/sh -c '/usr/bin/yes | /usr/bin/bash {{backup_docker_to_local_cleanup_folder}}cleanup.sh {{backup_docker_to_local_cleanup_machine_id}} {{backup_docker_to_local_cleanup_trigger_directory}}'

View File

@ -1,10 +1,7 @@
--- ---
- name: "include tasks nginx-docker-proxy-domain.yml" - name: "include tasks nginx-docker-proxy-domain.yml"
include_tasks: nginx-docker-proxy-domain.yml include_tasks: nginx-docker-proxy-domain.yml
loop: "{{ domains }}"
loop_control:
loop_var: domain
- name: "create {{docker_compose_instance_directory}}" - name: "create {{docker_compose_instance_directory}}"
file: file:
path: "{{docker_compose_instance_directory}}" path: "{{docker_compose_instance_directory}}"

View File

@ -5,4 +5,4 @@ OnFailure=systemd-notifier@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} heal-docker --timeout "{{system_maintenance_timeout_heal_docker}}"' ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services| join(' ') }} heal-docker --timeout "{{system_maintenance_timeout_heal_docker}}"'
ExecStart=/bin/sh -c '/bin/python {{heal_docker}}heal-docker.py && systemctl start system-maintenance-service-defrost.service' ExecStart=/bin/sh -c '/bin/python {{heal_docker}}heal-docker.py'

View File

@ -3,6 +3,12 @@
## Overview ## Overview
This Ansible role is designed to manage system services through freezing (disabling) and defrosting (enabling) actions. It automates the process of managing crucial system services, especially useful for maintenance tasks like backups, cleanups, and updates. This Ansible role is designed to manage system services through freezing (disabling) and defrosting (enabling) actions. It automates the process of managing crucial system services, especially useful for maintenance tasks like backups, cleanups, and updates.
## Monitoring
To monitor the sucess of the script and the status of systemctl timers execute:
```bash
watch -n 2 systemctl list-timers
```
## Role Variables ## Role Variables
- `system_maintenance_services`: List of services to be managed by this role. - `system_maintenance_services`: List of services to be managed by this role.

View File

@ -1,3 +1,9 @@
- name: "start backup-docker-to-local.service"
systemd:
name: backup-docker-to-local.service
state: started
when: force_backup_before_update | bool
- name: create {{update_docker_script}} - name: create {{update_docker_script}}
copy: copy:
src: update-docker.py src: update-docker.py

View File

@ -4,6 +4,5 @@ OnFailure=systemd-notifier@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
{% if force_backup_before_update | bool %}ExecStartPre=/bin/sh -c 'systemctl start backup-docker-to-local.service'{% endif %}
ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services | join(' ') }} update-docker --timeout "{{system_maintenance_timeout_heal_docker}}"' ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_maintenance_service_freezer_script }} freeze {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services | join(' ') }} update-docker --timeout "{{system_maintenance_timeout_heal_docker}}"'
ExecStart=/bin/sh -c '/usr/bin/python {{update_docker_script}} {{path_docker_compose_instances}} && systemctl start system-maintenance-service-defrost.service' ExecStart=/bin/sh -c '/usr/bin/python {{update_docker_script}} {{path_docker_compose_instances}}'

View File

@ -105,7 +105,6 @@
- role: docker-pixelfed - role: docker-pixelfed
vars: vars:
domain: "{{domain_pixelfed}}" domain: "{{domain_pixelfed}}"
domains: "{{ [domain] + domains_pixelfed }}"
http_port: 8010 http_port: 8010
- name: setup peertube hosts - name: setup peertube hosts