From 85816d702af61043ebc5bbc80c75c285bc021805 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sun, 1 Dec 2024 15:02:09 +0100 Subject: [PATCH] Added auto restart role for docker --- group_vars/all | 2 + roles/docker/meta/main.yml | 1 + roles/restart-docker/README.md | 47 +++++++++++++++++++ roles/restart-docker/files/restart-docker.py | 34 ++++++++++++++ roles/restart-docker/handlers/main.yml | 5 ++ roles/restart-docker/meta/main.yml | 2 + roles/restart-docker/tasks/main.yml | 37 +++++++++++++++ .../templates/restart-docker.service.j2 | 8 ++++ roles/restart-docker/vars/main.yml | 2 + 9 files changed, 138 insertions(+) create mode 100644 roles/restart-docker/README.md create mode 100644 roles/restart-docker/files/restart-docker.py create mode 100644 roles/restart-docker/handlers/main.yml create mode 100644 roles/restart-docker/meta/main.yml create mode 100644 roles/restart-docker/tasks/main.yml create mode 100644 roles/restart-docker/templates/restart-docker.service.j2 create mode 100644 roles/restart-docker/vars/main.yml diff --git a/group_vars/all b/group_vars/all index bd583374..6b8db5d8 100644 --- a/group_vars/all +++ b/group_vars/all @@ -50,6 +50,7 @@ on_calendar_deploy_mailu_certificates: "*-*-* 13,01:30:00" on_calendar_msi_keyboard_color: "*-*-* *:*:00" # Change the keyboard color every minute on_calendar_cleanup_failed_docker: "*-*-* 12:00:00" # Clean up failed docker backups every noon on_calendar_btrfs_auto_balancer: "Sat *-*-01..07 00:00:00" # Execute btrfs auto balancer every first Saturday of a month +on_calendar_restart_docker: "Sun *-*-* 08:00:00" # Restart docker instances every Sunday at 8:00 AM # Storage Space-Related Configurations size_percent_maximum_backup: 75 # Maximum storage space in percent for backups @@ -98,6 +99,7 @@ system_maintenance_manipulation_services: - "heal-docker" - "update-docker" - "system-storage-optimizer" + - "restart-docker" ## Total System Maintenance Services system_maintenance_services: "{{ system_maintenance_backup_services + system_maintenance_cleanup_services + system_maintenance_manipulation_services }}" diff --git a/roles/docker/meta/main.yml b/roles/docker/meta/main.yml index 0a837198..d8d8f830 100644 --- a/roles/docker/meta/main.yml +++ b/roles/docker/meta/main.yml @@ -4,3 +4,4 @@ dependencies: - health-docker-container - health-docker-volumes - heal-docker + - restart-docker diff --git a/roles/restart-docker/README.md b/roles/restart-docker/README.md new file mode 100644 index 00000000..a529fa33 --- /dev/null +++ b/roles/restart-docker/README.md @@ -0,0 +1,47 @@ +# Docker Auto Restart + +This role was implemented to address the classic issue: ["Have you tried turning it off and on again?"](https://www.youtube.com/watch?v=rksCTVFtjM4). The problem initially arose with the `fetchmail` container in [Mailu](../roles/docker/mailu), which fails if only some containers, and not the full docker-compose composition, are restarted. + +## Overview + +This role automates the restart process for all `docker-compose` instances within a specified directory. It ensures consistent restarts of services while avoiding issues caused by partial restarts. + +## Features +- Automatically detects and restarts `docker-compose` instances in the given directory. +- Uses a Python script to perform service restarts with `docker-compose restart`. +- Integrates with `systemd` for scheduled or manual execution. +- Designed for idempotency and ease of integration. + +## Requirements +- `docker-compose` must be installed on the target system. +- Python 3.x is required to execute the provided script. +- This role depends on the `system-maintenance-lock` role for handling system-wide locking during restarts. + +## Installation +1. Clone or include this role in your Ansible project. +2. Define the required variables in your playbook or inventory: + ```yaml + path_administrator_scripts: "/path/to/administrator/scripts/" + restart_docker_volumes_folder: "/path/to/restart/volumes/" + on_calendar_restart_dockers: "daily" + ``` + +## Usage +Include this role in your playbook: +```yaml +- hosts: all + roles: + - docker-auto-restart +``` + +The role will: +1. Set up the necessary directories and scripts. +2. Configure a `systemd` service to restart docker-compose instances. +3. Optionally schedule restarts via a systemd timer. + +## Acknowledgments +This role was developed with the assistance of [ChatGPT](https://openai.com/chatgpt), including insights and optimizations from this [conversation](https://chatgpt.com/share/674c6870-fcc4-800f-a19e-b20621b24317). Special thanks for providing guidance on error handling, Ansible best practices, and Python integration. + +--- + +Feel free to contribute or provide feedback via the [repository issues page](https://github.com/kevinveenbirkenbach/cymais/issues). \ No newline at end of file diff --git a/roles/restart-docker/files/restart-docker.py b/roles/restart-docker/files/restart-docker.py new file mode 100644 index 00000000..0c16632c --- /dev/null +++ b/roles/restart-docker/files/restart-docker.py @@ -0,0 +1,34 @@ +import os +import sys +import subprocess + +def restart_docker_services(dir_path): + """ + Restart docker-compose services in the given directory. + """ + try: + print(f"Restarting docker-compose services in: {dir_path}") + subprocess.run(["docker-compose", "restart"], cwd=dir_path, check=True) + print(f"Services restarted successfully in: {dir_path}") + except subprocess.CalledProcessError as e: + print(f"Error restarting services in {dir_path}: {e}") + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Please provide the path to the parent directory as a parameter.") + sys.exit(1) + + parent_directory = sys.argv[1] + + for dir_entry in os.scandir(parent_directory): + if dir_entry.is_dir(): + dir_path = dir_entry.path + print(f"Checking directory: {dir_path}") + + docker_compose_file = os.path.join(dir_path, "docker-compose.yml") + + if os.path.isfile(docker_compose_file): + print(f"Found docker-compose.yml in {dir_path}. Restarting services...") + restart_docker_services(dir_path) + else: + print(f"No docker-compose.yml found in {dir_path}. Skipping.") \ No newline at end of file diff --git a/roles/restart-docker/handlers/main.yml b/roles/restart-docker/handlers/main.yml new file mode 100644 index 00000000..5209f53b --- /dev/null +++ b/roles/restart-docker/handlers/main.yml @@ -0,0 +1,5 @@ +- name: "reload restart-docker.cymais.service" + systemd: + name: restart-docker.cymais.service + enabled: yes + daemon_reload: yes diff --git a/roles/restart-docker/meta/main.yml b/roles/restart-docker/meta/main.yml new file mode 100644 index 00000000..6f898c6d --- /dev/null +++ b/roles/restart-docker/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - system-maintenance-lock diff --git a/roles/restart-docker/tasks/main.yml b/roles/restart-docker/tasks/main.yml new file mode 100644 index 00000000..3f2087c6 --- /dev/null +++ b/roles/restart-docker/tasks/main.yml @@ -0,0 +1,37 @@ +- block: + - name: "create {{restart_docker_volumes_folder}}" + file: + path: "{{restart_docker_volumes_folder}}" + state: directory + mode: 0755 + + - name: create {{restart_docker_script}} + copy: + src: restart-docker.py + dest: "{{restart_docker_script}}" + when: run_once_restart_docker is not defined + + - name: configure restart-docker.cymais.service + template: + src: restart-docker.service.j2 + dest: /etc/systemd/system/restart-docker.cymais.service + notify: "reload restart-docker.cymais.service" + when: run_once_restart_docker is not defined + + - name: set service_name to the name of the current role + set_fact: + service_name: "{{ role_name }}" + when: run_once_restart_docker is not defined +when: run_once_restart_docker is not defined + +- name: "include role for systemd-timer for {{service_name}}" + include_role: + name: systemd-timer + vars: + on_calendar: "{{on_calendar_restart_docker}}" + when: run_once_restart_docker is not defined + +- name: run the restart_docker_volumes tasks once + set_fact: + run_once_restart_docker: true + when: run_once_restart_docker is not defined \ No newline at end of file diff --git a/roles/restart-docker/templates/restart-docker.service.j2 b/roles/restart-docker/templates/restart-docker.service.j2 new file mode 100644 index 00000000..c6121921 --- /dev/null +++ b/roles/restart-docker/templates/restart-docker.service.j2 @@ -0,0 +1,8 @@ +[Unit] +Description=Restart Docker Instances +OnFailure=systemd-notifier.cymais@%n.service + +[Service] +Type=oneshot +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_lock_script }} {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_cleanup_services | join(' ') }} restart-docker --timeout "{{system_maintenance_lock_timeout_heal_docker}}"' +ExecStart=/bin/sh -c '/usr/bin/python {{update_docker_script}} {{path_docker_compose_instances}}' \ No newline at end of file diff --git a/roles/restart-docker/vars/main.yml b/roles/restart-docker/vars/main.yml new file mode 100644 index 00000000..68e5e63b --- /dev/null +++ b/roles/restart-docker/vars/main.yml @@ -0,0 +1,2 @@ +restart_docker_folder: "{{path_administrator_scripts}}restart-docker/" +restart_docker_script: "{{restart_docker_folder}}restart-docker.py" \ No newline at end of file