From ec6e4c1a9fb8a89b9709bfeaccc9beaa8f5d66e6 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sun, 26 Mar 2023 20:36:36 +0200 Subject: [PATCH] Added programm to restart unhealthy docker compose containers --- .../README.md | 2 + .../docker-compose-restart-unhealthy.timer | 9 ++++ .../handlers/main.yml | 12 +++++ .../tasks/main.yml | 19 +++++++ .../docker-compose-restart-unhealthy.py.j2 | 54 +++++++++++++++++++ ...ocker-compose-restart-unhealthy.service.j2 | 7 +++ .../vars/main.yml | 1 + roles/native-docker/meta/main.yml | 1 + 8 files changed, 105 insertions(+) create mode 100644 roles/native-docker-compose-restart-unhealthy/README.md create mode 100644 roles/native-docker-compose-restart-unhealthy/files/docker-compose-restart-unhealthy.timer create mode 100644 roles/native-docker-compose-restart-unhealthy/handlers/main.yml create mode 100644 roles/native-docker-compose-restart-unhealthy/tasks/main.yml create mode 100644 roles/native-docker-compose-restart-unhealthy/templates/docker-compose-restart-unhealthy.py.j2 create mode 100644 roles/native-docker-compose-restart-unhealthy/templates/docker-compose-restart-unhealthy.service.j2 create mode 100644 roles/native-docker-compose-restart-unhealthy/vars/main.yml diff --git a/roles/native-docker-compose-restart-unhealthy/README.md b/roles/native-docker-compose-restart-unhealthy/README.md new file mode 100644 index 00000000..8bf0d0fe --- /dev/null +++ b/roles/native-docker-compose-restart-unhealthy/README.md @@ -0,0 +1,2 @@ +# native-docker-compose-restart-unhealthy +docker-compose restart for containers which are unhealty or excited \ No newline at end of file diff --git a/roles/native-docker-compose-restart-unhealthy/files/docker-compose-restart-unhealthy.timer b/roles/native-docker-compose-restart-unhealthy/files/docker-compose-restart-unhealthy.timer new file mode 100644 index 00000000..6d8ab9ea --- /dev/null +++ b/roles/native-docker-compose-restart-unhealthy/files/docker-compose-restart-unhealthy.timer @@ -0,0 +1,9 @@ +[Unit] +Description=starts docker-compose-restart-unhealthy.service + +[Timer] +OnBootSec=1min +OnUnitActiveSec=1h + +[Install] +WantedBy=timers.target diff --git a/roles/native-docker-compose-restart-unhealthy/handlers/main.yml b/roles/native-docker-compose-restart-unhealthy/handlers/main.yml new file mode 100644 index 00000000..37c62838 --- /dev/null +++ b/roles/native-docker-compose-restart-unhealthy/handlers/main.yml @@ -0,0 +1,12 @@ +- name: "restart docker-compose-restart-unhealthy.service" + systemd: + name: docker-compose-restart-unhealthy.service + state: restarted + enabled: yes + daemon_reload: yes +- name: "restart docker-compose-restart-unhealthy.timer" + systemd: + name: docker-compose-restart-unhealthy.timer + state: restarted + enabled: yes + daemon_reload: yes diff --git a/roles/native-docker-compose-restart-unhealthy/tasks/main.yml b/roles/native-docker-compose-restart-unhealthy/tasks/main.yml new file mode 100644 index 00000000..ad1b66ae --- /dev/null +++ b/roles/native-docker-compose-restart-unhealthy/tasks/main.yml @@ -0,0 +1,19 @@ +- name: "create {{docker_docker_compose_restart_unhealthy}}" + file: + path: "{{docker_docker_compose_restart_unhealthy}}" + state: directory + mode: 0755 + +- name: create docker-compose-restart-unhealthy.py + template: src=docker-compose-restart-unhealthy.py.j2 dest={{docker_docker_compose_restart_unhealthy}}docker-compose-restart-unhealthy.py + notify: restart docker-compose-restart-unhealthy.service + +- name: create docker-compose-restart-unhealthy.service + template: src=docker-health-check.service.j2 dest=/etc/systemd/system/docker-compose-restart-unhealthy.service + notify: restart docker-compose-restart-unhealthy.service + +- name: create docker-compose-restart-unhealthy.timer + copy: + src: docker-compose-restart-unhealthy.timer + dest: "/etc/systemd/system/docker-compose-restart-unhealthy.timer" + notify: restart docker-compose-restart-unhealthy.timer \ No newline at end of file diff --git a/roles/native-docker-compose-restart-unhealthy/templates/docker-compose-restart-unhealthy.py.j2 b/roles/native-docker-compose-restart-unhealthy/templates/docker-compose-restart-unhealthy.py.j2 new file mode 100644 index 00000000..2ba4e013 --- /dev/null +++ b/roles/native-docker-compose-restart-unhealthy/templates/docker-compose-restart-unhealthy.py.j2 @@ -0,0 +1,54 @@ +#!/bin/python +# +# restart docker-compose configurations who have exited or unhealthy containers +# +import subprocess +import time + +def bash(command): + print(command) + process = subprocess.Popen([command], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + out, err = process.communicate() + stdout = out.splitlines() + output = [] + for line in stdout: + output.append(line.decode("utf-8")) + if process.wait() > bool(0): + print(command, out, err) + raise Exception("Exitcode is greater then 0") + return output + +def list_to_string(list): + return str(' '.join(list)) + +def print_bash(command): + output = bash(command) + print(list_to_string(output)) + return output + +waiting_time=1800 +backup_running=True +while backup_running: + print("stop program for " + str(waiting_time) + "seconds.") + time.sleep(waiting_time) + try: + bash("systemctl is-active --quiet docker-volume-backup.service") + print("backup is running.") + except: + backup_running=False + print("no backup is running.") + +unhealthy_container_names=print_bash('docker ps --filter health=unhealthy --format \'{{.Names}}\'') +exited_container_names=print_bash('docker ps --filter status=exited --format \'{{.Names}}\'') +failed_containers=unhealthy_container_names + exited_container_names + +unfiltered_failed_docker_compose_repositories=[] +for failed_container in failed_containers: + unfiltered_failed_docker_compose_repositories.append(failed_container.split('-')[0]) + +filtered_failed_docker_compose_repositories=list(dict.fromkeys(unfiltered_failed_docker_compose_repositories)) +for filtered_failed_docker_compose_repository in filtered_failed_docker_compose_repositories: + print("restarting unhealthy container: " + filtered_failed_docker_compose_repository) + print_bash('cd /home/administrator/docker-compose/' + filtered_failed_docker_compose_repository + '/ && docker-compose restart') + +print("finished restart procedure.") \ No newline at end of file diff --git a/roles/native-docker-compose-restart-unhealthy/templates/docker-compose-restart-unhealthy.service.j2 b/roles/native-docker-compose-restart-unhealthy/templates/docker-compose-restart-unhealthy.service.j2 new file mode 100644 index 00000000..4d15bcd4 --- /dev/null +++ b/roles/native-docker-compose-restart-unhealthy/templates/docker-compose-restart-unhealthy.service.j2 @@ -0,0 +1,7 @@ +[Unit] +Description=restart unhealthy docker containers +OnFailure=systemd-email@%n.service + +[Service] +Type=oneshot +ExecStart=/bin/bash {{docker_docker_compose_restart_unhealthy}}docker-compose-restart-unhealthy.py diff --git a/roles/native-docker-compose-restart-unhealthy/vars/main.yml b/roles/native-docker-compose-restart-unhealthy/vars/main.yml new file mode 100644 index 00000000..c1a26e1f --- /dev/null +++ b/roles/native-docker-compose-restart-unhealthy/vars/main.yml @@ -0,0 +1 @@ +docker_docker_compose_restart_unhealthy: "/home/administrator/scripts/docker-compose-restart-unhealthy/" \ No newline at end of file diff --git a/roles/native-docker/meta/main.yml b/roles/native-docker/meta/main.yml index a1abc078..baa65ef5 100644 --- a/roles/native-docker/meta/main.yml +++ b/roles/native-docker/meta/main.yml @@ -2,3 +2,4 @@ dependencies: - native-docker-volume-backup - native-user-administrator - native-docker-health-check + - native-docker-compose-restart-unhealthy