Added programm to restart unhealthy docker compose containers

This commit is contained in:
Kevin Veen-Birkenbach 2023-03-26 20:36:36 +02:00
parent 95c1e9e942
commit ec6e4c1a9f
8 changed files with 105 additions and 0 deletions

View File

@ -0,0 +1,2 @@
# native-docker-compose-restart-unhealthy
docker-compose restart for containers which are unhealty or excited

View File

@ -0,0 +1,9 @@
[Unit]
Description=starts docker-compose-restart-unhealthy.service
[Timer]
OnBootSec=1min
OnUnitActiveSec=1h
[Install]
WantedBy=timers.target

View File

@ -0,0 +1,12 @@
- name: "restart docker-compose-restart-unhealthy.service"
systemd:
name: docker-compose-restart-unhealthy.service
state: restarted
enabled: yes
daemon_reload: yes
- name: "restart docker-compose-restart-unhealthy.timer"
systemd:
name: docker-compose-restart-unhealthy.timer
state: restarted
enabled: yes
daemon_reload: yes

View File

@ -0,0 +1,19 @@
- name: "create {{docker_docker_compose_restart_unhealthy}}"
file:
path: "{{docker_docker_compose_restart_unhealthy}}"
state: directory
mode: 0755
- name: create docker-compose-restart-unhealthy.py
template: src=docker-compose-restart-unhealthy.py.j2 dest={{docker_docker_compose_restart_unhealthy}}docker-compose-restart-unhealthy.py
notify: restart docker-compose-restart-unhealthy.service
- name: create docker-compose-restart-unhealthy.service
template: src=docker-health-check.service.j2 dest=/etc/systemd/system/docker-compose-restart-unhealthy.service
notify: restart docker-compose-restart-unhealthy.service
- name: create docker-compose-restart-unhealthy.timer
copy:
src: docker-compose-restart-unhealthy.timer
dest: "/etc/systemd/system/docker-compose-restart-unhealthy.timer"
notify: restart docker-compose-restart-unhealthy.timer

View File

@ -0,0 +1,54 @@
#!/bin/python
#
# restart docker-compose configurations who have exited or unhealthy containers
#
import subprocess
import time
def bash(command):
print(command)
process = subprocess.Popen([command], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
out, err = process.communicate()
stdout = out.splitlines()
output = []
for line in stdout:
output.append(line.decode("utf-8"))
if process.wait() > bool(0):
print(command, out, err)
raise Exception("Exitcode is greater then 0")
return output
def list_to_string(list):
return str(' '.join(list))
def print_bash(command):
output = bash(command)
print(list_to_string(output))
return output
waiting_time=1800
backup_running=True
while backup_running:
print("stop program for " + str(waiting_time) + "seconds.")
time.sleep(waiting_time)
try:
bash("systemctl is-active --quiet docker-volume-backup.service")
print("backup is running.")
except:
backup_running=False
print("no backup is running.")
unhealthy_container_names=print_bash('docker ps --filter health=unhealthy --format \'{{.Names}}\'')
exited_container_names=print_bash('docker ps --filter status=exited --format \'{{.Names}}\'')
failed_containers=unhealthy_container_names + exited_container_names
unfiltered_failed_docker_compose_repositories=[]
for failed_container in failed_containers:
unfiltered_failed_docker_compose_repositories.append(failed_container.split('-')[0])
filtered_failed_docker_compose_repositories=list(dict.fromkeys(unfiltered_failed_docker_compose_repositories))
for filtered_failed_docker_compose_repository in filtered_failed_docker_compose_repositories:
print("restarting unhealthy container: " + filtered_failed_docker_compose_repository)
print_bash('cd /home/administrator/docker-compose/' + filtered_failed_docker_compose_repository + '/ && docker-compose restart')
print("finished restart procedure.")

View File

@ -0,0 +1,7 @@
[Unit]
Description=restart unhealthy docker containers
OnFailure=systemd-email@%n.service
[Service]
Type=oneshot
ExecStart=/bin/bash {{docker_docker_compose_restart_unhealthy}}docker-compose-restart-unhealthy.py

View File

@ -0,0 +1 @@
docker_docker_compose_restart_unhealthy: "/home/administrator/scripts/docker-compose-restart-unhealthy/"

View File

@ -2,3 +2,4 @@ dependencies:
- native-docker-volume-backup - native-docker-volume-backup
- native-user-administrator - native-user-administrator
- native-docker-health-check - native-docker-health-check
- native-docker-compose-restart-unhealthy