renamed to heal-docker

roles/heal-docker/README.md

@@ -0,0 +1,2 @@
+# heal-docker
+docker-compose restart for containers which are unhealty or excited

roles/heal-docker/files/heal-docker.py

@@ -0,0 +1,55 @@
+#!/bin/python
+#
+# restart docker-compose configurations who have exited or unhealthy containers
+#
+import subprocess
+import time
+
+def bash(command):
+    print(command)
+    process = subprocess.Popen([command], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+    out, err = process.communicate()
+    stdout = out.splitlines()
+    output = []
+    for line in stdout:
+        output.append(line.decode("utf-8"))
+    if process.wait() > bool(0):
+        print(command, out, err)
+        raise Exception("Exitcode is greater then 0")
+    return output
+
+def list_to_string(list):
+    return str(' '.join(list))
+
+def print_bash(command):
+    output = bash(command)
+    print(list_to_string(output))
+    return output
+
+waiting_time=600
+blocker_running=True
+while blocker_running:
+    try: 
+        bash("systemctl is-active --quiet backup-docker-to-local.service")
+        bash("systemctl is-active --quiet update-docker.service")
+        print("backup is running.")
+        print("trying again in  " + str(waiting_time) + " seconds.")
+        time.sleep(waiting_time)
+    except:
+        blocker_running=False
+        print("No blocking service is running.")
+
+unhealthy_container_names=print_bash('docker ps --filter health=unhealthy --format \'{{.Names}}\'')
+exited_container_names=print_bash('docker ps --filter status=exited --format \'{{.Names}}\'')
+failed_containers=unhealthy_container_names + exited_container_names
+
+unfiltered_failed_docker_compose_repositories=[]
+for failed_container in failed_containers:
+    unfiltered_failed_docker_compose_repositories.append(failed_container.split('-')[0])
+
+filtered_failed_docker_compose_repositories=list(dict.fromkeys(unfiltered_failed_docker_compose_repositories))
+for filtered_failed_docker_compose_repository in filtered_failed_docker_compose_repositories:
+    print("restarting unhealthy container: " + filtered_failed_docker_compose_repository)
+    print_bash('cd /home/administrator/docker-compose/' + filtered_failed_docker_compose_repository + '/ && docker-compose restart')
+
+print("finished restart procedure.")

roles/heal-docker/handlers/main.yml

@@ -0,0 +1,12 @@
+- name: "reload heal-docker.service"
+  systemd:
+    name: heal-docker.service
+    state: reloaded
+    enabled: yes
+    daemon_reload: yes
+- name: "restart heal-docker.timer"
+  systemd:
+    name: heal-docker.timer
+    state: restarted
+    enabled: yes
+    daemon_reload: yes

roles/heal-docker/tasks/main.yml

@@ -0,0 +1,22 @@
+- name: "create {{docker_compose_restart_unhealthy}}"
+  file:
+    path: "{{docker_compose_restart_unhealthy}}"
+    state: directory
+    mode: 0755
+
+- name: create heal-docker.py
+  copy:
+    src: heal-docker.py
+    dest: "{{docker_compose_restart_unhealthy}}heal-docker.py"
+
+- name: create heal-docker.service
+  template: 
+    src: heal-docker.service.j2 
+    dest: /etc/systemd/system/heal-docker.service
+  notify: reload heal-docker.service
+
+- name: create heal-docker.timer
+  template:
+    src: heal-docker.timer.j2
+    dest: "/etc/systemd/system/heal-docker.timer"
+  notify: restart heal-docker.timer

roles/heal-docker/templates/heal-docker.service.j2

@@ -0,0 +1,7 @@
+[Unit]
+Description=restart unhealthy docker containers
+OnFailure=systemd-notifier@%n.service
+
+[Service]
+Type=oneshot
+ExecStart=/bin/python {{docker_compose_restart_unhealthy}}heal-docker.py

roles/heal-docker/templates/heal-docker.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=starts heal-docker.service
+
+[Timer]
+OnCalendar={{on_calendar_docker_compose_restart_unhealthy}}
+RandomizedDelaySec={{randomized_delay_sec}}
+Persistent=false
+
+[Install]
+WantedBy=timers.target

roles/heal-docker/vars/main.yml

@@ -0,0 +1 @@
+docker_compose_restart_unhealthy: "{{path_administrator_scripts}}heal-docker/"