Another big round of refactoring and cleaning...

roles/sys-opt-ssd-hdd/README.md

@@ -0,0 +1,29 @@
+# Storage Optimizer
+
+## Description
+
+This role optimizes storage allocation for Docker volumes by migrating volumes between SSD (rapid storage) and HDD (mass storage) based on container image types. It creates symbolic links to maintain consistent storage paths after migration.
+
+## Overview
+
+The role performs the following tasks:
+- Migrates Docker volumes with database workloads to rapid storage (SSD) for improved performance.
+- Moves non-database Docker volumes to mass storage (HDD) to optimize storage usage.
+- Manages container stopping and restarting during the migration process.
+- Creates symbolic links to preserve consistent file paths.
+
+## Purpose
+
+The primary purpose of this role is to enhance system performance by ensuring that Docker volumes are stored on the most appropriate storage medium, optimizing both speed and capacity.
+
+## Features
+
+- **Dynamic Volume Migration:** Moves Docker volumes based on container image types.
+- **Symbolic Link Creation:** Maintains consistent access paths after migration.
+- **Container Management:** Safely stops and starts containers during volume migration.
+- **Performance Optimization:** Improves overall system performance by leveraging appropriate storage media.
+
+## Credits 📝
+
+For detailed context and the development history of this role, refer to [this conversation](https://chat.openai.com/share/40fef8a6-5e9b-4b5e-8e68-7f2fd9abf5cc).
+

roles/sys-opt-ssd-hdd/files/sys-opt-ssd-hdd.py

@@ -0,0 +1,145 @@
+import subprocess
+import os
+import time
+import sys
+import shutil
+import argparse
+
+
+def run_command(command):
+    """ Run a shell command and return its output """
+    print(command)
+    output = subprocess.check_output(command, shell=True).decode('utf-8').strip()
+    print(output)
+    return output
+
+
+def stop_containers(containers):
+    """Stop a list of containers."""
+    container_list = ' '.join(containers)
+    print(f"Stopping containers {container_list}...")
+    run_command(f"docker stop {container_list}")
+
+
+def start_containers(containers):
+    """Start a list of containers."""
+    container_list = ' '.join(containers)
+    print(f"Starting containers {container_list}...")
+    run_command(f"docker start {container_list}")
+
+
+def is_database(image):
+    databases = {"postgres", "mariadb", "redis", "memcached", "mongo"}
+    prefix = image.split(':')[0]
+    return prefix in databases
+
+
+def is_symbolic_link(file_path):
+    return os.path.islink(file_path)
+
+
+def get_volume_path(volume):
+    return run_command(f"docker volume inspect --format '{{{{ .Mountpoint }}}}' {volume}")
+
+
+def get_image(container):
+    return run_command(f"docker inspect --format='{{{{.Config.Image}}}}' {container}")
+
+
+def has_healthcheck(container):
+    """Check if a container has a HEALTHCHECK defined."""
+    result = run_command(
+        f"docker inspect --format='{{{{json .State.Health}}}}' {container}"
+    )
+    return result not in ("null", "")
+
+
+def get_health_status(container):
+    """Return the health status."""
+    status = run_command(
+        f"docker inspect --format='{{{{.State.Health.Status}}}}' {container}"
+    )
+    return status
+
+
+def run_rsync(src, dest):
+    run_command(f"rsync -aP --remove-source-files {src} {dest}")
+
+
+def delete_directory(path):
+    """Deletes a directory and all its contents."""
+    try:
+        shutil.rmtree(path)
+        print(f"Directory {path} was successfully deleted.")
+    except OSError as e:
+        print(f"Error deleting directory {path}: {e}")
+
+
+def pause_and_move(storage_path, volume, volume_path, containers):
+    stop_containers(containers)
+    storage_volume_path = os.path.join(storage_path, 'data', 'docker', 'volumes', volume)
+    os.makedirs(storage_volume_path, exist_ok=False)
+    run_rsync(f"{volume_path}/", f"{storage_volume_path}/")
+    delete_directory(volume_path)
+    os.symlink(storage_volume_path, volume_path)
+    start_containers(containers)
+
+
+def has_container_with_database(containers):
+    for container in containers:
+        image = get_image(container)
+        if is_database(image):
+            return True
+    return False
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description='Migrate Docker volumes to SSD or HDD based on container image.'
+    )
+    parser.add_argument(
+        '--rapid-storage-path', type=str, required=True,
+        help='Path to the SSD storage'
+    )
+    parser.add_argument(
+        '--mass-storage-path', type=str, required=True,
+        help='Path to the HDD storage'
+    )
+    args = parser.parse_args()
+
+    rapid_storage_path = args.rapid_storage_path
+    mass_storage_path = args.mass_storage_path
+
+    volumes = run_command("docker volume ls -q").splitlines()
+
+    for volume in volumes:
+        volume_path = get_volume_path(volume)
+        containers = run_command(
+            f"docker ps -q --filter volume={volume}"
+        ).splitlines()
+
+        if not containers:
+            print(f"Skipped Volume {volume}. It does not belong to a running container.")
+            continue
+        if is_symbolic_link(volume_path):
+            print(f"Skipped Volume {volume}. The storage path {volume_path} is a symbolic link.")
+            continue
+
+        # Wait until containers with a healthcheck are healthy (not starting or unhealthy)
+        for container in containers:
+            if has_healthcheck(container):
+                status = get_health_status(container)
+                while status != 'healthy':
+                    print(f"Warte auf Container {container}, Status '{status}'...")
+                    time.sleep(1)
+                    status = get_health_status(container)
+
+        # Proceed with migration
+        if has_container_with_database(containers):
+            print(f"Safing volume {volume} on SSD.")
+            pause_and_move(rapid_storage_path, volume, volume_path, containers)
+        else:
+            print(f"Safing volume {volume} on HDD.")
+            pause_and_move(mass_storage_path, volume, volume_path, containers)
+
+    print("Operation completed.")

roles/sys-opt-ssd-hdd/handlers/main.yml

@@ -0,0 +1,5 @@
+- name: "reload sys-opt-ssd-hdd.cymais.service"
+  systemd:
+    name: sys-opt-ssd-hdd.cymais.service
+    state: reloaded
+    daemon_reload: yes

roles/sys-opt-ssd-hdd/meta/main.yml

@@ -0,0 +1,24 @@
+---
+galaxy_info:
+  author: "Kevin Veen-Birkenbach"
+  description: "Optimizes storage allocation for Docker volumes by migrating volumes between SSD and HDD based on container image types, and creates symbolic links to maintain consistent paths."
+  license: "CyMaIS NonCommercial License (CNCL)"
+  license_url: "https://s.veen.world/cncl"
+  company: |
+    Kevin Veen-Birkenbach
+    Consulting & Coaching Solutions
+    https://www.veen.world
+  min_ansible_version: "2.9"
+  platforms:
+    - name: Linux
+      versions:
+        - all
+  galaxy_tags:
+    - storage
+    - docker
+    - optimization
+    - performance
+  repository: "https://s.veen.world/cymais"
+  issue_tracker_url: "https://s.veen.world/cymaisissues"
+  documentation: "https://s.veen.world/cymais"
+dependencies: []

roles/sys-opt-ssd-hdd/tasks/main.yml

@@ -0,0 +1,22 @@
+- name: "create {{storage_optimizer_directory}}"
+  file:
+    path: "{{storage_optimizer_directory}}"
+    state: directory
+    mode: 0755
+
+- name: create sys-opt-ssd-hdd.cymais.service
+  template: 
+    src: sys-opt-ssd-hdd.service.j2
+    dest: /etc/systemd/system/sys-opt-ssd-hdd.cymais.service
+  notify: reload sys-opt-ssd-hdd.cymais.service
+
+- name: create sys-opt-ssd-hdd.py
+  copy: 
+    src: sys-opt-ssd-hdd.py
+    dest: "{{storage_optimizer_script}}"
+    mode: 0755
+
+- name: "optimize storage performance"
+  systemd:
+    name: sys-opt-ssd-hdd.cymais.service
+    state: started

roles/sys-opt-ssd-hdd/templates/sys-opt-ssd-hdd.service.j2

@@ -0,0 +1,8 @@
+[Unit]
+Description=Optimize storage paths
+OnFailure=sys-alm-compose.cymais@%n.service
+
+[Service]
+Type=oneshot
+ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_lock_script }} {{ system_maintenance_services | join(' ')  }} --ignore sys-opt-ssd-hdd svc-sys-bkp-rmt-2-loc --timeout "{{system_maintenance_lock_timeout_storage_optimizer}}"'
+ExecStart=/bin/sh -c '/usr/bin/python {{storage_optimizer_script}} --rapid-storage-path {{path_rapid_storage}} --mass-storage-path {{path_mass_storage}}'

roles/sys-opt-ssd-hdd/vars/main.yml

@@ -0,0 +1,2 @@
+storage_optimizer_directory: '{{path_administrator_scripts}}sys-opt-ssd-hdd/'
+storage_optimizer_script: '{{storage_optimizer_directory}}sys-opt-ssd-hdd.py'