Optimized database logic

group_vars/all

@@ -92,6 +92,7 @@ system_maintenance_cleanup_services:
 system_maintenance_manipulation_services:
   - "heal-docker"
   - "update-docker"
+  - "system-storage-optimizer"
   
 ## Total System Maintenance Services
 system_maintenance_services: "{{ system_maintenance_backup_services + system_maintenance_cleanup_services + system_maintenance_manipulation_services }}"

inventory.example.yml

@@ -45,4 +45,7 @@ pixelfed_database_password:         "I_NEED_TO_CHANGE_THIS_UNSECURE_PASSWORD"
 pixelfed_app_key:                   "I_NEED_TO_CHANGE_THIS_UNSECURE_PASSWORD"
 wordpress_database_password:        "I_NEED_TO_CHANGE_THIS_UNSECURE_PASSWORD"
 yourls_database_password:           "I_NEED_TO_CHANGE_THIS_UNSECURE_PASSWORD"
-yourls_user_password:               "I_NEED_TO_CHANGE_THIS_UNSECURE_PASSWORD"
+yourls_user_password:               "I_NEED_TO_CHANGE_THIS_UNSECURE_PASSWORD"
+
+path_mass_storage:                  "/mnt/hdd/"
+path_rapid_storage:                 "/mnt/ssd/"

playbook.destructor.yml

@@ -0,0 +1,6 @@
+- name: optimize storage performance
+  hosts:  all
+  become: true
+  roles:
+    - role: system-storage-optimizer
+      when: path_mass_storage or path_rapid_storage

playbook.pcs.yml

@@ -1,6 +1,6 @@
 ---
 
-- import_playbook: playbook-common.yml
+- import_playbook: playbook.constructor.yml
 
 ## pc applications
 - name: general host setup
@@ -102,4 +102,4 @@
   roles:
     - driver-msi-keyboard-color
 
-- import_playbook: destructor.yml
+- import_playbook: playbook.destructor.yml

playbook.servers.yml

@@ -1,6 +1,6 @@
 ---
 
-- import_playbook: constructor.yml
+- import_playbook: playbook.constructor.yml
 
 - name: servers host setup
   hosts: servers
@@ -269,3 +269,5 @@
   become: true
   roles:
    -  role: nginx-www-redirect
+
+- import_playbook: playbook.destructor.yml

roles/backup-remote-to-local/handlers/main.yml

@@ -2,10 +2,3 @@
   systemd:
     name: backup-remote-to-local.cymais.service
     daemon_reload: yes
-    
-- name: "restart backup-remote-to-local timer"
-  systemd:
-    name: backup-remote-to-local.cymais.timer
-    state: started
-    enabled: yes
-    daemon_reload: yes

roles/docker-listmonk/templates/docker-compose.yml.j2

@@ -16,6 +16,6 @@ services:
 {% include 'templates/docker-container-networks.yml.j2' %}
 {% include 'templates/docker-container-depends-on-just-database.yml.j2' %}
 
-{% include 'templates/docker-compose-volumes.yml.j2' %}
+{% include 'templates/docker-compose-volumes-just-database.yml.j2' %}
 
 {% include 'templates/docker-compose-networks.yml.j2' %}

roles/driver-msi-keyboard-color/tasks/main.yml

@@ -5,5 +5,29 @@
     name:
       - msi-perkeyrgb
 
-- include_tasks: setup_script.yml
-- include_tasks: setup_timers.yml
+- name: Copy keyboard_color.sh script
+  copy:
+    src: keyboard_color.py
+    dest: /opt/keyboard_color.py
+    mode: 0755
+
+- name: Copy keyboard-color.cymais.service file
+  template:
+    src: keyboard-color.service.j2
+    dest: /etc/systemd/system/keyboard-color.cymais.service
+    mode: 0644
+
+- name: Reload systemd daemon
+  systemd:
+    daemon_reload: yes
+
+- name: set service_name to the name of the current role
+  set_fact:
+    service_name: "{{ role_name }}"
+
+- name: "include role for systemd-timer for {{service_name}}"
+  include_role:
+    name: systemd-timer
+  vars:
+    on_calendar:  "{{on_calendar_msi_keyboard_color}}"
+    persistent:   "true"

roles/driver-msi-keyboard-color/tasks/setup_script.yml

@@ -1,8 +0,0 @@
----
-- name: Copy keyboard_color.sh script
-  copy:
-    src: keyboard_color.py
-    dest: /opt/keyboard_color.py
-    mode: 0755
-  tags:
-    - keyboard-color

roles/driver-msi-keyboard-color/tasks/setup_timers.yml

@@ -1,30 +0,0 @@
----
-- name: Copy keyboard-color.cymais.timer file
-  template:
-    src: keyboard-color.timer.j2
-    dest: /etc/systemd/system/keyboard-color.cymais.timer
-    mode: 0644
-  tags:
-    - keyboard-color
-
-- name: Copy keyboard-color.cymais.service file
-  template:
-    src: keyboard-color.service.j2
-    dest: /etc/systemd/system/keyboard-color.cymais.service
-    mode: 0644
-  tags:
-    - keyboard-color
-
-- name: Reload systemd daemon
-  systemd:
-    daemon_reload: yes
-  tags:
-    - keyboard-color
-
-- name: Enable and start keyboard-color.cymais.timer
-  systemd:
-    name: keyboard-color.cymais.timer
-    state: restarted
-    enabled: yes
-  tags:
-    - keyboard-color

roles/driver-msi-keyboard-color/templates/keyboard-color.timer.j2

@@ -1,9 +0,0 @@
-[Unit]
-Description=Keyboard Color Timer
-
-[Timer]
-OnCalendar={{on_calendar_msi_keyboard_color}}
-Persistent=true
-
-[Install]
-WantedBy=timers.target

roles/system-maintenance-lock/files/system-maintenance-lock.py

@@ -65,14 +65,21 @@ def wait_for_all_services_to_stop(filtered_services, max_attempts, attempt):
 def get_max_attempts(timeout_sec):
     return timeout_sec // BREAK_TIME_SECONDS
 
+def append_suffix_to_services(services, suffix=".cymais"):
+    """
+    Append a specified suffix to each service name in the list.
+    """
+    return [service + suffix for service in services]
+
 def main(services, ignored_services, timeout_sec):
     """
     Main function to process the command-line arguments and perform actions.
     """
-        
-    filtered_services = filter_services(services, ignored_services)
-    print(f"Services to handle: {services}")
-    print(f"Services to ignore: {ignored_services}")
+    services_with_suffix = append_suffix_to_services(services)
+    ignored_services_with_suffix = append_suffix_to_services(ignored_services)
+    filtered_services = filter_services(services_with_suffix, ignored_services_with_suffix )
+    print(f"Services to handle: {services_with_suffix}")
+    print(f"Services to ignore: {ignored_services_with_suffix}")
     print(f"Services filtered: {filtered_services}")
     
     print("Waiting for services to stop.")
@@ -82,7 +89,6 @@ def main(services, ignored_services, timeout_sec):
     while check_any_service_active(filtered_services):
         attempt = wait_for_all_services_to_stop(filtered_services, max_attempts, attempt)
     print("All required services have stopped.")
-
     
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Blocks the code execution as long as defined services are running. Terminates with 0 when all services stopped')

roles/system-storage-optimizer/README.md

@@ -0,0 +1,23 @@
+# System Storage Optimizer Role
+
+## Overview
+
+The `system-storage-optimizer` role is designed for managing and optimizing storage allocation of Docker volumes. It automates the process of moving Docker volumes between SSD (rapid storage) and HDD (mass storage) based on the container image types, enhancing performance and efficiency.
+
+## Features
+
+- Dynamically moves Docker volumes based on container image types.
+- Utilizes SSDs for database-related volumes for faster access.
+- Moves non-database volumes to HDDs for efficient mass storage.
+- Handles container stopping and starting during the optimization process.
+- Creates symbolic links to maintain consistent paths.
+
+
+## Usage
+
+To deploy this role, include it in your Ansible playbook and define the necessary paths for SSD and HDD storage.
+
+## Additional Information
+
+For detailed context and the development history of this role, refer to [this conversation](https://chat.openai.com/share/40fef8a6-5e9b-4b5e-8e68-7f2fd9abf5cc).
+

roles/system-storage-optimizer/files/system-storage-optimizer.py

@@ -1,11 +1,16 @@
 import subprocess
 import os
+import time
+import sys
 import shutil
 import argparse
 
 def run_command(command):
     """ Run a shell command and return its output """
-    return subprocess.check_output(command, shell=True).decode('utf-8').strip()
+    print(command)
+    output = subprocess.check_output(command, shell=True).decode('utf-8').strip()
+    print(output)
+    return output
 
 def stop_containers(containers):
     """Stop a list of containers."""
@@ -32,15 +37,28 @@ def get_volume_path(volume):
 def get_image(container):
     return run_command(f"docker inspect --format='{{{{.Config.Image}}}}' {container}")
 
+def run_rsync(src, dest):
+    run_command(f"rsync -aP --remove-source-files {src} {dest}")
+
+def delete_directory(path):
+    """Deletes a directory and all its contents."""
+    try:
+        shutil.rmtree(path)
+        print(f"Directory {path} was successfully deleted.")
+    except OSError as e:
+        print(f"Error deleting directory {path}: {e}")
+
 def pause_and_move(storage_path, volume, volume_path, containers):
     stop_containers(containers)
     # Create a new directory on the Storage
-    storage_volume_path = os.path.join(storage_path, volume)
-    os.makedirs(storage_volume_path, exist_ok=False)
+    storage_volume_path = os.path.join(storage_path, 'data', 'docker', 'volumes', volume)
+    os.makedirs(storage_volume_path,exist_ok=False)
 
     # Move the data
-    for item in os.listdir(volume_path):
-        shutil.move(os.path.join(volume_path, item), storage_volume_path)
+    run_rsync(volume_path, storage_volume_path)
+    
+    # Delete the source directory
+    delete_directory(volume_path)
 
     # Create a symbolic link
     os.symlink(storage_volume_path, volume_path)
@@ -73,9 +91,11 @@ if __name__ == "__main__":
     volumes = run_command("docker volume ls -q").splitlines()
     
     for volume in volumes:
-        containers = run_command(f"docker ps -q --filter volume={volume}").splitlines()
         volume_path = get_volume_path(volume)
-        if is_symbolic_link(volume_path):
+        containers = run_command(f"docker ps -q --filter volume={volume}").splitlines()
+        if not containers:
+            print(f"Skipped Volume {volume}. It does not belong to a running container.")
+        elif is_symbolic_link(volume_path):
             print(f"Skipped Volume {volume}. The storage path {volume_path} is a symbolic link.")
         elif has_container_with_database(containers):
             print(f"Safing volume {volume} on SSD.")

roles/system-storage-optimizer/tasks/main.yml

@@ -0,0 +1,21 @@
+- name: "create {{storage_optimizer_directory}}"
+  file:
+    path: "{{storage_optimizer_directory}}"
+    state: directory
+    mode: 0755
+
+- name: create system-storage-optimizer.cymais.service
+  template: 
+    src: system-storage-optimizer.service.j2
+    dest: /etc/systemd/system/system-storage-optimizer.cymais.service
+
+- name: create system-storage-optimizer.py
+  copy: 
+    src: system-storage-optimizer.py
+    dest: "{{storage_optimizer_script}}"
+    mode: 0755
+
+- name: "optimize storage performance"
+  systemd:
+    name: system-storage-optimizer.cymais.service
+    state: started

roles/system-storage-optimizer/templates/system-storage-optimizer.service.j2

@@ -0,0 +1,8 @@
+[Unit]
+Description=Optimize storage paths
+OnFailure=systemd-notifier.cymais@%n.service
+
+[Service]
+Type=oneshot
+ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_lock_script }} {{ system_maintenance_services | join(' ')  }} --ignore system-storage-optimizer'
+ExecStart=/bin/sh -c '/usr/bin/python {{storage_optimizer_script}} --rapid-storage-path {{path_rapid_storage}} --mass-storage-path {{path_mass_storage}}'

roles/system-storage-optimizer/vars/main.yml

@@ -0,0 +1,2 @@
+storage_optimizer_directory: "{{path_administrator_scripts}}system-storage-optimizer/"
+storage_optimizer_script:    "{{storage_optimizer_directory}}system-storage-optimizer.py"