From 59af670009f2b340e7c76741f73397be50a05165 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 10 Jan 2024 14:00:43 +0100 Subject: [PATCH] Solved bugs and optimized storage-optimiter --- constructor.yml | 2 +- group_vars/all | 2 +- roles/storage-optimizer/handlers/main.yml | 4 ---- roles/storage-optimizer/tasks/main.yml | 18 ---------------- roles/storage-optimizer/vars/main.yml | 2 -- .../files/system-storage-optimizer.py} | 12 ++++++++--- roles/system-storage-optimizer/tasks/main.yml | 21 +++++++++++++++++++ .../system-storage-optimizer.service.j2} | 2 +- roles/system-storage-optimizer/vars/main.yml | 2 ++ 9 files changed, 35 insertions(+), 30 deletions(-) delete mode 100644 roles/storage-optimizer/handlers/main.yml delete mode 100644 roles/storage-optimizer/tasks/main.yml delete mode 100644 roles/storage-optimizer/vars/main.yml rename roles/{storage-optimizer/files/storage-optimizer.py => system-storage-optimizer/files/system-storage-optimizer.py} (90%) create mode 100644 roles/system-storage-optimizer/tasks/main.yml rename roles/{storage-optimizer/templates/storage-optimizer.service.j2 => system-storage-optimizer/templates/system-storage-optimizer.service.j2} (74%) create mode 100644 roles/system-storage-optimizer/vars/main.yml diff --git a/constructor.yml b/constructor.yml index 87fbbd29..b49e754f 100644 --- a/constructor.yml +++ b/constructor.yml @@ -11,7 +11,7 @@ hosts: all become: true roles: - - role: storage-optimizer + - role: system-storage-optimizer when: path_mass_storage or path_rapid_storage - name: setup standard wireguard diff --git a/group_vars/all b/group_vars/all index a25b465f..e7779984 100644 --- a/group_vars/all +++ b/group_vars/all @@ -92,7 +92,7 @@ system_maintenance_cleanup_services: system_maintenance_manipulation_services: - "heal-docker" - "update-docker" - - "storage-optimizer" + - "system-storage-optimizer" ## Total System Maintenance Services system_maintenance_services: "{{ system_maintenance_backup_services + system_maintenance_cleanup_services + system_maintenance_manipulation_services }}" diff --git a/roles/storage-optimizer/handlers/main.yml b/roles/storage-optimizer/handlers/main.yml deleted file mode 100644 index f640deee..00000000 --- a/roles/storage-optimizer/handlers/main.yml +++ /dev/null @@ -1,4 +0,0 @@ -- name: "start storage-optimizer service" - systemd: - name: backup-docker-to-local-everything.cymais.service - state: started \ No newline at end of file diff --git a/roles/storage-optimizer/tasks/main.yml b/roles/storage-optimizer/tasks/main.yml deleted file mode 100644 index ee8447e3..00000000 --- a/roles/storage-optimizer/tasks/main.yml +++ /dev/null @@ -1,18 +0,0 @@ -- name: "create {{storage_optimizer_directory}}" - file: - path: "{{storage_optimizer_directory}}" - state: directory - mode: 0755 - -- name: create storage-optimizer.cymais.service - template: - src: storage-optimizer.service.j2 - dest: /etc/systemd/system/storage-optimizer.cymais.service - notify: start storage-optimizer service - -- name: create storage-optimizer.py - template: - src: storage-optimizer.py.j2 - dest: "{{storage_optimizer_script}}" - mode: 0755 - notify: start storage-optimizer service \ No newline at end of file diff --git a/roles/storage-optimizer/vars/main.yml b/roles/storage-optimizer/vars/main.yml deleted file mode 100644 index 0e7afd7b..00000000 --- a/roles/storage-optimizer/vars/main.yml +++ /dev/null @@ -1,2 +0,0 @@ -storage_optimizer_directory: "{{path_administrator_scripts}}storage-optimizer/" -storage_optimizer_script: "{{storage_optimizer_directory}}storage-optimizer.py" \ No newline at end of file diff --git a/roles/storage-optimizer/files/storage-optimizer.py b/roles/system-storage-optimizer/files/system-storage-optimizer.py similarity index 90% rename from roles/storage-optimizer/files/storage-optimizer.py rename to roles/system-storage-optimizer/files/system-storage-optimizer.py index e01d26c9..b5776e30 100644 --- a/roles/storage-optimizer/files/storage-optimizer.py +++ b/roles/system-storage-optimizer/files/system-storage-optimizer.py @@ -36,11 +36,15 @@ def pause_and_move(storage_path, volume, volume_path, containers): stop_containers(containers) # Create a new directory on the Storage storage_volume_path = os.path.join(storage_path, 'data', 'docker', 'volumes', volume) - os.makedirs(storage_volume_path, exist_ok=False) + os.makedirs(storage_volume_path,exist_ok=False) # Move the data for item in os.listdir(volume_path): shutil.move(os.path.join(volume_path, item), storage_volume_path) + + # Ensure the volume_path is empty and remove it + if not os.listdir(volume_path): + os.rmdir(volume_path) # Create a symbolic link os.symlink(storage_volume_path, volume_path) @@ -73,9 +77,11 @@ if __name__ == "__main__": volumes = run_command("docker volume ls -q").splitlines() for volume in volumes: - containers = run_command(f"docker ps -q --filter volume={volume}").splitlines() volume_path = get_volume_path(volume) - if is_symbolic_link(volume_path): + containers = run_command(f"docker ps -q --filter volume={volume}").splitlines() + if not containers: + print(f"Skipped Volume {volume}. It does not belong to a running container.") + elif is_symbolic_link(volume_path): print(f"Skipped Volume {volume}. The storage path {volume_path} is a symbolic link.") elif has_container_with_database(containers): print(f"Safing volume {volume} on SSD.") diff --git a/roles/system-storage-optimizer/tasks/main.yml b/roles/system-storage-optimizer/tasks/main.yml new file mode 100644 index 00000000..aca01bfd --- /dev/null +++ b/roles/system-storage-optimizer/tasks/main.yml @@ -0,0 +1,21 @@ +- name: "create {{storage_optimizer_directory}}" + file: + path: "{{storage_optimizer_directory}}" + state: directory + mode: 0755 + +- name: create system-storage-optimizer.cymais.service + template: + src: system-storage-optimizer.service.j2 + dest: /etc/systemd/system/system-storage-optimizer.cymais.service + +- name: create system-storage-optimizer.py + copy: + src: system-storage-optimizer.py + dest: "{{storage_optimizer_script}}" + mode: 0755 + +- name: "optimize storage performance" + systemd: + name: system-storage-optimizer.cymais.service + state: started \ No newline at end of file diff --git a/roles/storage-optimizer/templates/storage-optimizer.service.j2 b/roles/system-storage-optimizer/templates/system-storage-optimizer.service.j2 similarity index 74% rename from roles/storage-optimizer/templates/storage-optimizer.service.j2 rename to roles/system-storage-optimizer/templates/system-storage-optimizer.service.j2 index 640543db..dad6d917 100644 --- a/roles/storage-optimizer/templates/storage-optimizer.service.j2 +++ b/roles/system-storage-optimizer/templates/system-storage-optimizer.service.j2 @@ -4,5 +4,5 @@ OnFailure=systemd-notifier.cymais@%n.service [Service] Type=oneshot -ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_lock_script }} {{ system_maintenance_services | join(' ') }} --ignore {{system_maintenance_backup_services| join(' ') }}' +ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_lock_script }} {{ system_maintenance_services | join(' ') }} --ignore system-storage-optimizer' ExecStart=/bin/sh -c '/usr/bin/python {{storage_optimizer_script}} --rapid-storage-path {{path_rapid_storage}} --mass-storage-path {{path_mass_storage}}' \ No newline at end of file diff --git a/roles/system-storage-optimizer/vars/main.yml b/roles/system-storage-optimizer/vars/main.yml new file mode 100644 index 00000000..9a84ca22 --- /dev/null +++ b/roles/system-storage-optimizer/vars/main.yml @@ -0,0 +1,2 @@ +storage_optimizer_directory: "{{path_administrator_scripts}}system-storage-optimizer/" +storage_optimizer_script: "{{storage_optimizer_directory}}system-storage-optimizer.py" \ No newline at end of file