Finished new role identification system implementation

cli/deploy.py

@@ -80,7 +80,7 @@ def main():
     )
     parser.add_argument(
         "--host-type",
-        choices=["server", "personal-computer"],
+        choices=["server", "desktop"],
         default="server",
         help="Specify whether the target is a server or a personal computer. Affects role selection and variables."
     )

cli/fix/vars_main_files.py

@@ -10,7 +10,7 @@ from pathlib import Path
 
 # Directory containing roles; can be overridden by tests
 MODULE_DIR = Path(__file__).resolve().parent
-ROLES_DIR = (MODULE_DIR.parent / "roles").resolve()
+ROLES_DIR = (MODULE_DIR.parent.parent / "roles").resolve()
 
 def process_role(role_dir: Path, prefix: str, preview: bool, overwrite: bool):
     name = role_dir.name

group_vars/all/05_maintenace.yml

@@ -28,7 +28,7 @@ system_maintenance_cleanup_services:
 system_maintenance_manipulation_services:
   - "maint-docker-heal"
   - "update-docker"
-  - "maint-docker-storage-optimizer"
+  - "cln-docker-storage-optimizer"
   - "maint-docker-restart"
   
 ## Total System Maintenance Services

playbook.yml

@@ -2,9 +2,9 @@
   hosts: all
   tasks:
   - name: "Load 'constructor' tasks"
-    include_tasks: "tasks/plays/01_constructor.yml"
+    include_tasks: "tasks/stages/01_constructor.yml"
   - name: "Load '{{host_type}}' tasks"
-    include_tasks: "tasks/plays/02_{{host_type}}.yml"
+    include_tasks: "tasks/stages/02_{{host_type}}.yml"
   - name: "Load 'destructor' tasks"
-    include_tasks: "tasks/plays/03_destructor.yml"
+    include_tasks: "tasks/stages/03_destructor.yml"
   become: true

roles/bkp-data-to-usb/vars/main.yml

@@ -1,4 +1,6 @@
-backup_to_usb_script_path:    "/usr/local/sbin/bkp-data-to-usb.python"
-backup_to_usb_destination:    "{{backup_to_usb_mount}}{{backup_to_usb_destination_subdirectory}}"
-backups_folder_path:          "{{backup_to_usb_destination}}"
-systemctl_mount_service_name: "{{ backup_to_usb_mount | trim('/') | replace('/', '-') }}.mount"
+backup_to_usb_script_path: /usr/local/sbin/bkp-data-to-usb.python
+backup_to_usb_destination: '{{backup_to_usb_mount}}{{backup_to_usb_destination_subdirectory}}'
+backups_folder_path: '{{backup_to_usb_destination}}'
+systemctl_mount_service_name: '{{ backup_to_usb_mount | trim(''/'') | replace(''/'',
+  ''-'') }}.mount'
+application_id: data-to-usb

roles/bkp-directory-validator/vars/main.yml

@@ -1 +1,2 @@
-backup_directory_validator_folder: "{{path_administrator_scripts}}directory-validator/"
+backup_directory_validator_folder: '{{path_administrator_scripts}}directory-validator/'
+application_id: directory-validator

roles/bkp-docker-to-local/vars/main.yml

@@ -1 +1,2 @@
-bkp_docker_to_local_pkg: backup-docker-to-local
+bkp_docker_to_local_pkg: backup-docker-to-local
+application_id: docker-to-local

roles/bkp-provider-user/vars/main.yml

@@ -1,2 +1,3 @@
-authorized_keys_path: "{{ inventory_dir }}/files/{{ inventory_hostname }}/home/backup/.ssh/authorized_keys"
-authorized_keys_list: "{{ lookup('file', authorized_keys_path).splitlines() }}"
+authorized_keys_path: '{{ inventory_dir }}/files/{{ inventory_hostname }}/home/backup/.ssh/authorized_keys'
+authorized_keys_list: '{{ lookup(''file'', authorized_keys_path).splitlines() }}'
+application_id: provider-user

roles/bkp-provider/vars/main.yml

@@ -0,0 +1 @@
+application_id: provider

roles/bkp-remote-to-local/vars/main.yml

@@ -1 +1,2 @@
-docker_backup_remote_to_local_folder: "{{path_administrator_scripts}}bkp-remote-to-local/"
+docker_backup_remote_to_local_folder: '{{path_administrator_scripts}}bkp-remote-to-local/'
+application_id: remote-to-local

roles/categories.yml

@@ -92,7 +92,7 @@ roles:
     title: "Backup & Restore"
     description: "Backup strategies & restore procedures"
     icon: "fas fa-hdd"
-    invokable: false
+    invokable: true
   update:
     title: "Updates & Package Management"
     description: "OS & package updates"
@@ -103,3 +103,8 @@ roles:
     description: "User accounts & access control"
     icon: "fas fa-users"
     invokable: false
+  cln:
+    title: "Cleanup"
+    description: "Roles for cleaning up various system resources—old backups, unused certificates, temporary files, Docker volumes, disk caches, deprecated domains, and more."
+    icon: "fas fa-trash-alt"
+    invokable: true

roles/cln-backups-service/vars/main.yml

@@ -1 +1,2 @@
-cleanup_backups_directory: "{{path_administrator_scripts}}cln-backups/"
+cleanup_backups_directory: '{{path_administrator_scripts}}cln-backups/'
+application_id: backups-service

roles/cln-backups-timer/vars/main.yml

@@ -0,0 +1 @@
+application_id: backups-timer

roles/cln-certs/vars/main.yml

@@ -0,0 +1 @@
+application_id: certs

roles/cln-disc-space/vars/main.yml

@@ -1 +1,2 @@
-cleanup_disc_space_folder: "{{path_administrator_scripts}}cln-disc-space/"
+cleanup_disc_space_folder: '{{path_administrator_scripts}}cln-disc-space/'
+application_id: disc-space

roles/cln-docker-anonymous-volumes/vars/main.yml

@@ -0,0 +1 @@
+application_id: docker-anonymous-volumes

roles/cln-docker-storage-optimizer/handlers/main.yml

@@ -0,0 +1,5 @@
+- name: "reload cln-docker-storage-optimizer.cymais.service"
+  systemd:
+    name: cln-docker-storage-optimizer.cymais.service
+    state: reloaded
+    daemon_reload: yes

roles/cln-docker-storage-optimizer/tasks/main.yml

@@ -0,0 +1,22 @@
+- name: "create {{storage_optimizer_directory}}"
+  file:
+    path: "{{storage_optimizer_directory}}"
+    state: directory
+    mode: 0755
+
+- name: create cln-docker-storage-optimizer.cymais.service
+  template: 
+    src: cln-docker-storage-optimizer.service.j2
+    dest: /etc/systemd/system/cln-docker-storage-optimizer.cymais.service
+  notify: reload cln-docker-storage-optimizer.cymais.service
+
+- name: create cln-docker-storage-optimizer.py
+  copy: 
+    src: cln-docker-storage-optimizer.py
+    dest: "{{storage_optimizer_script}}"
+    mode: 0755
+
+- name: "optimize storage performance"
+  systemd:
+    name: cln-docker-storage-optimizer.cymais.service
+    state: started

roles/cln-docker-storage-optimizer/templates/cln-docker-storage-optimizer.service.j2

@@ -4,5 +4,5 @@ OnFailure=alert-compose.cymais@%n.service
 
 [Service]
 Type=oneshot
-ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_lock_script }} {{ system_maintenance_services | join(' ')  }} --ignore maint-docker-storage-optimizer bkp-remote-to-local --timeout "{{system_maintenance_lock_timeout_storage_optimizer}}"'
+ExecStartPre=/bin/sh -c '/usr/bin/python {{ path_system_lock_script }} {{ system_maintenance_services | join(' ')  }} --ignore cln-docker-storage-optimizer bkp-remote-to-local --timeout "{{system_maintenance_lock_timeout_storage_optimizer}}"'
 ExecStart=/bin/sh -c '/usr/bin/python {{storage_optimizer_script}} --rapid-storage-path {{path_rapid_storage}} --mass-storage-path {{path_mass_storage}}'

roles/cln-docker-storage-optimizer/vars/main.yml

@@ -0,0 +1,3 @@
+storage_optimizer_directory: '{{path_administrator_scripts}}cln-docker-storage-optimizer/'
+storage_optimizer_script: '{{storage_optimizer_directory}}cln-docker-storage-optimizer.py'
+application_id: docker-storage-optimizer

roles/cln-domains/vars/main.yml

@@ -0,0 +1 @@
+application_id: domains

roles/cln-failed-docker-backups/vars/main.yml

@@ -1 +1,2 @@
-cln_failed_docker_backups_pkg: cleanup-failed-docker-backups
+cln_failed_docker_backups_pkg: cleanup-failed-docker-backups
+application_id: failed-docker-backups

roles/desk-docker/vars/main.yml

@@ -1 +1 @@
-application_id: docker
+application_id: desk-docker

roles/desk-git/meta/main.yml

@@ -17,4 +17,4 @@ galaxy_info:
     - git
     - configuration
     - pacman
-    - personal-computer
+    - desktop

roles/docker-core/tasks/main.yml

@@ -16,7 +16,7 @@
     group: administrator
   when: run_once_docker is not defined
 
-- name: Set docker_enabled to true, to activate maint-docker-storage-optimizer 
+- name: Set docker_enabled to true, to activate cln-docker-storage-optimizer 
   set_fact:
     docker_enabled: true
   when: run_once_docker is not defined

roles/maint-docker-storage-optimizer/handlers/main.yml

@@ -1,5 +0,0 @@
-- name: "reload maint-docker-storage-optimizer.cymais.service"
-  systemd:
-    name: maint-docker-storage-optimizer.cymais.service
-    state: reloaded
-    daemon_reload: yes

roles/maint-docker-storage-optimizer/tasks/main.yml

@@ -1,22 +0,0 @@
-- name: "create {{storage_optimizer_directory}}"
-  file:
-    path: "{{storage_optimizer_directory}}"
-    state: directory
-    mode: 0755
-
-- name: create maint-docker-storage-optimizer.cymais.service
-  template: 
-    src: maint-docker-storage-optimizer.service.j2
-    dest: /etc/systemd/system/maint-docker-storage-optimizer.cymais.service
-  notify: reload maint-docker-storage-optimizer.cymais.service
-
-- name: create maint-docker-storage-optimizer.py
-  copy: 
-    src: maint-docker-storage-optimizer.py
-    dest: "{{storage_optimizer_script}}"
-    mode: 0755
-
-- name: "optimize storage performance"
-  systemd:
-    name: maint-docker-storage-optimizer.cymais.service
-    state: started

roles/maint-docker-storage-optimizer/vars/main.yml

@@ -1,3 +0,0 @@
-storage_optimizer_directory: '{{path_administrator_scripts}}maint-docker-storage-optimizer/'
-storage_optimizer_script: '{{storage_optimizer_directory}}maint-docker-storage-optimizer.py'
-application_id: docker-storage-optimizer

roles/web-svc-simpleicons/vars/main.yml

@@ -1,4 +1,4 @@
-application_id:                 simpleicons 
+application_id:                 simpleicons
 container_port:                 3000
 simpleicons_host_server_file:   "{{docker_compose.directories.config}}server.js"
 simpleicons_host_package_file:  "{{docker_compose.directories.config}}package.json"

tasks/plays/02_personal-computer.yml

@@ -1,91 +0,0 @@
----
-## pc applications
-- name: "setup docker role includes for desktop pc"
-  include_tasks: "./tasks/groups/{{ item }}-roles.yml"
-  loop:
-    - util-desk
-    - desk
-  loop_control:
-    label: "{{ item }}-roles.yml"
-
-- name: general host setup
-  when: ("personal_computers" in group_names)
-  include_role:
-    name: "{{ item }}"
-  loop:
-    - util-gen-admin
-    - drv-non-free
-
-- name: util-desk-office-tools
-  when: ("collection_officetools" in group_names)
-  include_role:
-    name: "{{ item }}"
-  loop:
-    - util-desk-office-tools
-    - desk-jrnl
-
-- name: personal computer for business
-  when: ("business_personal_computer" in group_names)
-  include_role:
-    name: desk-gnucash
-
-- name: util-desk-design
-  when: ("collection_designer" in group_names)
-  include_role:
-    name: util-desk-design
-
-- name: desk-qbittorrent
-  when: ("collection_torrent" in group_names)
-  include_role:
-    name: desk-qbittorrent
-
-- name: desk-obs
-  when: ("collection_streamer" in group_names)
-  include_role:
-    name: desk-obs
-
-- name: desk-bluray-player
-  when: ("collection_bluray_player" in group_names)
-  include_role:
-    name: desk-bluray-player
-
-- name: GNOME setup
-  when: ("gnome" in group_names)
-  include_role:
-    name: desk-gnome
-
-- name: setup ssh client
-  when: ("ssh-client" in group_names)
-  include_role:
-    name: desk-ssh
-
-- name: setup gaming hosts
-  when: ("gaming" in group_names)
-  include_role:
-    name: util-desk-game-compose
-
-- name: setup entertainment hosts
-  when: ("entertainment" in group_names)
-  include_role:
-    name: desk-spotify
-
-- name: setup torbrowser hosts
-  when: ("torbrowser" in group_names)
-  include_role:
-    name: desk-torbrowser
-
-- name: setup nextcloud-client
-  when: ("nextcloud_client" in group_names)
-  include_role:
-    name: desk-nextcloud-client
-
-- name: setup docker
-  when: ("docker_client" in group_names)
-  include_role:
-    name: desk-docker
-
-# driver 
-- name: setup msi rgb keyboard
-  when: ("msi_perkeyrgb" in group_names)
-  include_role:
-    name: drv-msi-keyboard-color

tasks/plays/03_destructor.yml

@@ -1,15 +0,0 @@
-- name: optimize storage performance
-  include_role:
-    name: maint-docker-storage-optimizer
-  when: ('storage-optimizer' | application_allowed(group_names, allowed_applications))
-  
-- name: Cleanup Docker Anonymous Volumes
-  import_role:
-    name: cln-docker-anonymous-volumes
-  when: mode_cleanup | bool
-
-- name: Show all facts
-  debug:
-    var: ansible_facts
-  when: enable_debug | bool
-

tasks/stages/01_constructor.yml

@@ -101,7 +101,7 @@
     name: update
   when: mode_update | bool
 
-- name: "Integrate Docker Role includes"
+- name: "Load base roles"
   include_tasks: "./tasks/groups/{{ item }}-roles.yml"
   loop:
     - core
@@ -111,55 +111,8 @@
     - alert
     - mon
     - maint
+    - update
+    - bkp
+    - cln
   loop_control:
-    label: "{{ item }}-roles.yml"
-
-- name: setup standard wireguard
-  when: ('wireguard_server' | application_allowed(group_names, allowed_applications))
-  include_role:
-    name: net-wireguard-core
-
-# vpn setup
-- name: setup wireguard client behind firewall\nat
-  when: ('wireguard_behind_firewall' | application_allowed(group_names, allowed_applications))
-  include_role:
-    name: net-wireguard-firewalled
-
-- name: setup wireguard client
-  when: ('wireguard_client' | application_allowed(group_names, allowed_applications))
-  include_role:
-    name: net-wireguard-plain
-
-## backup setup
-- name: setup replica backup hosts
-  when: ('backup_remote_to_local' | application_allowed(group_names, allowed_applications))
-  include_role:
-    name: bkp-remote-to-local
-
-- name: setup backup to swappable
-  when: ('backup_to_usb' | application_allowed(group_names, allowed_applications))
-  include_role:
-    name: bkp-data-to-usb
-
-## driver setup
-- name: drv-intel
-  when: ('intel' | application_allowed(group_names, allowed_applications))
-  include_role:
-    name: drv-intel
-
-- name: setup multiprinter hosts
-  when: ('epson_multiprinter' | application_allowed(group_names, allowed_applications))
-  include_role:
-    name: drv-epson-multiprinter
-
-
-- name: setup hibernate lid switch
-  when: ('drv-lid-switch' | application_allowed(group_names, allowed_applications))
-  include_role:
-    name: drv-lid-switch
-
-## system setup 
-- name: setup swapfile hosts
-  when: ('swapfile' | application_allowed(group_names, allowed_applications))
-  include_role:
-    name: maint-swapfile
+    label: "{{ item }}-roles.yml"

tasks/stages/02_personal-computer.yml

@@ -0,0 +1,16 @@
+---
+- name: "setup docker role includes for desktop pc"
+  include_tasks: "./tasks/groups/{{ item }}-roles.yml"
+  loop:
+    - util-srv  # Services need to run before applications
+    - util-desk
+  loop_control:
+    label: "{{ item }}-roles.yml"
+
+- name: general host setup
+  when: ("personal_computers" in group_names)
+  include_role:
+    name: "{{ item }}"
+  loop:
+    - util-gen-admin
+    - drv-non-free

tasks/stages/02_server.yml

@@ -1,5 +1,5 @@
 ---
-- name: servers host setup
+- name: Setup server base
   when: ("servers" in group_names)
   include_role:
     name: "{{ item }}"
@@ -11,10 +11,10 @@
     - mon-bot-btrfs
     - maint-btrfs-auto-balancer
 
-- name: "Integrate Docker Server Roles"
+- name: "Include server roles"
   include_tasks: "./tasks/groups/{{ item }}-roles.yml"
   loop:
-    - svc
-    - web
+    - web-svc # Services need to run before applications
+    - web-app
   loop_control:
     label: "{{ item }}-roles.yml"

tasks/stages/03_destructor.yml

@@ -0,0 +1,6 @@
+- name: "Load destruction roles"
+  include_tasks: "./tasks/groups/{{ item }}-roles.yml"
+  loop:
+    - cln
+  loop_control:
+    label: "{{ item }}-roles.yml"

tests/integration/test_application_id_consistency.py

@@ -39,7 +39,7 @@ class TestApplicationIdConsistency(unittest.TestCase):
                     continue
 
             actual_id = vars_data.get("application_id")
-            if actual_id != expected_id:
+            if actual_id not in [expected_id, role_name]:
                 failed_roles.append((
                     role_name,
                     f"application_id is '{actual_id}', expected '{expected_id}'"

tests/integration/test_meta_roles_integration.py

@@ -0,0 +1,67 @@
+import unittest
+from pathlib import Path
+import re
+import os
+import sys
+
+# Ensure your project root is on PYTHONPATH so filter_plugins can be imported
+ROOT = Path(__file__).parents[2]
+sys.path.insert(0, str(ROOT))
+
+from filter_plugins.invokable_paths import get_invokable_paths
+
+STAGES_DIR = ROOT / "tasks" / "stages"
+GROUPS_DIR = ROOT / "tasks" / "groups"
+
+class TestMetaRolesIntegration(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        # Use the filter directly
+        cls.role_files = get_invokable_paths(suffix="-roles.yml")
+        cls.invokable_items = get_invokable_paths()
+
+        # Read all playbook YAML contents under tasks/stages
+        cls.playbook_contents = {}
+        for path in STAGES_DIR.rglob("*.yml"):
+            cls.playbook_contents[path] = path.read_text(encoding="utf-8")
+
+        # Regex for include_tasks line with {{ item }}-roles.yml
+        cls.include_pattern = re.compile(
+            r'include_tasks:\s*["\']\./tasks/groups/\{\{\s*item\s*\}\}-roles\.yml["\']'
+        )
+
+    def test_all_role_files_exist(self):
+        """Each '-roles.yml' path returned by the filter must exist in the project root."""
+        missing = []
+        for fname in self.role_files:
+            path = GROUPS_DIR / fname
+            if not path.is_file():
+                missing.append(fname)
+        self.assertFalse(
+            missing,
+            f"The following role files are missing at project root: {missing}"
+        )
+
+    def test_each_invokable_item_referenced_in_playbooks(self):
+        """
+        Each invokable item (without suffix) must be looped through in at least one playbook
+        and include its corresponding include_tasks entry.
+        """
+        not_referenced = []
+        for item in self.invokable_items:
+            found = False
+            loop_entry = re.compile(rf"-\s*{re.escape(item)}\b")
+            for content in self.playbook_contents.values():
+                if self.include_pattern.search(content) and loop_entry.search(content):
+                    found = True
+                    break
+            if not found:
+                not_referenced.append(item)
+
+        self.assertEqual(
+            not_referenced, [],
+            f"The following invokable items are not referenced in any playbook: {not_referenced}"
+        )
+
+if __name__ == "__main__":
+    unittest.main()

tests/integration/test_unique_application_id.py

@@ -0,0 +1,39 @@
+import os
+import glob
+import yaml
+import unittest
+
+
+def find_application_ids():
+    """
+    Scans all roles/*/vars/main.yml files and collects application_id values.
+    Returns a dict mapping application_id to list of file paths where it appears.
+    """
+    ids = {}
+    # Wenn der Test unter tests/integration liegt, gehen wir zwei Ebenen hoch zum Projekt-Root
+    base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+    pattern = os.path.join(base_dir, "roles", "*", "vars", "main.yml")
+
+    for file_path in glob.glob(pattern):
+        with open(file_path, 'r') as f:
+            data = yaml.safe_load(f) or {}
+        app_id = data.get('application_id')
+        if app_id is not None:
+            ids.setdefault(app_id, []).append(file_path)
+    return ids
+
+
+class TestUniqueApplicationId(unittest.TestCase):
+    def test_application_ids_are_unique(self):
+        ids = find_application_ids()
+        duplicates = {app_id: paths for app_id, paths in ids.items() if len(paths) > 1}
+        if duplicates:
+            messages = []
+            for app_id, paths in duplicates.items():
+                file_list = '\n    '.join(paths)
+                messages.append(f"application_id '{app_id}' found in multiple files:\n    {file_list}")
+            self.fail("\n\n".join(messages))
+
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2)

tests/unit/roles/cln-docker-storage-optimizer/test_storage_optimizer.py

@@ -8,7 +8,7 @@ import sys
 def load_optimizer_module():
     module_path = os.path.abspath(os.path.join(
         os.path.dirname(__file__),
-        '..', "..", "..","..",'roles', 'maint-docker-storage-optimizer', 'files', 'maint-docker-storage-optimizer.py'
+        '..', "..", "..","..",'roles', 'cln-docker-storage-optimizer', 'files', 'cln-docker-storage-optimizer.py'
     ))
     spec = importlib.util.spec_from_file_location('storage_optimizer', module_path)
     optimizer = importlib.util.module_from_spec(spec)