Restructured libraries

cli/build/defaults/applications.py

@@ -5,7 +5,7 @@ import sys
 import time
 from pathlib import Path
 
-# Ensure project root on PYTHONPATH so utils is importable
+# Ensure project root on PYTHONPATH so module_utils is importable
 repo_root = Path(__file__).resolve().parent.parent.parent.parent
 sys.path.insert(0, str(repo_root))
 
@@ -13,7 +13,7 @@ sys.path.insert(0, str(repo_root))
 plugin_path = repo_root / "lookup_plugins"
 sys.path.insert(0, str(plugin_path))
 
-from utils.dict_renderer import DictRenderer
+from module_utils.dict_renderer import DictRenderer
 from application_gid import LookupModule
 
 def load_yaml_file(path: Path) -> dict:

cli/create/credentials.py

@@ -4,9 +4,9 @@ import sys
 from pathlib import Path
 import yaml
 from typing import Dict, Any
-from utils.manager.inventory import InventoryManager
-from utils.handler.vault   import VaultHandler, VaultScalar
-from utils.handler.yaml    import YamlHandler
+from module_utils.manager.inventory import InventoryManager
+from module_utils.handler.vault   import VaultHandler, VaultScalar
+from module_utils.handler.yaml    import YamlHandler
 from yaml.dumper import SafeDumper
 
 

cli/deploy.py

@@ -88,7 +88,7 @@ def validate_application_ids(inventory, app_ids):
     """
     Abort the script if any application IDs are invalid, with detailed reasons.
     """
-    from utils.valid_deploy_id import ValidDeployId
+    from module_utils.valid_deploy_id import ValidDeployId
     validator = ValidDeployId()
     invalid = validator.validate(inventory, app_ids)
     if invalid:

cli/encrypt/inventory.py

@@ -4,8 +4,8 @@ import sys
 from pathlib import Path
 import yaml
 from typing import Dict, Any
-from utils.handler.vault import VaultHandler, VaultScalar
-from utils.handler.yaml import YamlHandler
+from module_utils.handler.vault import VaultHandler, VaultScalar
+from module_utils.handler.yaml import YamlHandler
 from yaml.dumper import SafeDumper
 
 def ask_for_confirmation(key: str) -> bool:

filter_plugins/README.md

@@ -0,0 +1,27 @@
+# Custom Filter Plugins for CyMaIS
+
+This directory contains custom **Ansible filter plugins** used within the CyMaIS project.
+
+## When to Use a Filter Plugin
+
+- **Transform values:** Use filters to transform, extract, reformat, or compute values from existing variables or facts.
+- **Inline data manipulation:** Filters are designed for inline use in Jinja2 expressions (in templates, tasks, vars, etc.).
+- **No external lookups:** Filters only operate on data you explicitly pass to them and cannot access external files, the Ansible inventory, or runtime context.
+
+### Examples
+
+```jinja2
+{{ role_name | get_entity_name }}
+{{ my_list | unique }}
+{{ user_email | regex_replace('^(.+)@.*$', '\\1') }}
+````
+
+## When *not* to Use a Filter Plugin
+
+* If you need to **load data from an external source** (e.g., file, environment, API), use a lookup plugin instead.
+* If your logic requires **access to inventory, facts, or host-level information** that is not passed as a parameter.
+
+## Further Reading
+
+* [Ansible Filter Plugins Documentation](https://docs.ansible.com/ansible/latest/plugins/filter.html)
+* [Developing Ansible Filter Plugins](https://docs.ansible.com/ansible/latest/dev_guide/developing_plugins.html#developing-filter-plugins)

filter_plugins/get_application_id.py

@@ -1,5 +1,3 @@
-# filter_plugins/get_application_id.py
-
 import os
 import re
 import yaml

filter_plugins/get_docker_paths.py

@@ -1,4 +1,4 @@
-def get_docker_compose(path_docker_compose_instances: str, application_id: str) -> dict:
+def get_docker_paths(path_docker_compose_instances: str, application_id: str) -> dict:
     """
     Build the docker_compose dict based on
     path_docker_compose_instances and application_id.
@@ -23,5 +23,5 @@ def get_docker_compose(path_docker_compose_instances: str, application_id: str)
 class FilterModule(object):
     def filters(self):
         return {
-            'get_docker_compose':   get_docker_compose,
+            'get_docker_paths':   get_docker_paths,
         }

filter_plugins/get_public_id.py

@@ -1,17 +0,0 @@
-class FilterModule(object):
-    def filters(self):
-        return {
-            'get_public_id': self.get_public_id
-        }
-
-    def get_public_id(self, value):
-        """
-        Extract the substring after the last hyphen in the input string.
-        Example:
-          'service-user-abc123' => 'abc123'
-        """
-        if not isinstance(value, str):
-            raise ValueError("Expected a string")
-        if '-' not in value:
-            raise ValueError("No hyphen found in input string")
-        return value.rsplit('-', 1)[-1]

filter_plugins/role_path_by_app_id.py

@@ -1,5 +1,3 @@
-# filter_plugins/role_path_by_app_id.py
-
 import os
 import glob
 import yaml

filter_plugins/safe_join.py

@@ -1,4 +1,3 @@
-# file: filter_plugins/safe_join.py
 """
 Ansible filter plugin that joins a base string and a tail path safely.
 If the base is falsy (None, empty, etc.), returns an empty string.

filter_plugins/text_filters.py

@@ -1,5 +1,3 @@
-# filter_plugins/text_filters.py
-
 from ansible.errors import AnsibleFilterError
 import re
 

library/README.md

@@ -0,0 +1,40 @@
+# Custom Modules (`library/`) for CyMaIS
+
+This directory contains **custom Ansible modules** developed specifically for the CyMaIS project.
+
+## When to Use the `library/` Directory
+
+- **Place custom Ansible modules here:**  
+  Use this directory for any Python modules you have written yourself that are not part of the official Ansible distribution.
+- **Extend automation capabilities:**  
+  Custom modules allow you to implement logic, workflows, or integrations that are not available through built-in Ansible modules or existing community collections.
+- **Project-specific functionality:**  
+  Use for project- or infrastructure-specific tasks, such as managing custom APIs, provisioning special infrastructure resources, or integrating with internal systems.
+
+### Examples
+
+- Managing a special internal API for your company.
+- Automating a resource that has no official Ansible module.
+- Creating a highly customized deployment step for your environment.
+
+## Usage Example
+
+In your playbook, call your custom module as you would any other Ansible module:
+```yaml
+- name: Use custom CyMaIS module
+  cymais_my_custom_module:
+    option1: value1
+    option2: value2
+````
+
+Ansible automatically looks in the `library/` directory for custom modules during execution.
+
+## When *not* to Use the `library/` Directory
+
+* Do **not** place shared utility code here—put that in `module_utils/` for use across multiple modules or plugins.
+* Do **not** put filter or lookup plugins here—those belong in `filter_plugins/` or `lookup_plugins/` respectively.
+
+## Further Reading
+
+* [Developing Ansible Modules](https://docs.ansible.com/ansible/latest/dev_guide/developing_modules.html)
+* [Best Practices: Organizing Custom Modules](https://docs.ansible.com/ansible/latest/dev_guide/developing_modules_documenting.html)

lookup_plugins/README.md

@@ -0,0 +1,28 @@
+# Custom Lookup Plugins for CyMaIS
+
+This directory contains custom **Ansible lookup plugins** used within the CyMaIS project.
+
+## When to Use a Lookup Plugin
+
+- **Load external data:** Use lookups to retrieve data from files, APIs, databases, environment variables, or other external sources.
+- **Context-aware data access:** Lookups can access the full Ansible context, including inventory, facts, and runtime variables.
+- **Generate dynamic lists:** Lookups are often used to build inventories, secrets, or host lists dynamically.
+
+### Examples
+
+```yaml
+# Load the contents of a file as a variable
+my_secret: "{{ lookup('file', '/path/to/secret.txt') }}"
+
+# Retrieve a list of hostnames from an external source
+host_list: "{{ lookup('cymais_inventory_hosts', 'group_name') }}"
+````
+
+## When *not* to Use a Lookup Plugin
+
+* If you only need to **manipulate or transform data already available** in your playbook, prefer a filter plugin instead.
+
+## Further Reading
+
+* [Ansible Lookup Plugins Documentation](https://docs.ansible.com/ansible/latest/plugins/lookup.html)
+* [Developing Ansible Lookup Plugins](https://docs.ansible.com/ansible/latest/dev_guide/developing_plugins.html#developing-lookup-plugins)

main.py

@@ -35,7 +35,7 @@ if _IN_DOCKER:
 
     Sound = Quiet
 else:
-    from utils.sounds import Sound
+    from module_utils.sounds import Sound
 
 
 def color_text(text, color):

module_utils/README.md

@@ -0,0 +1,32 @@
+# Shared Utility Code (`module_utils/`) for CyMaIS
+
+This directory contains shared Python utility code (also known as "library code") for use by custom Ansible modules, plugins, or roles in the CyMaIS project.
+
+## When to Use `module_utils`
+
+- **Shared logic:** Use `module_utils` to define functions, classes, or helpers that are shared across multiple custom modules, plugins, or filter/lookups in your project.
+- **Reduce duplication:** Centralize code such as API clients, input validation, complex calculations, or protocol helpers.
+- **Maintainability:** If you find yourself repeating code in different custom modules/plugins, refactor it into `module_utils/`.
+
+### Examples
+
+- Shared HTTP(S) connection handler for multiple modules.
+- Common validation or transformation functions for user input.
+- Utility functions for interacting with Docker, LDAP, etc.
+
+## Usage Example
+
+In a custom Ansible module or plugin:
+```python
+from ansible.module_utils.cymais_utils import my_shared_function
+````
+
+## When *not* to Use `module_utils`
+
+* Do not place standalone Ansible modules or plugins here—those go into `library/`, `filter_plugins/`, or `lookup_plugins/` respectively.
+* Only use for code that will be **imported** by other plugins or modules.
+
+## Further Reading
+
+* [Ansible Module Utilities Documentation](https://docs.ansible.com/ansible/latest/dev_guide/developing_module_utilities.html)
+* [Best Practices: Reusing Code with module\_utils](https://docs.ansible.com/ansible/latest/dev_guide/developing_plugins.html#sharing-code-among-plugins)

module_utils/handler/yaml.py

@@ -1,7 +1,7 @@
 import yaml
 from yaml.loader import SafeLoader
 from typing import Any, Dict
-from utils.handler.vault import VaultScalar
+from module_utils.handler.vault import VaultScalar
 
 class YamlHandler:
     @staticmethod

module_utils/manager/inventory.py

@@ -3,8 +3,8 @@ import hashlib
 import bcrypt
 from pathlib import Path
 from typing import Dict
-from utils.handler.yaml import YamlHandler
-from utils.handler.vault import VaultHandler, VaultScalar
+from module_utils.handler.yaml import YamlHandler
+from module_utils.handler.vault import VaultHandler, VaultScalar
 import string
 import sys
 import base64

module_utils/valid_deploy_id.py

@@ -1,4 +1,4 @@
-# File: utils/valid_deploy_id.py
+# File: module_utils/valid_deploy_id.py
 """
 Utility for validating deployment application IDs against defined roles and inventory.
 """

roles/cmp-rdbms/vars/database.yml

@@ -1,17 +1,17 @@
 # Helper variables
-_database_id:                 "svc-db-{{ database_type }}"
-_database_central_name:       "{{ applications | get_app_conf( _database_id, 'docker.services.' ~ database_type ~ '.name') }}"
-_database_consumer_public_id: "{{ database_application_id | get_public_id }}"
-_database_central_enabled:    "{{ applications | get_app_conf(database_application_id, 'features.central_database', False) }}"
+_database_id:                     "svc-db-{{ database_type }}"
+_database_central_name:           "{{ applications | get_app_conf( _database_id, 'docker.services.' ~ database_type ~ '.name') }}"
+_database_consumer_entity_name:   "{{ database_application_id | get_entity_name }}"
+_database_central_enabled:        "{{ applications | get_app_conf(database_application_id, 'features.central_database', False) }}"
 
 # Definition
-database_name:      "{{ applications | get_app_conf( database_application_id, 'database.name', false, _database_consumer_public_id ) }}"                      # The overwritte configuration is needed by bigbluebutton
+database_name:      "{{ applications | get_app_conf( database_application_id, 'database.name', false, _database_consumer_entity_name ) }}"                      # The overwritte configuration is needed by bigbluebutton
 database_instance:  "{{ _database_central_name if _database_central_enabled else database_name }}"                                                            # This could lead to bugs at dedicated database @todo cleanup
 database_host:      "{{ _database_central_name if _database_central_enabled else 'database' }}"                                                               # This could lead to bugs at dedicated database @todo cleanup
-database_username:  "{{ applications | get_app_conf(database_application_id, 'database.username', false, _database_consumer_public_id)}}"                     # The overwritte configuration is needed by bigbluebutton
+database_username:  "{{ applications | get_app_conf(database_application_id, 'database.username', false, _database_consumer_entity_name)}}"                     # The overwritte configuration is needed by bigbluebutton
 database_password:  "{{ applications | get_app_conf(database_application_id, 'credentials.database_password', true) }}"
 database_port:      "{{ ports.localhost.database[ _database_id ] }}"
 database_env:       "{{docker_compose.directories.env}}{{database_type}}.env"
 database_url_jdbc:  "jdbc:{{ database_type if database_type == 'mariadb' else 'postgresql' }}://{{ database_host }}:{{ database_port }}/{{ database_name }}"
 database_url_full:  "{{database_type}}://{{database_username}}:{{database_password}}@{{database_host}}:{{database_port}}/{{ database_name }}"
-database_volume:    "{{ _database_consumer_public_id ~ '_' if not _database_central_enabled }}{{ database_host }}"
+database_volume:    "{{ _database_consumer_entity_name ~ '_' if not _database_central_enabled }}{{ database_host }}"

roles/docker-compose/vars/docker-compose.yml

@@ -1,2 +1,2 @@
 # @See https://chatgpt.com/share/67a23d18-fb54-800f-983c-d6d00752b0b4
-docker_compose: "{{ path_docker_compose_instances | get_docker_compose(application_id) }}"
+docker_compose: "{{ path_docker_compose_instances | get_docker_paths(application_id) }}"

roles/svc-db-redis/templates/service.yml.j2

@@ -3,7 +3,7 @@
 {% set redis_version = applications | get_app_conf('svc-db-redis', 'docker.services.redis.version')%}
   redis:
     image: "{{ redis_image }}:{{ redis_version }}"
-    container_name: {{ application_id | get_public_id }}-redis
+    container_name: {{ application_id | get_entity_name }}-redis
     restart: {{ docker_restart_policy }}
     logging:
       driver: journald

roles/web-app-oauth2-proxy/tasks/main.yml

@@ -1,6 +1,6 @@
-- name: "Transfering oauth2-proxy-keycloak.cfg.j2 to {{(path_docker_compose_instances | get_docker_compose(application_id)).directories.volumes}}"
+- name: "Transfering oauth2-proxy-keycloak.cfg.j2 to {{(path_docker_compose_instances | get_docker_paths(application_id)).directories.volumes}}"
   template:
     src:  "{{ playbook_dir }}/roles/web-app-oauth2-proxy/templates/oauth2-proxy-keycloak.cfg.j2"
-    dest: "{{(path_docker_compose_instances | get_docker_compose(application_id)).directories.volumes}}{{applications | get_app_conf('oauth2-proxy','configuration_file')}}"
+    dest: "{{(path_docker_compose_instances | get_docker_paths(application_id)).directories.volumes}}{{applications | get_app_conf('oauth2-proxy','configuration_file')}}"
   notify:
     - docker compose up

tests/unit/cli/create/test_credentials.py

@@ -11,7 +11,7 @@ sys.path.insert(0, dir_path)
 
 # Import functions and classes to test
 from cli.create.credentials import ask_for_confirmation, main
-from utils.handler.vault import VaultHandler, VaultScalar
+from module_utils.handler.vault import VaultHandler, VaultScalar
 import subprocess
 import tempfile
 import yaml

tests/unit/cli/test_inventory_manager.py

@@ -14,9 +14,9 @@ sys.path.insert(
     ),
 )
 
-from utils.handler.yaml import YamlHandler
-from utils.handler.vault import VaultHandler, VaultScalar
-from utils.manager.inventory import InventoryManager
+from module_utils.handler.yaml import YamlHandler
+from module_utils.handler.vault import VaultHandler, VaultScalar
+from module_utils.manager.inventory import InventoryManager
 
 
 class TestInventoryManager(unittest.TestCase):

tests/unit/filter_plugins/test_get_all_invokable_apps.py

@@ -34,10 +34,10 @@ class TestGetAllInvokableApps(unittest.TestCase):
                     "invokable": True
                 },
                 "util": {
-                    "title": "Utils",
+                    "title": "module_utils",
                     "invokable": False,
                     "desk": {
-                        "title": "Desktop Utils",
+                        "title": "Desktop module_utils",
                         "invokable": True
                     }
                 }

tests/unit/filter_plugins/test_get_public_id.py

@@ -1,26 +0,0 @@
-import unittest
-from filter_plugins.get_public_id import FilterModule
-
-class TestGetPublicId(unittest.TestCase):
-    def setUp(self):
-        self.filter = FilterModule().filters()['get_public_id']
-
-    def test_extract_public_id(self):
-        self.assertEqual(self.filter("svc-user-abc123"), "abc123")
-        self.assertEqual(self.filter("something-simple-xyz"), "xyz")
-        self.assertEqual(self.filter("a-b-c-d-e"), "e")
-
-    def test_no_hyphen(self):
-        with self.assertRaises(ValueError):
-            self.filter("nohyphenhere")
-
-    def test_non_string_input(self):
-        with self.assertRaises(ValueError):
-            self.filter(12345)
-
-    def test_empty_string(self):
-        with self.assertRaises(ValueError):
-            self.filter("")
-
-if __name__ == '__main__':
-    unittest.main()

tests/unit/utils/test_dict_renderer.py

@@ -1,5 +1,5 @@
 import unittest
-from utils.dict_renderer import DictRenderer
+from module_utils.dict_renderer import DictRenderer
 
 class TestDictRenderer(unittest.TestCase):
     def setUp(self):

tests/unit/utils/test_valid_deploy_id.py

@@ -1,9 +1,9 @@
-# File: tests/unit/utils/test_valid_deploy_id.py
+# File: tests/unit/module_utils/test_valid_deploy_id.py
 import os
 import tempfile
 import unittest
 import yaml
-from utils.valid_deploy_id import ValidDeployId
+from module_utils.valid_deploy_id import ValidDeployId
 
 class TestValidDeployId(unittest.TestCase):
     def setUp(self):