Different optimations and mig integration. test will fail due to strickter validation checks. need to be cleaned up tomorrow

filter_plugins/get_app_conf.py

@@ -3,6 +3,12 @@ import re
 import yaml
 from ansible.errors import AnsibleFilterError
 
+from ansible.errors import AnsibleUndefinedVariable
+try:
+    from ansible.utils.unsafe_proxy import AnsibleUndefined
+except ImportError:
+    class AnsibleUndefined: pass
+
 class AppConfigKeyError(AnsibleFilterError, ValueError):
     """
     Raised when a required application config key is missing (strict mode).
@@ -50,6 +56,16 @@ def get_app_conf(applications, application_id, config_path, strict=True, default
             )
         k, idx = m.group(1), m.group(2)
 
+        if (hasattr(obj, '__class__') and obj.__class__.__name__ == 'AnsibleUndefined') \
+            or isinstance(obj, AnsibleUndefinedVariable):
+            if not strict:
+                return default if default is not None else False
+            raise AppConfigKeyError(
+                f"Key '{k}' is undefined at '{'.'.join(path_trace)}'\n"
+                f"application_id: {application_id}\n"
+                f"config_path: {config_path}"
+            )
+
         # Access dict key
         if isinstance(obj, dict):
             if k not in obj:

group_vars/all/09_ports.yml

@@ -21,6 +21,7 @@ ports:
     ldap:
       svc-db-openldap: 389
     http:
+      # Ports which are exposed to the World Wide Web
       web-app-nextcloud: 8001
       web-app-gitea: 8002
       web-app-wordpress: 8003
@@ -65,11 +66,10 @@ ports:
       collabora: 8042
       mobilizon: 8043
       simpleicons: 8044
-      libretranslate: 8055
-      pretix:         8056
+      libretranslate: 8045
+      pretix: 8046
+      web-app-mig: 8047
       web-app-bigbluebutton: 48087    # This port is predefined by bbb. @todo Try to change this to a 8XXX port
-  # Ports which are exposed to the World Wide Web
-
   public:
     # The following ports should be changed to 22 on the subdomain via stream mapping
     ssh:

group_vars/all/10_networks.yml

@@ -92,10 +92,14 @@ defaults_networks:
       subnet: 192.168.103.128/28
     pretix:
       subnet: 192.168.103.144/28
+    web-app-mig:
+      subnet: 192.168.103.160/28
 
     # /24 Networks / 254 Usable Clients
     web-app-bigbluebutton:
-      subnet: 10.7.7.0/24 # This variable does not have an impact. It's just there for documentation reasons, because this network is used in bbb
+      # This network variable isn't used.
+      # It's registered here to make transparent which network bbb is using and to avoid conflicts.
+      subnet: 10.7.7.0/24
     svc-db-postgres:
       subnet: 192.168.200.0/24
     svc-db-mariadb:

group_vars/all/Todo.md

@@ -0,0 +1,5 @@
+# Todos
+- Remove, *_users.yml, *_applications.yml, *_ports.yml and *_networks.yml
+- Move their values to the roles/*/config/main.yml file of each role
+- Implement validators (e.g. no overlappsing networks, unique ports) for all of this configurations
+- Keep this design decissions in ming during new feature implementations

roles/cmp-db-docker-proxy/tasks/main.yml

@@ -1,3 +1,5 @@
+# run_once_cmp_db_docker_proxy: deactivated
+
 - name: "For '{{ application_id }}': load docker and db"
   include_role: 
     name: cmp-db-docker

roles/desk-gnome-caffeine/meta/main.yml

@@ -14,4 +14,4 @@ galaxy_info:
     - autostart
     - archlinux
 dependencies:
-  - sys-pgm-aur
+  - dev-yay

roles/desk-qbittorrent/README.md

@@ -11,7 +11,7 @@ The `main.yml` file in the `desk-qbittorrent` role includes the following task:
 
 ## Dependencies
 This role depends on:
-- **sys-pgm-aur**: Ensures that an Arch User Repository (AUR) helper is installed, which is necessary for installing packages like `qbittorrent` that are not available in the standard repositories.
+- **dev-yay**: Ensures that an Arch User Repository (AUR) helper is installed, which is necessary for installing packages like `qbittorrent` that are not available in the standard repositories.
 
 ## Purpose and Usage
 The `desk-qbittorrent` role is tailored for users who require a reliable and user-friendly torrent client for downloading and sharing files via the BitTorrent protocol. qBittorrent is known for its balance of features, simplicity, and minimal impact on system resources.

roles/desk-qbittorrent/meta/main.yml

@@ -19,5 +19,5 @@ galaxy_info:
     - name: Archlinux
       versions: [ all ]
 dependencies:
-  - sys-pgm-aur
+  - dev-yay
 

roles/desk-spotify/README.md

@@ -16,18 +16,18 @@ To automate the installation of Spotify on Arch-based systems while ensuring pro
 
 - 🎧 Installs the official [Spotify AUR package](https://aur.archlinux.org/packages/spotify)
 - 🛠 Uses `yay` (or other helper) via [`kewlfft.aur`](https://github.com/kewlfft/ansible-aur) Ansible module
-- 🔗 Declares dependency on `sys-pgm-aur` for seamless integration
+- 🔗 Declares dependency on `dev-yay` for seamless integration
 
 ## Requirements
 
-- The `sys-pgm-aur` role must be applied before using this role.
+- The `dev-yay` role must be applied before using this role.
 - An AUR helper like `yay` must be available on the system.
 
 ## Dependencies
 
 This role depends on:
 
-- [`sys-pgm-aur`](../sys-pgm-aur) – provides and configures an AUR helper like `yay`
+- [`dev-yay`](../dev-yay) – provides and configures an AUR helper like `yay`
 
 ## Credits 📝
 

roles/desk-spotify/meta/main.yml

@@ -24,4 +24,4 @@ galaxy_info:
   issue_tracker_url: https://s.veen.world/cymaisissues
   documentation: https://s.veen.world/cymais
 dependencies:
-  - sys-pgm-aur
+  - dev-yay

roles/desk-zoom/README.md

@@ -14,7 +14,7 @@ The `main.yml` file in the `desk-zoom` role includes tasks for setting up video
 
 ## Dependencies
 This role relies on:
-- **sys-pgm-aur**: Ensures that an Arch User Repository (AUR) helper is installed, necessary for installing software like Zoom which may not be available in standard repositories.
+- **dev-yay**: Ensures that an Arch User Repository (AUR) helper is installed, necessary for installing software like Zoom which may not be available in standard repositories.
 
 ## Purpose and Usage
 The `desk-zoom` role is particularly useful for professionals, educators, and anyone who needs reliable video conferencing capabilities on their Linux system. With the increasing demand for remote communication, this role provides an efficient way to set up key video conferencing tools.

roles/desk-zoom/meta/main.yml

@@ -20,5 +20,5 @@ galaxy_info:
     - name: Archlinux
       versions: [ all ]
 dependencies:
-  - sys-pgm-aur
+  - dev-yay
 

roles/dev-base-devel/README.md

@@ -0,0 +1,20 @@
+# dev-base-devel Role
+
+This Ansible role installs the essential `base-devel` package group on Arch Linux systems.
+
+## Description
+
+The `base-devel` package group contains all fundamental development tools required for building and compiling software from source on Arch Linux and compatible distributions.
+
+## Usage
+
+After deploying this role, all common build dependencies will be available on the system, allowing you to compile and install software packages that require development tools.
+
+## Features
+
+- Installs all packages from the `base-devel` group
+- Ensures your system is ready for software compilation and development
+
+## Further Resources
+
+- [Arch Wiki: base-devel](https://wiki.archlinux.org/title/Development_packages)

roles/dev-base-devel/meta/main.yml

@@ -0,0 +1,24 @@
+---
+galaxy_info:
+  author: "Kevin Veen-Birkenbach"
+  description: >
+    This role installs the base-devel package group, providing all core development tools needed for building software on Arch Linux systems.
+  license: "CyMaIS NonCommercial License (CNCL)"
+  license_url: "https://s.veen.world/cncl"
+  company: | 
+    Kevin Veen-Birkenbach
+    Consulting & Coaching Solutions
+    https://www.veen.world
+  galaxy_tags:
+    - archlinux
+    - development
+    - base-devel
+    - build
+    - tools
+  repository: "https://github.com/kevinveenbirkenbach/cymais"
+  issue_tracker_url: "https://github.com/kevinveenbirkenbach/cymais/issues"
+  documentation: "https://wiki.archlinux.org/title/Development_packages"
+  logo:
+    class: "fas fa-tools"
+  run_after: []
+dependencies: []

roles/dev-base-devel/tasks/main.yml

@@ -0,0 +1,5 @@
+- name: install base-devel
+  community.general.pacman:
+    name: base-devel
+    state: present
+  become: true

roles/dev-yay/meta/main.yml

@@ -21,5 +21,9 @@ galaxy_info:
   repository: "https://s.veen.world/cymais"
   issue_tracker_url: "https://s.veen.world/cymaisissues"
   documentation: "https://s.veen.world/cymais"
+  logo:
+    class: "fas fa-archive"
 dependencies:
   - dev-fakeroot
+  - dev-git
+  - dev-base-devel

roles/dev-yay/tasks/main.yml

@@ -1,9 +1,8 @@
 - name: install yay
   community.general.pacman:
     name:
-      - yay
-      - patch
       - base-devel
+      - patch
     state: present
 
 - name: Create the `aur_builder` user
@@ -20,3 +19,21 @@
     line: 'aur_builder ALL=(ALL) NOPASSWD: /usr/bin/pacman'
     create: yes
     validate: 'visudo -cf %s'
+
+- name: Clone yay from AUR
+  become: yes
+  become_user: aur_builder
+  git:
+    repo: https://aur.archlinux.org/yay.git
+    dest: /home/aur_builder/yay
+    clone: yes
+    update: yes
+
+- name: Build and install yay
+  become: yes
+  become_user: aur_builder
+  shell: |
+    cd /home/aur_builder/yay
+    makepkg -si --noconfirm
+  args:
+    creates: /usr/bin/yay

roles/docker-compose/Todo.md

@@ -0,0 +1,2 @@
+# Todos
+- Make network name equal to get_entity_name

roles/docker-compose/defaults/main.yml

@@ -1,3 +1,3 @@
 docker_compose_skipp_file_creation: false # If set to true the file creation will be skipped
-docker_repository:                  false # Activates docker repository download and routine
+docker_pull_git_repository:                  false # Activates docker repository download and routine
 docker_compose_flush_handlers:      false # Set to true in the vars/main.yml of the including role to autoflush after docker compose routine

roles/docker-compose/tasks/main.yml

@@ -14,10 +14,12 @@
     mode: '0755'
   with_dict: "{{ docker_compose.directories }}"
 
-- include_tasks: "repository.yml"
-  when: docker_repository | bool
+- name: "Include routines to set up a git repository based installaion for '{{application_id}}'."
+  include_tasks: "repository.yml"
+  when: docker_pull_git_repository | bool
 
-- include_tasks: "files.yml"
+- name: "Include routines file management routines for '{{application_id}}'."
+  include_tasks: "files.yml"
   when: not docker_compose_skipp_file_creation | bool
 
 - name: "flush database, docker and proxy for '{{ application_id }}'" 

roles/drv-epson-multiprinter/meta/main.yml

@@ -20,5 +20,5 @@ galaxy_info:
     - name: Archlinux
       versions: [ all ]
 dependencies:
-  - sys-pgm-aur
+  - dev-yay
 

roles/drv-msi-keyboard-color/meta/main.yml

@@ -8,5 +8,5 @@ galaxy_info:
       versions:
         - all
 dependencies:
-  - sys-pgm-aur
+  - dev-yay
   - sys-alm-compose

roles/pkgmgr-install/Todo.md

@@ -0,0 +1,2 @@
+# Todos
+- Implement, that the already during a play installed apps get stored, so that the installation process isn't triggered twice

roles/srv-web-7-6-composer/tasks/main.yml

@@ -1,3 +1,5 @@
+# run_once_srv_web_7_7_inj_compose: deactivated
+
 - name: "include role srv-web-7-7-inj-compose for {{domain}}"
   include_role: 
     name: srv-web-7-7-inj-compose

roles/srv-web-7-7-dns-records/tasks/main.yml

@@ -1,3 +1,5 @@
+# run_once_srv_web_7_7_dns_records: deactivated
+
 - name: Create or update Cloudflare A-record for {{ item }}
   community.general.cloudflare_dns:
     api_token: "{{ cloudflare_api_token }}"

roles/sys-cli/README.md

@@ -0,0 +1,29 @@
+# CyMaIS CLI
+
+This Ansible role installs and makes the CyMaIS CLI available on your system.
+
+## Description
+
+After deploying this role, you will have access to the `cymais` command-line interface (CLI), which is the central tool for managing and operating all aspects of your CyMaIS environment.
+
+## Usage
+
+Once this role has been applied, you can run all CLI commands using:
+
+```
+
+cymais --help
+
+```
+
+to get a list of available commands and options.
+
+## Features
+
+- Installs the CyMaIS CLI automatically
+- Ensures the CLI is available system-wide
+- All commands accessible via `cymais --help`
+
+## Further Resources
+
+- [CyMaIS Documentation](https://github.com/kevinveenbirkenbach/cymais/)

roles/sys-cli/meta/main.yml

@@ -0,0 +1,24 @@
+
+galaxy_info:
+  author: "Kevin Veen-Birkenbach"
+  description: >
+    This role installs and provides the CyMaIS CLI, enabling you to manage your entire CyMaIS environment from the command line. After deployment, the `cymais` command is available.
+  license: "CyMaIS NonCommercial License (CNCL)"
+  license_url: "https://s.veen.world/cncl"
+  company: | 
+    Kevin Veen-Birkenbach
+    Consulting & Coaching Solutions
+    https://www.veen.world
+  galaxy_tags:
+    - cymais
+    - cli
+    - management
+    - automation
+  repository: "https://github.com/kevinveenbirkenbach/cymais"
+  issue_tracker_url: "https://github.com/kevinveenbirkenbach/cymais/issues"
+  documentation: "https://github.com/kevinveenbirkenbach/cymais/"
+  logo:
+    class: "fa-solid fa-terminal"
+  run_after: []
+dependencies:
+  - dev-yay

roles/sys-cli/tasks/main.yml

@@ -0,0 +1,10 @@
+- name: "pkgmgr install cymais"
+  include_role:
+    name: pkgmgr-install
+  vars:
+    package_name: cymais
+
+- name: Get cymais base path
+  command: pkgmgr path cymais
+  register: cymais_path_cmd
+  changed_when: false

roles/sys-timer/tasks/main.yml

@@ -1,6 +1,7 @@
+
 - name: "reset (if enabled)"
   include_tasks: reset.yml 
-  when: mode_reset | bool and run_once_gen_timer is not defined
+  when: mode_reset | bool and run_once_sys_timer is not defined
 
 - name: create {{service_name}}.cymais.timer
   template:
@@ -18,5 +19,5 @@
 
 - name: run {{ role_name }} once
   set_fact:
-    run_once_gen_timer: true
-  when: run_once_gen_timer is not defined
+    run_once_sys_timer: true
+  when: run_once_sys_timer is not defined

roles/update-yay/meta/main.yml

@@ -24,4 +24,4 @@ galaxy_info:
   issue_tracker_url: "https://s.veen.world/cymaisissues"
   documentation: "https://s.veen.world/cymais"
 dependencies:
-  - sys-pgm-aur
+  - dev-yay

roles/util-desk-design/meta/main.yml

@@ -21,5 +21,5 @@ galaxy_info:
     - name: Archlinux
       versions: [ all ]
 dependencies:
-  - sys-pgm-aur
+  - dev-yay
 

roles/util-dev-admin/meta/main.yml

@@ -26,3 +26,4 @@ dependencies:
   - desk-git
   - dev-make
   - dev-gcc
+  - dev-yay

roles/util-dev-admin/tasks/main.yml

@@ -1,8 +1,7 @@
 - name: install administration tools
-  community.general.pacman:
+  pacman:
     name:
       - base-devel
-      - yay
       - cmake
       - fdupes
       - p7zip

roles/web-app-bigbluebutton/vars/main.yml

@@ -16,7 +16,3 @@ docker_compose_skipp_file_creation: true  # Skipp creation of docker-compose.yml
 
 # Setup 
 bigbluebutton_setup:                      "{{ applications | get_app_conf(application_id, 'setup') }}"
-
-# Credentials 
-bigbluebutton_shared_secret:              "{{ applications | get_app_conf(application_id, 'credentials.shared_secret') }}"
-bigbluebutton_api_suffix:                 "{{ applications | get_app_conf(application_id, 'api_suffix') }}"

roles/web-app-discourse/config/main.yml

@@ -24,12 +24,9 @@ docker:
       enabled: true
     redis:
       enabled: true
-    # This container is propably wrong name. 
-    # Chance is high that the name is discourse_application.
-    # @todo check this out and repair it if necessary
     discourse:
       name:             "discourse"
-      image:            "local_discourse/discourse_application" # Necessary to define this for the docker 2 loc backup
+      image:            "local_discourse/<< defaults_applications[web-app-discourse].docker.services.discourse.name >>" # Necessary to define this for the docker 2 loc backup
       backup:
         no_stop_required: true
   volumes: 

roles/web-app-discourse/handlers/main.yml

@@ -1,14 +1,14 @@
 ---
 - name: "stop and remove discourse container if it exist"
   docker_container:
-    name: "{{ discourse_name }}"
+    name: "{{ discourse_container }}"
     state: absent
   register: container_action
   failed_when: container_action.failed and 'No such container' not in container_action.msg
   listen: recreate discourse
   
-- name: "add central database temporary to {{application_id}}_default"
-  command: docker network connect {{applications | get_app_conf(application_id, 'network', True)}} {{ database_host }}
+- name: "add central database temporary to {{ discourse_network }}"
+  command: "docker network connect {{ discourse_network }} {{ database_host }}"
   failed_when: >
     result.rc != 0 and
     'already exists in network' not in result.stderr
@@ -17,7 +17,7 @@
   listen: recreate discourse
 
 - name: rebuild discourse
-  shell: ./launcher rebuild {{ discourse_name }}
+  shell: ./launcher rebuild {{ discourse_container }}
   args:
     executable: /bin/bash  
     chdir: "{{docker_repository_directory }}"

roles/web-app-discourse/tasks/main.yml

@@ -1,21 +1,22 @@
 ---
-- name: "reset (if enabled)"
+- name: "Setup {{ application_id }}"
+  when: run_once_web_app_discourse is not defined
+  block:
+  - name: "reset (if enabled)"
     include_tasks: reset.yml 
-  when: mode_reset | bool and run_once_docker_discourse is not defined
+    when: mode_reset | bool
 
-# Necessary for building: https://chat.openai.com/share/99d258cc-294b-4924-8eef-02fe419bb838
-- name: install which
+  # Necessary for building: https://chat.openai.com/share/99d258cc-294b-4924-8eef-02fe419bb838
+  - name: install which
     pacman:
       name: which
       state: present
-  when: run_once_docker_discourse is not defined
 
-- name: "load docker, db and proxy for {{application_id}}"
+  - name: "load docker, db and proxy for {{application_id}}"
     include_role: 
       name: cmp-db-docker-proxy
-  when: run_once_docker_discourse is not defined
 
-- name: pull docker repository
+  - name: pull docker repository
     git:
       repo: "https://github.com/discourse/discourse_docker.git"
       dest: "{{docker_repository_directory }}"
@@ -23,40 +24,45 @@
     notify: recreate discourse
     become: true
     ignore_errors: true
-  when: run_once_docker_discourse is not defined
 
-- name: set chmod 700 for {{docker_repository_directory }}containers
+  - name: set chmod 700 for {{docker_repository_directory }}containers
     ansible.builtin.file:
       path: "{{docker_repository_directory }}/containers"
       mode: '700'
       state: directory
-  when: run_once_docker_discourse is not defined
 
-- name: "copy configuration to {{discourse_application_yml_destination}}"
+  - name: "copy configuration to {{discourse_application_yml_destination}}"
     template: 
-    src: discourse_application.yml.j2
-    dest: "{{discourse_application_yml_destination}}"
+      src: config.yml.j2
+      dest: "{{ discourse_application_yml_destination }}"
     notify: recreate discourse
-  when: run_once_docker_discourse is not defined
 
-- name: flush, to recreate discourse app
+  - name: "Verify that {{ discourse_container }} is running"
+    command:      docker compose ps --filter status=running --format '{{"{{"}}.Name{{"}}"}}' | grep -x {{ discourse_container }}
+    register:     docker_ps
+    changed_when: docker_ps.rc == 1
+    failed_when:  docker_ps.rc not in [0, 1]
+    notify:       recreate discourse
+
+  - name: flush, to recreate discourse app
     meta: flush_handlers
-  when: run_once_docker_discourse is not defined
 
-- name: "Connect {{ discourse_name }} to network {{ applications | get_app_conf('svc-db-postgres', 'docker.network' ) }}"
+  - name: Set error string for network already exists
+    set_fact:
+      docker_discourse_already_in_net: "Error response from daemon: endpoint with name {{ discourse_container }} already exists in network {{ discourse_pg_network }}"
+
+  - name: "Connect {{ discourse_container }} to network {{ discourse_pg_network }}"
     command: >
-    docker network connect {{ applications | get_app_conf('svc-db-postgres', 'docker.network' ) }} {{ discourse_name }}
+      docker network connect {{ discourse_pg_network }} {{ discourse_container }}
     register: network_connect
     failed_when: >
       network_connect.rc != 0 and
-    "Error response from daemon: endpoint with name {{ discourse_name }} already exists in network {{ applications | get_app_conf('svc-db-postgres', 'docker.network' ) }}"
-    not in network_connect.stderr
+      docker_discourse_already_in_net not in network_connect.stderr
     changed_when: network_connect.rc == 0
     when:
       - applications | get_app_conf(application_id, 'features.central_database', False)
-    - run_once_docker_discourse is not defined
 
-- name: "Remove {{ discourse_network }} from {{ database_host }}"
+  - name: "Remove {{ discourse_network }} from {{ database_host }}"
     command: >
       docker network disconnect {{ discourse_network }} {{ database_host }}
     register: network_disconnect
@@ -66,9 +72,8 @@
     changed_when: network_disconnect.rc == 0
     when:
       - applications | get_app_conf(application_id, 'features.central_database', False)
-    - run_once_docker_discourse is not defined
 
-- name: run the docker_discourse tasks once
+  - name: run the docker_discourse tasks once
     set_fact:
-    run_once_docker_discourse: true
-  when: run_once_docker_discourse is not defined
+      run_once_web_app_discourse: true
+    when: run_once_web_app_discourse is not defined

roles/web-app-discourse/tasks/reset.yml

@@ -6,9 +6,9 @@
     cmd: "docker network disconnect {{applications | get_app_conf(application_id, 'network', True)}} {{ database_host }}"
   ignore_errors: true
 
-- name: "destroy container discourse_application"
+- name: "destroy container {{ discourse_container }}"
   command:
-    cmd: "./launcher destroy discourse_application"
+    cmd: "./launcher destroy {{ discourse_container }}"
     chdir: "{{ docker_repository_directory }}"
   ignore_errors: true
   notify: recreate discourse

roles/web-app-discourse/templates/config.yml.j2

@@ -41,7 +41,7 @@ env:
   UNICORN_WORKERS: 8
 
   ## Required. Discourse will not work with a bare IP number.
-  DISCOURSE_HOSTNAME: {{domains | get_domain(application_id)}}
+  DISCOURSE_HOSTNAME: {{ domains | get_domain(application_id) }}
 
   ## Uncomment if you want the container to be started with the same
   ## hostname (-h option) as specified above (default "$hostname-$config")
@@ -139,7 +139,7 @@ run:
   #- exec: rails r "User.find_by_email('{{ users.administrator.email }}').update(username: '{{users.administrator.username}}')"
 
   # The following code is just an inspiration, how to connect with the oidc account. as long as this is not set the admini account needs to be manually connected with oidc 
-  # docker exec -it discourse_application rails runner "user = User.find_by_email('test@cymais.cloud'); UserAuth.create(user_id: user.id, provider: 'oidc', uid: 'eindeutige_oidc_id', info: { name: user.username, email: user.email })"
+  # docker exec -it {{ discourse_container }} rails runner "user = User.find_by_email('test@cymais.cloud'); UserAuth.create(user_id: user.id, provider: 'oidc', uid: 'eindeutige_oidc_id', info: { name: user.username, email: user.email })"
 
   # OIDC Activation
   - exec: rails r "SiteSetting.openid_connect_enabled = true"
@@ -178,4 +178,4 @@ run:
 
 docker_args:
   - --network={{application_id}}_default
-  - --name={{ discourse_name }}
+  - --name={{ discourse_container }}

roles/web-app-discourse/vars/main.yml

@@ -1,15 +1,16 @@
 application_id:                         "web-app-discourse"
 
 # Database
-database_password:                      "{{ applications | get_app_conf(application_id, 'credentials.database_password', True) }}"
+database_password:                      "{{ applications | get_app_conf(application_id, 'credentials.database_password') }}"
 database_type:                          "postgres"
 
 # Discourse
-discourse_name:                         "{{ applications | get_app_conf(application_id, 'docker.services.discourse.name', True) }}"
-discourse_application_yml_destination:  "{{ docker_repository_directory }}containers/{{discourse_name }}.yml"
-discourse_network:                      "{{ applications | get_app_conf(application_id, 'docker.network', True) }}"
-discourse_volume:                       "{{ applications | get_app_conf(application_id, 'docker.volumes.data', True) }}"
+discourse_container:                    "{{ applications | get_app_conf(application_id, 'docker.services.discourse.name') }}"
+discourse_application_yml_destination:  "{{ docker_repository_directory }}containers/{{ discourse_container }}.yml"
+discourse_network:                      "{{ applications | get_app_conf(application_id, 'docker.network') }}"
+discourse_volume:                       "{{ applications | get_app_conf(application_id, 'docker.volumes.data') }}"
+discourse_pg_network:                   "{{ applications | get_app_conf('svc-db-postgres', 'docker.network' ) }}"
 
 # General Docker Configuration
-docker_repository_directory :           "{{ docker_compose.directories.services}}{{applications | get_app_conf(application_id, 'repository', True) }}/"
+docker_repository_directory :           "{{ docker_compose.directories.services}}{{applications | get_app_conf( application_id, 'repository') }}/"
 docker_compose_flush_handlers:          false

roles/web-app-friendica/tasks/main.yml

@@ -34,6 +34,9 @@
   loop_control:
     label: "{{ item.key }}"
 
+- name: flush handlers to ensure that friendica is up before friendica addon configuration
+  meta: flush_handlers
+
 - name: Ensure Friendica addons are in sync
   command: >
     docker compose exec --user www-data

roles/web-app-mig/README.md

@@ -0,0 +1,21 @@
+# MIG
+
+This folder contains the Ansible role to deploy the Meta Infinite Graph for CyMaIS.
+
+## Description
+
+This role sets up the [Ansible Meta Infinite Graph](https://github.com/kevinveenbirkenbach/meta-infinite-graph) for CyMaIS. The Meta Infinite Graph visualizes all dependencies and relationships between CyMaIS roles, making the overall infrastructure structure transparent and easy to understand.
+
+## Overview
+
+The Meta Infinite Graph is an essential tool for analyzing, auditing, and maintaining the modular structure of the CyMaIS ecosystem. It provides a clear overview of all roles and how they are interconnected.
+
+## Features
+
+- Automatic deployment of the Meta Infinite Graph web application
+- Shows all dependencies and connections between CyMaIS roles
+- Useful for documentation and architecture transparency
+
+## Further Resources
+
+- [Meta Infinite Graph Homepage](https://github.com/kevinveenbirkenbach/meta-infinite-graph)

roles/web-app-mig/TODO.md

@@ -0,0 +1,2 @@
+# Todos
+- Use svc-meta-creator to speed up deployment

roles/web-app-mig/config/main.yml

@@ -0,0 +1,37 @@
+docker:
+  services:
+    redis:
+      enabled:        false   # No redis needed
+    database:
+      enabled:        false   # No database needed
+features:
+  matomo:             true  # activate tracking
+  css:                true  # use custom cymais stile 
+  port-ui-desktop:    true  # Enable in port-ui
+csp:
+  whitelist:
+    script-src-elem:
+      - https://cdn.jsdelivr.net
+      - https://kit.fontawesome.com
+      - https://code.jquery.com/
+      - https://unpkg.com/
+    style-src:
+      - https://cdn.jsdelivr.net
+      - https://cdnjs.cloudflare.com
+    font-src:
+      - https://cdnjs.cloudflare.com
+      - https://ka-f.fontawesome.com
+      - https://cdn.jsdelivr.net
+    connect-src:
+      - https://ka-f.fontawesome.com
+    #frame-src:
+    #  - "{{ web_protocol }}://*.{{primary_domain}}"
+  flags:
+    style-src:
+      unsafe-inline: true
+domains:
+  canonical:
+    - "mig.{{ primary_domain }}"
+  aliases:
+    - "meta-infinite-graph.{{ primary_domain }}"
+build_data:           true  # Enables the building of the meta data which the graph requiers

roles/web-app-mig/meta/main.yml

@@ -0,0 +1,25 @@
+---
+galaxy_info:
+  author: "Kevin Veen-Birkenbach"
+  description: >
+    The Meta Infinite Graph offers you an interactive, visual map of all CyMaIS roles and their dependencies—making it easy to explore, understand, and navigate the complete structure of your infrastructure.
+  license: "CyMaIS NonCommercial License (CNCL)"
+  license_url: "https://s.veen.world/cncl"
+  company: | 
+    Kevin Veen-Birkenbach
+    Consulting & Coaching Solutions
+    https://www.veen.world
+  galaxy_tags:
+    - cymais
+    - meta
+    - visualization
+    - dependencies
+    - graph
+  repository: "https://github.com/kevinveenbirkenbach/meta-infinite-graph"
+  issue_tracker_url: "https://github.com/kevinveenbirkenbach/meta-infinite-graph/issues"
+  documentation: "https://github.com/kevinveenbirkenbach/meta-infinite-graph/"
+  logo:
+    class: ""
+  run_after: []
+dependencies:
+  - sys-cli

roles/web-app-mig/tasks/main.yml

@@ -0,0 +1,35 @@
+---
+- block: 
+  - name: Load docker compose vars
+    include_vars:
+      file: roles/docker-compose/vars/docker-compose.yml
+      name: mig_docker_compose
+
+  - name: Set roles volume variable
+    set_fact:
+      mig_roles_meta_volume:  "{{ mig_docker_compose.docker_compose.directories.volumes }}/roles/"
+    
+  - name: Set roles list variable
+    set_fact:
+      mig_roles_meta_list:    "{{ mig_roles_meta_volume }}list.json"
+
+  - name: "load docker, proxy for '{{application_id}}'"
+    include_role: 
+      name: cmp-docker-proxy
+
+  - name: Create tree
+    command: "cymais build tree --no-signal --alarm-timeout 0 -s {{  mig_roles_meta_volume }}"
+    when: 
+      - mig_build_data
+
+  - name: Create roles list
+    command: "cymais build roles_list --no-signal --alarm-timeout 0 -o {{ mig_roles_meta_list }}"
+    when: 
+      - mig_build_data
+
+  - name: run the web-app-mig tasks once
+    set_fact:
+      run_once_docker_web_app_mig: true
+  name: "Setup Meta Infinite Graph"
+  when: run_once_docker_web_app_mig is not defined
+  

roles/web-app-mig/templates/docker-compose.yml.j2

@@ -0,0 +1,21 @@
+
+{% include 'roles/docker-compose/templates/base.yml.j2' %}
+  {{ application_id | get_entity_name }}:
+{% set container_port = 80 %}
+    {% include 'roles/docker-container/templates/base.yml.j2' %}
+    image: "{{ mig_image }}"
+    container_name: "{{ mig_container }}"
+    ports:
+      - 127.0.0.1:{{ports.localhost.http[application_id]}}:{{ container_port }}
+    build:
+      context:    "{{docker_repository_path}}"
+      dockerfile: Dockerfile
+    volumes:
+      - "{{ mig_roles_meta_volume }}:/usr/share/nginx/html/roles:ro"
+      - "{{ docker_repository_path }}:/usr/share/nginx/html"
+{% include 'roles/docker-container/templates/networks.yml.j2' %}
+{% include 'roles/docker-container/templates/healthcheck/curl.yml.j2' %}
+
+{% include 'roles/docker-compose/templates/networks.yml.j2' %}
+
+

roles/web-app-mig/vars/main.yml

@@ -0,0 +1,12 @@
+# General 
+application_id:                         web-app-mig          # ID of the application, should be the name of the role folder
+
+# Docker
+docker_compose_flush_handlers:          true
+docker_pull_git_repository:             true
+docker_repository_address:              "https://github.com/kevinveenbirkenbach/meta-infinite-graph"
+
+# Helper variables
+mig_image:                              "mig:latest"
+mig_container:                          "mig"
+mig_build_data:                         "{{ applications | get_app_conf(application_id, 'build_data') }}"

roles/web-app-moodle/tasks/main.yml

@@ -23,12 +23,13 @@
   include_tasks: oidc.yml
   when: applications | get_app_conf(application_id, 'features.oidc', False)
 
-- name: Run Moodle system check
-  command: >
-    docker exec --user {{ bitnami_user }} {{ moodle_container }}
-    php /opt/bitnami/moodle/admin/cli/checks.php
-  register: moodle_checks
-  changed_when: false
-  failed_when: >
-    moodle_checks.rc != 0 or
-    "OK: All" not in moodle_checks.stdout
+# Deactivated because it doesn't give helpfull warnings back
+#- name: Run Moodle system check
+#  command: >
+#    docker exec --user {{ bitnami_user }} {{ moodle_container }}
+#    php /opt/bitnami/moodle/admin/cli/checks.php
+#  register: moodle_checks
+#  changed_when: false
+#  failed_when: >
+#    moodle_checks.rc != 0 or
+#    "OK: All" not in moodle_checks.stdout

roles/web-app-nextcloud/vars/plugins/bbb.yml

@@ -1,7 +1,7 @@
 plugin_configuration:
   - appid:        "bbb"
     configkey:    "api.secret"
-    configvalue:  "{{ bigbluebutton_shared_secret }}"
+    configvalue:  "{{ applications | get_app_conf('web-app-bigbluebutton', 'credentials.shared_secret', False,'') }}"
   - appid:        "bbb"
     configkey:    "api.url"
-    configvalue:  "{{ domains | get_url('web-app-bigbluebutton', web_protocol) }}{{ bigbluebutton_api_suffix }}"
+    configvalue:  "{{ domains | get_url('web-app-bigbluebutton', web_protocol) }}{{ applications | get_app_conf('web-app-bigbluebutton', 'api_suffix', False,'') }}"

roles/web-app-oauth2-proxy/tasks/main.yml

@@ -1,6 +1,6 @@
-- name: "Transfering oauth2-proxy-keycloak.cfg.j2 to {{( path_docker_compose_instances | get_docker_paths(application_id)).directories.volumes }}"
+- name: "Transfering oauth2-proxy-keycloak.cfg.j2 to {{( application_id | get_docker_paths(path_docker_compose_instances) ).directories.volumes }}"
   template:
     src:  "{{ playbook_dir }}/roles/web-app-oauth2-proxy/templates/oauth2-proxy-keycloak.cfg.j2"
-    dest: "{{( path_docker_compose_instances | get_docker_paths(application_id)).directories.volumes }}{{applications | get_app_conf('oauth2-proxy','configuration_file')}}"
+    dest: "{{( application_id | get_docker_paths(path_docker_compose_instances) ).directories.volumes }}{{applications | get_app_conf('oauth2-proxy','configuration_file')}}"
   notify:
     - docker compose up

roles/web-app-openproject/vars/main.yml

@@ -1,7 +1,7 @@
 application_id:               "web-app-openproject"
 docker_repository_address:    "https://github.com/opf/openproject-deploy"
 database_type:                "postgres"
-docker_repository:            true
+docker_pull_git_repository:            true
 openproject_version:          "{{ applications | get_app_conf(application_id, 'docker.services.web.version', True) }}"
 openproject_image:            "{{ applications | get_app_conf(application_id, 'docker.services.web.image', True) }}"
 openproject_volume:           "{{ applications | get_app_conf(application_id, 'docker.volumes.data', True) }}"

roles/web-app-peertube/tasks/main.yml

@@ -5,7 +5,7 @@
   
 - name: "include create-domains.yml for peertube"
   include_tasks: create-domains.yml
-  loop: "{{ domains.peertube }}"
+  loop: "{{ domains['web-app-peertube'] }}"
   loop_control:
     loop_var: domain
   vars:

roles/web-app-peertube/vars/main.yml

@@ -1,7 +1,15 @@
+# General
 application_id:                 "web-app-peertube"
 database_type:                  "postgres"
-oidc_plugin:        "peertube-plugin-auth-openid-connect"
+
+# Docker Specific
+docker_compose_flush_handlers:  true
+
+# Role variables
 peertube_version:               "{{ applications | get_app_conf(application_id, 'docker.services.peertube.version', True) }}"
 peertube_image:                 "{{ applications | get_app_conf(application_id, 'docker.services.peertube.image', True) }}"
 peertube_name:                  "{{ applications | get_app_conf(application_id, 'docker.services.peertube.name', True) }}"
 peertube_volume:                "{{ applications | get_app_conf(application_id, 'docker.volumes.data', True) }}"
+
+# OIDC
+oidc_plugin:                    "peertube-plugin-auth-openid-connect"

roles/web-app-port-ui/TODO.md

@@ -0,0 +1,2 @@
+# Todos
+- Solve OAuth2 Login Issue - See https://chatgpt.com/c/687a50b4-8d78-800f-a202-1631aa05fd4f

roles/web-app-port-ui/vars/main.yml

@@ -1,4 +1,4 @@
 application_id:             "web-app-port-ui"
 docker_repository_address:  "https://github.com/kevinveenbirkenbach/port-ui"
 config_inventory_path:      "{{ inventory_dir }}/files/{{ inventory_hostname }}/docker/web-app-port-ui/config.yaml.j2"
-docker_repository:          true
+docker_pull_git_repository:          true

roles/web-app-taiga/vars/main.yml

@@ -10,7 +10,7 @@ taiga_image_frontend: >-
   {{ 'robrotheram/taiga-front-openid' if applications | get_app_conf(application_id, 'features.oidc', True) and applications | get_app_conf(application_id, 'oidc.flavor', True) == 'robrotheram'
      else 'taigaio/taiga-front' }}
 taiga_frontend_conf_path:   "{{docker_compose.directories.config}}conf.json"
-docker_repository:          true
+docker_pull_git_repository:          true
 settings_files:
 - urls
 - local

roles/web-opt-rdr-domains/tasks/main.yml

@@ -1,4 +1,6 @@
 ---
+# run_once_web_opt_rdr_domains: deactivated
+
 - name: "Include domains redirects"
   include_tasks: redirect-domain.yml
   vars:

templates/roles/web-app/config/main.yml.j2

@@ -5,16 +5,16 @@ docker:
       enabled:          false   # Enable Redis 
     database:
       enabled:          false   # Enable the database
-    {{ application_id }}:  
+    {{ application_id | get_entity_name }}:  
       backup:
         no_stop_required: true  # The images that don't need to stop
         disabled:         true  # Disables the image
         database_routine: true  # Instead of copying a database routine will be triggered for this container
-      image:            ""
-      version:          "latest"
-      name:             "web-app-{{ application_id }}"
+      image:            ""                                            # The docker image of the software you want to use
+      version:          "latest"                                      # The docker version of the software you want to use
+      container:        "{{ application_id | get_entity_name }}"      # The container name
     volumes:
-      data:             "web-app-{{ application_id }}_data"
+      data:             "{{ application_id | get_entity_name }}_data"
 features:
   matomo:           true    # Enable Matomo Tracking
   css:              true    # Enable Global CSS Styling
@@ -25,8 +25,20 @@ features:
   oauth2:           false   # Enable the OAuth2-Proy
   javascript:       false   # Enables the custom JS in the javascript.js.j2 file    
 csp:
-  whitelist:        {}      # URL's which should be whitelisted           
-  flags:            {}      # Flags which should be set 
+  whitelist:                # URL's which should be whitelisted           
+    script-src-elem:  []
+    style-src:        []
+    font-src:         []
+    connect-src:      []
+    frame-src:        []
+  flags:                    # Flags which should be set
+    style-src:
+      unsafe-inline:  false
+    script-src:
+      unsafe-inline:  false
+    script-src-elem:
+      unsafe-inline:  false
+domains:
 domains:
   canonical:        {}      # Urls under which the domain should be directly accessible
   aliases:          []      # Alias redirections to the first element of the canonical domains

templates/roles/web-app/tasks/main.yml.j2

@@ -5,19 +5,19 @@
 - name: "load docker, db and proxy for '{{application_id}}'"
   include_role: 
     name: cmp-db-docker-proxy
-  when: run_once_docker_{% endraw %}{{ application_id }}{% raw %} is not defined
+  when: run_once_docker_{% endraw %}{{ application_id | replace("_", "-") }}{% raw %} is not defined
 {% endraw %}
 {% else %}
 {% raw %}
 - name: "load docker, proxy for '{{application_id}}'"
   include_role: 
     name: cmp-db-docker-proxy
-  when: run_once_docker_{% endraw %}{{ application_id }}{% raw %} is not defined
+  when: run_once_docker_{% endraw %}{{ application_id | replace("_", "-") }}{% raw %} is not defined
 {% endraw %}
 {% endif %}
 {% raw %}
-- name: run the {% endraw %}{{ application_id }}{% raw %} tasks once
+- name: run the {% endraw %}{{ application_id | replace("_", "-") }}{% raw %} tasks once
   set_fact:
-    run_once_docker_{% endraw %}{{ application_id }}{% raw %}: true
-  when: run_once_docker_{% endraw %}{{ application_id }}{% raw %} is not defined
+    run_once_docker_{% endraw %}{{ application_id | replace("_", "-") }}{% raw %}: true
+  when: run_once_docker_{% endraw %}{{ application_id | replace("_", "-") }}{% raw %} is not defined
 {% endraw %}

templates/roles/web-app/vars/main.yml.j2

@@ -8,13 +8,19 @@ application_id:                         {{ application_id }} # ID of the applica
 database_type:                          0                       # Database type [postgres, mariadb]
 
 # Docker
+# Checkout roles/docker-compose/defaults/main.yml for all configuration options
+
 docker_compose_flush_handlers:          true                    # When this is set to true an auto-flush after the docker-compose.yml, and env deploy is triggered, otherwise you have todo it manual.
 docker_compose_skipp_file_creation:     false                   # Skipp creation of docker-compose.yml file
 
+docker_pull_git_repository:                      true                    # This will automaticly pull a repository from the 'docker_repository_address'
+docker_repository_address:              ""                      # The address of a repository which should be pulled
 
-# The following variable mapping is optional, but imt makes it easier to read the code. 
-# I recommend, to use this mappings, but you can skipp it and access the config entries direct via get_app_conf
-{{ application_id | get_cymais_dir }}_version:   "{% raw %}{{ applications | get_app_conf(application_id, 'docker.services.{% endraw %}{{ application_id | get_cymais_dir }}{% raw %}.version',        True) }}"{% endraw %}
-{{ application_id | get_cymais_dir }}_image:     "{% raw %}{{ applications | get_app_conf(application_id, 'docker.services.{% endraw %}{{ application_id | get_cymais_dir }}{% raw %}.image',          True) }}"{% endraw %}
-{{ application_id | get_cymais_dir }}_name:      "{% raw %}{{ applications | get_app_conf(application_id, 'docker.services.{% endraw %}{{ application_id | get_cymais_dir }}{% raw %}.name',           True) }}"{% endraw %}
-{{ application_id | get_cymais_dir }}_volume:    "{% raw %}{{ applications | get_app_conf(application_id, 'docker.volumes.data',   True) }}"{% endraw %}
+{# The following variable mapping is optional, but it makes it easier to read the code.#}
+{# I recommend, to use this mappings, but you can skipp it and access the config entries direct via 'get_app_conf' #}
+
+# This variables had been autocreated. For a detailled explanation checkout the config/main.yml file 
+{{ application_id | get_entity_name }}_version:   "{% raw %}{{ applications | get_app_conf(application_id, 'docker.services.{% endraw %}{{ application_id | get_entity_name }}{% raw %}.version') }}"{% endraw %}
+{{ application_id | get_entity_name }}_image:     "{% raw %}{{ applications | get_app_conf(application_id, 'docker.services.{% endraw %}{{ application_id | get_entity_name }}{% raw %}.image') }}"{% endraw %}
+{{ application_id | get_entity_name }}_container: "{% raw %}{{ applications | get_app_conf(application_id, 'docker.services.{% endraw %}{{ application_id | get_entity_name }}{% raw %}.name') }}"{% endraw %}
+{{ application_id | get_entity_name }}_volume:    "{% raw %}{{ applications | get_app_conf(application_id, 'docker.volumes.data') }}"{% endraw %}

tests/integration/test_roles_have_meta_main.py

@@ -0,0 +1,22 @@
+import os
+import unittest
+
+ROLES_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../roles'))
+
+class TestRolesHaveMetaMain(unittest.TestCase):
+    def test_each_role_has_meta_main(self):
+        missing_meta = []
+        for role in os.listdir(ROLES_DIR):
+            role_path = os.path.join(ROLES_DIR, role)
+            if os.path.isdir(role_path):
+                meta_main = os.path.join(role_path, 'meta', 'main.yml')
+                if not os.path.isfile(meta_main):
+                    missing_meta.append(role)
+        if missing_meta:
+            self.fail(
+                "The following roles are missing meta/main.yml:\n" +
+                "\n".join(missing_meta)
+            )
+
+if __name__ == '__main__':
+    unittest.main()