svc-bkp-rmt-2-loc: migrate pull script to Python + add unit tests; lock down backup-provider ACLs

- Replace Bash pull-specific-host.sh with Python pull-specific-host.py (argparse, identical logic)
- Update role vars and runner template to call python script
- Add __init__.py files for test discovery/imports
- Add unittest: tests/unit/roles/svc-bkp-rmt-2-loc/files/test_pull_specific_host.py (mocks subprocess/os/time; covers success, no types, find-fail, retry-exhaustion)
- Backup provider SSH wrapper: align allowed ls path (backup-docker-to-local)
- Split user role tasks: 01_core (sudoers), 02_permissions_ssh (SSH keys + wrapper), 03_permissions_folders (ownership + default ACLs + depth-limited chown/chmod)
- Ensure default ACLs grant rwx to 'backup' and none to group/other; keep sudo rsync working

Ref: ChatGPT discussion (2025-10-14) — https://chatgpt.com/share/68ee920a-9b98-800f-8806-ddcfe0255149

group_vars/all/00_general.yml

@@ -76,8 +76,9 @@ _applications_nextcloud_oidc_flavor: >-
           False,
           'oidc_login'
           if applications
-            | get_app_conf('web-app-nextcloud','features.ldap',False, True)
-          else 'sociallogin'
+            | get_app_conf('web-app-nextcloud','features.ldap',False, True, True)
+          else 'sociallogin',
+          True
         )
   }}
 

group_vars/all/01_modes.yml

@@ -5,6 +5,6 @@ MODE_DUMMY:   false                       # Executes dummy/test routines instead
 MODE_UPDATE:  true                        # Executes updates
 MODE_DEBUG:   false                       # This enables debugging in ansible and in the apps, You SHOULD NOT enable this on production servers
 MODE_RESET:   false                       # Cleans up all Infinito.Nexus files. It's necessary to run to whole playbook and not particial roles when using this function.
-MODE_CLEANUP: "{{ MODE_DEBUG  | bool }}"  # Cleanup unused files and configurations
+MODE_CLEANUP: true                        # Cleanup unused files and configurations
 MODE_ASSERT:  "{{ MODE_DEBUG  | bool }}"  # Executes validation tasks during the run.
 MODE_BACKUP:  true                        # Executes the Backup before the deployment

module_utils/config_utils.py

@@ -24,7 +24,7 @@ class ConfigEntryNotSetError(AppConfigKeyError):
     pass
 
 
-def get_app_conf(applications, application_id, config_path, strict=True, default=None):
+def get_app_conf(applications, application_id, config_path, strict=True, default=None, skip_missing_app=False):
     # Path to the schema file for this application
     schema_path = os.path.join('roles', application_id, 'schema', 'main.yml')
 
@@ -133,6 +133,9 @@ def get_app_conf(applications, application_id, config_path, strict=True, default
     try:
         obj = applications[application_id]
     except KeyError:
+        if skip_missing_app:
+            # Simply return default instead of failing
+            return default if default is not None else False
         raise AppConfigKeyError(
             f"Application ID '{application_id}' not found in applications dict.\n"
             f"path_trace: {path_trace}\n"

roles/categories.yml

@@ -153,6 +153,11 @@ roles:
       description: "Core AI building blocks—model serving, OpenAI-compatible gateways, vector databases, orchestration, and chat UIs."
       icon: "fas fa-brain"
       invokable: true
+    bkp:
+      title: "Backup Services"
+      description: "Service-level backup and recovery components—handling automated data snapshots, remote backups, synchronization services, and backup orchestration across databases, files, and containers."
+      icon: "fas fa-database"
+      invokable: true
   user:
     title: "Users & Access"
     description: "User accounts & access control"

roles/svc-bkp-rmt-2-loc/files/pull-specific-host.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import subprocess
+import time
+import sys
+
+
+def run_command(command, capture_output=True, check=False, shell=True):
+    """Run a shell command and return its output as string."""
+    try:
+        result = subprocess.run(
+            command,
+            capture_output=capture_output,
+            shell=shell,
+            text=True,
+            check=check
+        )
+        return result.stdout.strip()
+    except subprocess.CalledProcessError as e:
+        if capture_output:
+            print(e.stdout)
+            print(e.stderr)
+        raise
+
+
+def pull_backups(hostname: str):
+    print(f"pulling backups from: {hostname}")
+    errors = 0
+
+    print("loading meta data...")
+    remote_host = f"backup@{hostname}"
+    print(f"host address:         {remote_host}")
+
+    remote_machine_id = run_command(f'ssh "{remote_host}" sha256sum /etc/machine-id')[:64]
+    print(f"remote machine id:    {remote_machine_id}")
+
+    general_backup_machine_dir = f"/Backups/{remote_machine_id}/"
+    print(f"backup dir:           {general_backup_machine_dir}")
+
+    try:
+        remote_backup_types = run_command(
+            f'ssh "{remote_host}" "find {general_backup_machine_dir} -maxdepth 1 -type d -execdir basename {{}} ;"'
+        ).splitlines()
+        print(f"backup types:          {' '.join(remote_backup_types)}")
+    except subprocess.CalledProcessError:
+        sys.exit(1)
+
+    for backup_type in remote_backup_types:
+        if backup_type == remote_machine_id:
+            continue
+
+        print(f"backup type:              {backup_type}")
+
+        general_backup_type_dir = f"{general_backup_machine_dir}{backup_type}/"
+        general_versions_dir = general_backup_type_dir
+
+        # local previous version
+        try:
+            local_previous_version_dir = run_command(f"ls -d {general_versions_dir}* | tail -1")
+        except subprocess.CalledProcessError:
+            local_previous_version_dir = ""
+        print(f"last local backup:      {local_previous_version_dir}")
+
+        # remote versions
+        remote_backup_versions = run_command(
+            f'ssh "{remote_host}" "ls -d /Backups/{remote_machine_id}/backup-docker-to-local/*"'
+        ).splitlines()
+        print(f"remote backup versions:   {' '.join(remote_backup_versions)}")
+
+        remote_last_backup_dir = remote_backup_versions[-1] if remote_backup_versions else ""
+        print(f"last remote backup:       {remote_last_backup_dir}")
+
+        remote_source_path = f"{remote_host}:{remote_last_backup_dir}/"
+        print(f"source path:              {remote_source_path}")
+
+        local_backup_destination_path = remote_last_backup_dir
+        print(f"backup destination:       {local_backup_destination_path}")
+
+        print("creating local backup destination folder...")
+        os.makedirs(local_backup_destination_path, exist_ok=True)
+
+        rsync_command = (
+            f'rsync -abP --delete --delete-excluded --rsync-path="sudo rsync" '
+            f'--link-dest="{local_previous_version_dir}" "{remote_source_path}" "{local_backup_destination_path}"'
+        )
+        print("starting backup...")
+        print(f"executing:                {rsync_command}")
+
+        retry_count = 0
+        max_retries = 12
+        retry_delay = 300  # 5 minutes
+        last_retry_start = 0
+        max_retry_duration = 43200  # 12 hours
+
+        rsync_exit_code = 1
+        while retry_count < max_retries:
+            print(f"Retry attempt: {retry_count + 1}")
+            if retry_count > 0:
+                current_time = int(time.time())
+                last_retry_duration = current_time - last_retry_start
+                if last_retry_duration >= max_retry_duration:
+                    print("Last retry took more than 12 hours, increasing max retries to 12.")
+                    max_retries = 12
+            last_retry_start = int(time.time())
+            rsync_exit_code = os.system(rsync_command)
+            if rsync_exit_code == 0:
+                break
+            retry_count += 1
+            time.sleep(retry_delay)
+
+        if rsync_exit_code != 0:
+            print(f"Error: rsync failed after {max_retries} attempts")
+            errors += 1
+
+    sys.exit(errors)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Pull backups from a remote backup host via rsync."
+    )
+    parser.add_argument(
+        "hostname",
+        help="Hostname from which backup should be pulled"
+    )
+    args = parser.parse_args()
+    pull_backups(args.hostname)
+
+
+if __name__ == "__main__":
+    main()

roles/svc-bkp-rmt-2-loc/files/sys-bkp-rmt-2-loc.sh

@@ -1,85 +0,0 @@
-#!/bin/bash
-# @param $1 hostname from which backup should be pulled
-
-echo "pulling backups from: $1" &&
-
-# error counter
-errors=0 &&
-
-echo "loading meta data..." &&
-
-remote_host="backup@$1" &&
-echo "host address:         $remote_host" &&
-
-remote_machine_id="$( (ssh "$remote_host" sha256sum /etc/machine-id) | head -c 64 )" &&
-echo "remote machine id:    $remote_machine_id" &&
-
-general_backup_machine_dir="/Backups/$remote_machine_id/" &&
-echo "backup dir:           $general_backup_machine_dir" &&
-
-remote_backup_types="$(ssh "$remote_host" "find $general_backup_machine_dir -maxdepth 1 -type d -execdir basename {} ;")" &&
-echo "backup types:          $remote_backup_types" || exit 1
-
-for backup_type in $remote_backup_types; do
-  if [ "$backup_type" != "$remote_machine_id" ]; then
-    echo "backup type:              $backup_type" &&
-    
-    general_backup_type_dir="$general_backup_machine_dir""$backup_type/" &&
-    general_versions_dir="$general_backup_type_dir" &&
-    local_previous_version_dir="$(ls -d $general_versions_dir* | tail -1)" &&
-    echo "last local backup:      $local_previous_version_dir" &&
-
-    remote_backup_versions="$(ssh "$remote_host" ls -d "$general_backup_type_dir"\*)" &&
-    echo "remote backup versions:   $remote_backup_versions" &&
-
-
-    remote_last_backup_dir=$(echo "$remote_backup_versions" | tail -1) &&
-    echo "last remote backup:       $remote_last_backup_dir" &&
-
-    remote_source_path="$remote_host:$remote_last_backup_dir/" &&
-    echo "source path:              $remote_source_path" &&
-
-    local_backup_destination_path=$remote_last_backup_dir &&
-    echo "backup destination:       $local_backup_destination_path" &&
-
-    echo "creating local backup destination folder..." &&
-    mkdir -vp "$local_backup_destination_path" &&
-
-    echo "starting backup..."
-    rsync_command='rsync -abP --delete --delete-excluded --rsync-path="sudo rsync" --link-dest="'$local_previous_version_dir'" "'$remote_source_path'" "'$local_backup_destination_path'"'
-
-    echo "executing:                $rsync_command"
-
-    retry_count=0
-    max_retries=12
-    retry_delay=300  # Retry delay in seconds (5 minutes)
-    last_retry_start=0
-    max_retry_duration=43200  # Maximum duration for a single retry attempt (12 hours)
-
-    while [[ $retry_count -lt $max_retries ]]; do
-      echo "Retry attempt: $((retry_count + 1))"
-      if [[ $retry_count -gt 0 ]]; then
-        current_time=$(date +%s)
-        last_retry_duration=$((current_time - last_retry_start))
-        if [[ $last_retry_duration -ge $max_retry_duration ]]; then
-          echo "Last retry took more than 12 hours, increasing max retries to 12."
-          max_retries=12
-        fi
-      fi
-      last_retry_start=$(date +%s)
-      eval "$rsync_command"
-      rsync_exit_code=$?
-      if [[ $rsync_exit_code -eq 0 ]]; then
-        break
-      fi
-      retry_count=$((retry_count + 1))
-      sleep $retry_delay
-    done
-
-    if [[ $rsync_exit_code -ne 0 ]]; then
-      echo "Error: rsync failed after $max_retries attempts"
-      ((errors += 1))
-    fi
-  fi
-done
-exit $errors;

roles/svc-bkp-rmt-2-loc/tasks/main.yml

@@ -10,15 +10,15 @@
   - include_tasks: utils/run_once.yml
   when: run_once_svc_bkp_rmt_2_loc is not defined
 
-- name: "create {{ DOCKER_BACKUP_REMOTE_2_LOCAL_DIR }}"
+- name: "Create Directory '{{ DOCKER_BACKUP_REMOTE_2_LOCAL_DIR }}'"
   file:
     path: "{{ DOCKER_BACKUP_REMOTE_2_LOCAL_DIR }}"
     state: directory
     mode: "0755"
 
-- name: create svc-bkp-rmt-2-loc.sh
+- name: "Deploy '{{ DOCKER_BACKUP_REMOTE_2_LOCAL_SCRIPT }}'"
   copy:
-    src: svc-bkp-rmt-2-loc.sh
+    src:  "{{ DOCKER_BACKUP_REMOTE_2_LOCAL_FILE }}"
     dest: "{{ DOCKER_BACKUP_REMOTE_2_LOCAL_SCRIPT }}"
     mode: "0755"
 

roles/svc-bkp-rmt-2-loc/templates/script.sh.j2

@@ -3,6 +3,6 @@
 hosts="{{ DOCKER_BACKUP_REMOTE_2_LOCAL_BACKUP_PROVIDERS | join(' ') }}";
 errors=0
 for host in $hosts; do
-  bash {{ DOCKER_BACKUP_REMOTE_2_LOCAL_SCRIPT }} $host || ((errors+=1));
+  python {{ DOCKER_BACKUP_REMOTE_2_LOCAL_SCRIPT }} $host || ((errors+=1));
 done;
 exit $errors;

roles/svc-bkp-rmt-2-loc/vars/main.yml

@@ -1,5 +1,9 @@
+# General
 application_id:                                 svc-bkp-rmt-2-loc
 system_service_id:                              "{{ application_id }}" 
+
+# Role Specific
 DOCKER_BACKUP_REMOTE_2_LOCAL_DIR:               '{{ PATH_ADMINISTRATOR_SCRIPTS }}{{ application_id }}/'
-DOCKER_BACKUP_REMOTE_2_LOCAL_SCRIPT:            "{{ DOCKER_BACKUP_REMOTE_2_LOCAL_DIR }}svc-bkp-rmt-2-loc.sh"
+DOCKER_BACKUP_REMOTE_2_LOCAL_FILE:              'pull-specific-host.py'
+DOCKER_BACKUP_REMOTE_2_LOCAL_SCRIPT:            "{{ [ DOCKER_BACKUP_REMOTE_2_LOCAL_DIR , DOCKER_BACKUP_REMOTE_2_LOCAL_FILE ] | path_join }}"
 DOCKER_BACKUP_REMOTE_2_LOCAL_BACKUP_PROVIDERS:  "{{ applications | get_app_conf(application_id, 'backup_providers')  }}"

roles/sys-bkp-provider-user/files/ssh-wrapper.sh

@@ -13,7 +13,7 @@ get_backup_types="find /Backups/$hashed_machine_id/ -maxdepth 1 -type d -execdir
 
 
 # @todo This configuration is not scalable yet. If other backup services then sys-ctl-bkp-docker-2-loc are integrated, this logic needs to be optimized
-get_version_directories="ls -d /Backups/$hashed_machine_id/sys-ctl-bkp-docker-2-loc/*"
+get_version_directories="ls -d /Backups/$hashed_machine_id/backup-docker-to-local/*"
 last_version_directory="$($get_version_directories | tail -1)"
 rsync_command="sudo rsync --server --sender -blogDtpre.iLsfxCIvu . $last_version_directory/"
 

roles/sys-bkp-provider-user/tasks/01_core.yml

@@ -3,30 +3,6 @@
     name: backup
     create_home: yes
 
-- name: create .ssh directory
-  file:
-    path: /home/backup/.ssh
-    state: directory
-    owner: backup
-    group: backup
-    mode: '0700'
-
-- name: create /home/backup/.ssh/authorized_keys
-  template:
-    src: "authorized_keys.j2"
-    dest: /home/backup/.ssh/authorized_keys
-    owner: backup
-    group: backup
-    mode: '0644'
-
-- name: create /home/backup/ssh-wrapper.sh
-  copy:
-    src: "ssh-wrapper.sh"
-    dest: /home/backup/ssh-wrapper.sh
-    owner: backup
-    group: backup
-    mode: '0700'
-
 - name: grant backup sudo rights
   copy:
     src: "backup"
@@ -35,3 +11,9 @@
     owner: root
     group: root
   notify: sshd restart
+
+- include_tasks: 02_permissions_ssh.yml
+
+- include_tasks: 03_permissions_folders.yml
+
+- include_tasks: utils/run_once.yml

roles/sys-bkp-provider-user/tasks/02_permissions_ssh.yml

@@ -0,0 +1,23 @@
+- name: create .ssh directory
+  file:
+    path: /home/backup/.ssh
+    state: directory
+    owner: backup
+    group: backup
+    mode: '0700'
+
+- name: create /home/backup/.ssh/authorized_keys
+  template:
+    src: "authorized_keys.j2"
+    dest: /home/backup/.ssh/authorized_keys
+    owner: backup
+    group: backup
+    mode: '0644'
+
+- name: create /home/backup/ssh-wrapper.sh
+  copy:
+    src: "ssh-wrapper.sh"
+    dest: /home/backup/ssh-wrapper.sh
+    owner: backup
+    group: backup
+    mode: '0700'

roles/sys-bkp-provider-user/tasks/03_permissions_folders.yml

@@ -0,0 +1,64 @@
+# Ensure the backups root exists and is owned by backup
+- name: Ensure backups root exists and owned by backup
+  file:
+    path: "{{ BACKUPS_FOLDER_PATH }}"
+    state: directory
+    owner: backup
+    group: backup
+    mode: "0700"
+
+# Explicit ACL so 'backup' has rwx, others none
+- name: Grant ACL rwx on backups root to backup user
+  ansible.posix.acl:
+    path: "{{ BACKUPS_FOLDER_PATH }}"
+    entity: backup
+    etype: user
+    permissions: rwx
+    state: present
+
+# Set default ACLs so new entries inherit rwx for backup and nothing for others
+- name: Set default ACL (inherit) for backup user under backups root
+  ansible.posix.acl:
+    path: "{{ BACKUPS_FOLDER_PATH }}"
+    entity: backup
+    etype: user
+    permissions: rwx
+    default: true
+    state: present
+
+# Remove default ACLs for group/others (defensive hardening)
+# Default ACLs so new entries inherit only backup's rwx
+- name: Default ACL for backup user (inherit)
+  ansible.posix.acl:
+    path: "{{ BACKUPS_FOLDER_PATH }}"
+    etype: user
+    entity: backup
+    permissions: rwx
+    default: true
+    state: present
+
+# Explicitly set default group/other to no permissions (instead of absent)
+- name: Default ACL for group -> none
+  ansible.posix.acl:
+    path: "{{ BACKUPS_FOLDER_PATH }}"
+    etype: group
+    permissions: '---'
+    default: true
+    state: present
+
+- name: Default ACL for other -> none
+  ansible.posix.acl:
+    path: "{{ BACKUPS_FOLDER_PATH }}"
+    etype: other
+    permissions: '---'
+    default: true
+    state: present
+
+- name: Fix ownership level 0..2 directories to backup:backup
+  ansible.builtin.shell: >
+    find "{{ BACKUPS_FOLDER_PATH }}" -mindepth 0 -maxdepth 2 -xdev -type d -exec chown backup:backup {} +
+
+- name: Fix perms level 0..2 directories to 0700
+  ansible.builtin.shell: >
+    find "{{ BACKUPS_FOLDER_PATH }}" -mindepth 0 -maxdepth 2 -xdev -type d -exec chmod 700 {} +
+

roles/sys-bkp-provider-user/tasks/main.yml

@@ -1,4 +1,2 @@
-- block:
 - include_tasks: 01_core.yml
-    - include_tasks: utils/run_once.yml
   when: run_once_sys_bkp_provider_user is not defined

roles/sys-ctl-alm-email/tasks/01_core.yml

@@ -1,8 +1,7 @@
 - name: Include dependencies
   include_role:
-    name: '{{ item }}'
-  loop:
-  - sys-svc-msmtp
+    name: "sys-svc-msmtp"
+  when: run_once_sys_svc_msmtp is not defined or run_once_sys_svc_msmtp is false
 
 - include_role:
     name: sys-service

roles/sys-ctl-cln-disc-space/templates/script.sh.j2

@@ -39,6 +39,18 @@ if [ "$force_freeing" = true ]; then
       docker exec -u www-data $nextcloud_application_container /var/www/html/occ versions:cleanup || exit 6
     fi
 
+    # Mastodon cleanup (remote media cache)
+    mastodon_application_container="{{ applications | get_app_conf('web-app-mastodon', 'docker.services.mastodon.name') }}"
+    mastodon_cleanup_days="1"
+
+    if [ -n "$mastodon_application_container" ] && docker ps -a --format '{% raw %}{{.Names}}{% endraw %}' | grep -qw "$mastodon_application_container"; then
+      echo "Cleaning up Mastodon media cache (older than ${mastodon_cleanup_days} days)" &&
+      docker exec -u root "$mastodon_application_container" bash -lc "bin/tootctl media remove --days=${mastodon_cleanup_days}" || exit 8
+
+      # Optional: additionally remove local thumbnail/cache files older than X days
+      # Warning: these will be regenerated when accessed, which may cause extra CPU/I/O load
+      # docker exec -u root "$mastodon_application_container" bash -lc "find /mastodon/public/system/cache -type f -mtime +${mastodon_cleanup_days} -delete" || exit 9
+    fi
   fi
 
   if command -v pacman >/dev/null 2>&1 ; then

roles/sys-front-inj-all/tasks/01_dependencies.yml

@@ -0,0 +1,16 @@
+- name: "Load CDN for '{{ domain }}'"
+  include_role:
+    name: web-svc-cdn
+    public: false
+  when:
+    - application_id != 'web-svc-cdn'
+    - run_once_web_svc_cdn is not defined
+
+- name: Load Logout for '{{ domain }}'
+  include_role:
+    name: web-svc-logout
+    public: false
+  when: 
+    - run_once_web_svc_logout is not defined
+    - application_id != 'web-svc-logout'
+    - inj_enabled.logout

roles/sys-front-inj-all/tasks/main.yml

@@ -1,22 +1,41 @@
+- block:
+    - name: Include dependency 'sys-svc-webserver-core'
+      include_role:
+        name: sys-svc-webserver-core
+      when: run_once_sys_svc_webserver_core is not defined
+    - include_tasks: utils/run_once.yml
+  when: run_once_sys_front_inj_all is not defined
+
 - name: Build inj_enabled
   set_fact:
     inj_enabled: "{{ applications | inj_enabled(application_id, SRV_WEB_INJ_COMP_FEATURES_ALL) }}"
 
-- name: "Load CDN Service for '{{ domain }}'"
-  include_role:
-    name: sys-svc-cdn
-    public: true # Expose variables so that they can be used in all injection roles
+- name: "Included dependent services"
+  include_tasks: 01_dependencies.yml
+  vars:
+    proxy_extra_configuration: ""
 
-- name: Reinitialize 'inj_enabled' for '{{ domain }}', after modification by CDN
+- name: Reinitialize 'inj_enabled' for '{{ domain }}', after loading the required webservices
   set_fact:
     inj_enabled: "{{ applications | inj_enabled(application_id, SRV_WEB_INJ_COMP_FEATURES_ALL) }}"
     inj_head_features: "{{ SRV_WEB_INJ_COMP_FEATURES_ALL | inj_features('head') }}"
     inj_body_features: "{{ SRV_WEB_INJ_COMP_FEATURES_ALL | inj_features('body') }}"
 
+- name: "Load CDN Service for '{{ domain }}'"
+  include_role:
+    name: sys-svc-cdn
+    public: true
+
+- name: "Activate logout proxy for '{{ domain }}'"
+  include_role:
+    name: sys-front-inj-logout
+    public: true
+  when: inj_enabled.logout
+
 - name: "Activate Desktop iFrame notifier for '{{ domain }}'"
   include_role:
     name:   sys-front-inj-desktop
-    public: true # Vars used in templates
+    public: true
   when: inj_enabled.desktop
 
 - name: "Activate Corporate CSS for '{{ domain }}'"
@@ -33,17 +52,3 @@
   include_role:
     name: sys-front-inj-javascript
   when: inj_enabled.javascript
-
-- name: "Activate logout proxy for '{{ domain }}'"
-  include_role:
-    name: sys-front-inj-logout
-    public: true # Vars used in templates
-  when: inj_enabled.logout
-
-- block:
-  - name: Include dependency 'sys-svc-webserver-core'
-    include_role:
-      name: sys-svc-webserver-core
-    when: run_once_sys_svc_webserver_core is not defined
-  - include_tasks: utils/run_once.yml
-  when: run_once_sys_front_inj_all is not defined

roles/sys-front-inj-css/tasks/01_core.yml

@@ -1,8 +1,3 @@
-- name: Include dependency 'sys-svc-webserver-core'
-  include_role:
-    name: sys-svc-webserver-core
-  when: run_once_sys_svc_webserver_core is not defined
-
 - name: Generate color palette with colorscheme-generator
   set_fact:
     color_palette: "{{ lookup('colorscheme', CSS_BASE_COLOR, count=CSS_COUNT, shades=CSS_SHADES) }}"
@@ -19,3 +14,5 @@
     group:  "{{ NGINX.USER }}"
     mode:   '0644'
   loop: "{{ CSS_FILES }}"
+
+- include_tasks: utils/run_once.yml

roles/sys-front-inj-css/tasks/main.yml

@@ -1,6 +1,4 @@
-- block:
 - include_tasks: 01_core.yml
-    - include_tasks: utils/run_once.yml
   when: run_once_sys_front_inj_css is not defined
 
 - name: "Resolve optional app style.css source for '{{ application_id }}'"

roles/sys-front-inj-css/templates/head_sub.j2

@@ -3,6 +3,6 @@
 {% for css_file in ['default.css','bootstrap.css'] %}
 <link rel="stylesheet" href="{{ [ cdn_urls.shared.css, css_file, lookup('local_mtime_qs', [__css_tpl_dir, css_file ~ '.j2'] | path_join)] | url_join }}">
 {% endfor %}
-{% if app_style_present | bool %}
+{% if app_style_present | default(false) | bool %}
 <link rel="stylesheet" href="{{ [ cdn_urls.role.release.css, 'style.css', lookup('local_mtime_qs', app_style_src)] | url_join }}">
 {% endif %}

roles/sys-front-inj-desktop/tasks/main.yml

@@ -1,8 +1,4 @@
 - block:
-  - name: Include dependency 'sys-svc-webserver-core'
-    include_role:
-      name: sys-svc-webserver-core
-    when: run_once_sys_svc_webserver_core is not defined
   - include_tasks: 01_deploy.yml
   - include_tasks: utils/run_once.yml
   when: run_once_sys_front_inj_desktop is not defined

roles/sys-front-inj-javascript/tasks/main.yml

@@ -1,11 +1,4 @@
-- block:
-
-  - name: Include dependency 'sys-svc-webserver-core'
-    include_role:
-      name: sys-svc-webserver-core
-    when: run_once_sys_svc_webserver_core is not defined
-  - include_tasks: utils/run_once.yml
-  when: run_once_sys_front_inj_javascript is not defined
+# run_once_sys_front_inj_javascript: deactivated
 
 - name: "Load JavaScript code for '{{ application_id }}'"
   set_fact:

roles/sys-front-inj-logout/tasks/01_core.yml

@@ -1,8 +1,6 @@
-- name: Include dependency 'sys-svc-webserver-core'
-  include_role:
-    name: sys-svc-webserver-core
-  when: 
-    - run_once_sys_svc_webserver_core is not defined
-  
 - name: "deploy the logout.js"
   include_tasks: "02_deploy.yml"
+
+- set_fact:
+    run_once_sys_front_inj_logout: true
+  changed_when: false

roles/sys-front-inj-logout/tasks/02_deploy.yml

@@ -1,6 +1,6 @@
 - name: Deploy logout.js
-  template:
-    src: logout.js.j2
+  copy:
+    src:    logout.js
     dest:   "{{ INJ_LOGOUT_JS_DESTINATION }}"
     owner:  "{{ NGINX.USER }}"
     group:  "{{ NGINX.USER }}"

roles/sys-front-inj-logout/tasks/main.yml

@@ -1,16 +1,16 @@
-- block:
-  - include_tasks: 01_core.yml
-  - set_fact:
-      run_once_sys_front_inj_logout: true
+- name: "Load base for '{{ application_id }}'"
+  include_tasks: 01_core.yml
   when: run_once_sys_front_inj_logout is not defined
 
 - name: "Load logout code for '{{ application_id }}'"
   set_fact:
     logout_code: "{{ lookup('template', 'logout_one_liner.js.j2') }}"
+  changed_when: false
 
 - name: "Collapse logout code into one-liner for '{{ application_id }}'"
   set_fact:
     logout_code_one_liner: "{{ logout_code | to_one_liner }}"
+  changed_when: false
 
 - name: "Append logout CSP hash for '{{ application_id }}'"
   set_fact:

roles/sys-front-inj-logout/templates/head_sub.j2

@@ -1 +1 @@
-<script src="{{ cdn_urls.shared.js }}/{{ INJ_LOGOUT_JS_FILE_NAME }}{{ lookup('local_mtime_qs', [playbook_dir, 'roles', 'sys-front-inj-logout', 'templates', INJ_LOGOUT_JS_FILE_NAME ~ '.j2'] | path_join) }}"></script>
+<script src="{{ cdn_urls.shared.js }}/{{ INJ_LOGOUT_JS_FILE_NAME }}{{ lookup('local_mtime_qs', [playbook_dir, 'roles', 'sys-front-inj-logout', 'files', INJ_LOGOUT_JS_FILE_NAME] | path_join) }}"></script>

roles/sys-front-inj-matomo/tasks/main.yml

@@ -1,10 +1,4 @@
-- block:
-  - name: Include dependency 'sys-svc-webserver-core'
-    include_role:
-      name: sys-svc-webserver-core
-    when: run_once_sys_svc_webserver_core is not defined
-  - include_tasks: utils/run_once.yml
-  when: run_once_sys_front_inj_matomo is not defined
+# run_once_sys_front_inj_matomo: deactivated
 
 - name: "Relevant variables for role: {{ role_path | basename }}"
   debug:

roles/sys-svc-cdn/tasks/01_core.yml

@@ -1,21 +0,0 @@
-- name: "Load CDN for '{{ domain }}'"
-  include_role:
-    name: web-svc-cdn
-    public: false
-  when:
-  - application_id != 'web-svc-cdn'
-  - run_once_web_svc_cdn is not defined
-
-# ------------------------------------------------------------------
-# Only-once creations (shared root and vendor)
-# ------------------------------------------------------------------
-- name: Ensure shared root and vendor exist (run once)
-  ansible.builtin.file:
-    path:  "{{ item }}"
-    state: directory
-    owner: "{{ NGINX.USER }}"
-    group: "{{ NGINX.USER }}"
-    mode:  "0755"
-  loop: "{{ CDN_DIRS_GLOBAL }}"
-
-- include_tasks: utils/run_once.yml

roles/sys-svc-cdn/tasks/main.yml

@@ -1,6 +1,14 @@
 ---
 - block:
-    - include_tasks: 01_core.yml
+    - name: Ensure shared root and vendor exist (run once)
+      ansible.builtin.file:
+        path:  "{{ item }}"
+        state: directory
+        owner: "{{ NGINX.USER }}"
+        group: "{{ NGINX.USER }}"
+        mode:  "0755"
+      loop: "{{ CDN_DIRS_GLOBAL }}"
+    - include_tasks: utils/run_once.yml
   when:
     - run_once_sys_svc_cdn is not defined
 

roles/sys-svc-msmtp/tasks/01_core.yml

@@ -15,3 +15,6 @@
 - include_role:
     name: sys-ctl-hlth-msmtp
   when: run_once_sys_ctl_hlth_msmtp is not defined
+
+- set_fact:
+    run_once_sys_svc_msmtp: true

roles/sys-svc-msmtp/tasks/main.yml

@@ -1,5 +1,6 @@
-- block:
-  - include_tasks: 01_core.yml
-  - set_fact:
-      run_once_sys_svc_msmtp: true
-  when: run_once_sys_svc_msmtp is not defined
+- name: "Load MSMTP Core Once"
+  include_tasks: 01_core.yml
+  when:
+  - run_once_sys_svc_msmtp is not defined or run_once_sys_svc_msmtp is false
+  # Just execute when mailu_token is defined
+  - users['no-reply'].mailu_token is defined

roles/user/users/main.yml

@@ -5,7 +5,7 @@ users:
     username: "{{ PRIMARY_DOMAIN.split('.')[0] }}"
   tld:
     description: "Auto Generated Account to reserve the TLD"
-    username: "{{ PRIMARY_DOMAIN.split('.')[1] }}"
+    username: "{{ PRIMARY_DOMAIN.split('.')[1] if (PRIMARY_DOMAIN is defined and (PRIMARY_DOMAIN.split('.') | length) > 1) else (PRIMARY_DOMAIN ~ '_tld ') }}"
   root:
     username: root
     uid: 0

roles/web-app-desktop/tasks/01_core.yml

@@ -17,6 +17,8 @@
 - name: "load docker, proxy for '{{ application_id }}'"
   include_role:
     name: sys-stk-full-stateless
+  vars:
+    docker_compose_flush_handlers: false
 
 - name: "Check if host-specific config.yaml exists in {{ DESKTOP_CONFIG_INV_PATH }}"
   stat:
@@ -57,8 +59,16 @@
   notify: docker compose up
   when: not config_file.stat.exists
 
-- name: add docker-compose.yml
-  template:
-    src:  docker-compose.yml.j2 
-    dest: "{{ docker_compose.directories.instance }}docker-compose.yml"
-  notify: docker compose up
+- name: "Flush docker compose handlers"
+  meta: flush_handlers
+
+- name: Wait for Desktop HTTP endpoint (required so all logos can be downloaded during initialization)
+  uri:
+    url: "http://127.0.0.1:{{ http_port }}/"
+    status_code: 200
+  register: desktop_http
+  retries: 60
+  delay: 5
+  until: desktop_http.status == 200
+
+- include_tasks: utils/run_once.yml

roles/web-app-desktop/tasks/main.yml

@@ -1,5 +1,3 @@
 ---
-- block:
 - include_tasks: 01_core.yml  
-    - include_tasks: utils/run_once.yml
   when: run_once_web_app_desktop is not defined

roles/web-app-desktop/vars/main.yml

@@ -1,5 +1,6 @@
 # General
 application_id:                   "web-app-desktop"
+http_port:                        "{{ ports.localhost.http[application_id] }}"
 
 ## Webserver
 proxy_extra_configuration:        "{{ lookup('template', 'nginx/sso.html.conf.j2') }}"

roles/web-app-discourse/config/main.yml

@@ -43,9 +43,10 @@ plugins:
     enabled:  true
   discourse-akismet:
     enabled:  true
-  discourse-cakeday:
-    enabled:  true
-#  discourse-solved: Seems like this plugin is now also part of the default setup
+# The following plugins moved to the default setup
+#  discourse-cakeday:
+#    enabled:  true
+#  discourse-solved:
 #    enabled:  true 
 #  discourse-voting:
 #    enabled:  true

roles/web-app-discourse/tasks/01_core.yml

@@ -7,3 +7,5 @@
 
 - name: "Setup '{{ application_id }}' network"
   include_tasks: 04_network.yml
+
+- include_tasks: utils/run_once.yml

roles/web-app-discourse/tasks/main.yml

@@ -1,6 +1,4 @@
 ---
 - name: "Setup {{ application_id }}"
+  include_tasks: 01_core.yml
   when: run_once_web_app_discourse is not defined
-  block:
-  - include_tasks: 01_core.yml
-  - include_tasks: utils/run_once.yml

roles/web-app-espocrm/files/docker-entrypoint-custom.sh

@@ -1,11 +1,13 @@
 #!/bin/sh
-set -euo pipefail
+# POSIX-safe entrypoint for EspoCRM container
+# Compatible with /bin/sh (dash/busybox). Avoids 'pipefail' and non-portable features.
+set -eu
 
 log() { printf '%s %s\n' "[entrypoint]" "$*" >&2; }
 
 # --- Simple boolean normalization --------------------------------------------
 bool_norm () {
-  v="$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')"
+  v="$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]' 2>/dev/null || true)"
   case "$v" in
     1|true|yes|on)  echo "true" ;;
     0|false|no|off|"") echo "false" ;;
@@ -13,30 +15,45 @@ bool_norm () {
   esac
 }
 
-# Expected ENV (from env.j2)
+# --- Environment initialization ----------------------------------------------
 MAINTENANCE="$(bool_norm "${ESPO_INIT_MAINTENANCE_MODE:-false}")"
 CRON_DISABLED="$(bool_norm "${ESPO_INIT_CRON_DISABLED:-false}")"
 USE_CACHE="$(bool_norm "${ESPO_INIT_USE_CACHE:-true}")"
 
 APP_DIR="/var/www/html"
-SET_FLAGS_SCRIPT="${ESPOCRM_SET_FLAGS_SCRIPT}"
+
+# Provided by env.j2 (fallback ensures robustness)
+SET_FLAGS_SCRIPT="${ESPOCRM_SET_FLAGS_SCRIPT:-/usr/local/bin/set_flags.php}"
+if [ ! -f "$SET_FLAGS_SCRIPT" ]; then
+  log "WARN: SET_FLAGS_SCRIPT '$SET_FLAGS_SCRIPT' not found; falling back to /usr/local/bin/set_flags.php"
+  SET_FLAGS_SCRIPT="/usr/local/bin/set_flags.php"
+fi
 
 # --- Wait for bootstrap.php (max 60s, e.g. fresh volume) ----------------------
 log "Waiting for ${APP_DIR}/bootstrap.php..."
-for i in $(seq 1 60); do
-  [ -f "${APP_DIR}/bootstrap.php" ] && break
+count=0
+while [ $count -lt 60 ] && [ ! -f "${APP_DIR}/bootstrap.php" ]; do
   sleep 1
+  count=$((count + 1))
 done
 if [ ! -f "${APP_DIR}/bootstrap.php" ]; then
-  log "ERROR: bootstrap.php missing after 60s"; exit 1
+  log "ERROR: bootstrap.php missing after 60s"
+  exit 1
 fi
 
 # --- Apply config flags via set_flags.php ------------------------------------
 log "Applying runtime flags via set_flags.php..."
-php "${SET_FLAGS_SCRIPT}"
+if ! php "${SET_FLAGS_SCRIPT}"; then
+  log "ERROR: set_flags.php execution failed"
+  exit 1
+fi
 
 # --- Clear cache (safe) -------------------------------------------------------
-php "${APP_DIR}/clear_cache.php" || true
+if php "${APP_DIR}/clear_cache.php" 2>/dev/null; then
+  log "Cache cleared successfully."
+else
+  log "WARN: Cache clearing skipped or failed (non-critical)."
+fi
 
 # --- Hand off to CMD ----------------------------------------------------------
 if [ "$#" -gt 0 ]; then
@@ -56,5 +73,6 @@ for cmd in apache2-foreground httpd-foreground php-fpm php-fpm8.3 php-fpm8.2 sup
   fi
 done
 
+# --- Fallback ---------------------------------------------------------------
 log "No known server command found; tailing to keep container alive."
 exec tail -f /dev/null

roles/web-app-keycloak/config/main.yml

@@ -50,3 +50,9 @@ credentials:
   recaptcha:
     website_key:    "" # Required if you enabled recaptcha:
     secret_key:     ""  # Required if you enabled recaptcha:
+
+accounts:
+  bootstrap:
+    username: "administrator"
+  system:
+    username: "{{ SOFTWARE_NAME | replace('.', '_') | lower }}"

roles/web-app-keycloak/tasks/05_login.yml

@@ -0,0 +1,89 @@
+- name: "Wait until '{{ KEYCLOAK_CONTAINER }}' container is healthy"
+  community.docker.docker_container_info:
+    name: "{{ KEYCLOAK_CONTAINER }}"
+  register: kc_info
+  retries: 60
+  delay: 5
+  until: >
+    kc_info is succeeded and
+    (kc_info.container | default({})) != {} and
+    (kc_info.container.State | default({})) != {} and
+    (kc_info.container.State.Health | default({})) != {} and
+    (kc_info.container.State.Health.Status | default('')) == 'healthy'
+
+- name: Ensure permanent Keycloak admin exists and can log in (container env only)
+  block:
+
+    - name: Try login with permanent admin (uses container ENV)
+      shell: |
+        {{ KEYCLOAK_EXEC_CONTAINER }} sh -lc '
+          {{ KEYCLOAK_KCADM }} config credentials \
+            --server {{ KEYCLOAK_SERVER_INTERNAL_URL }} \
+            --realm master \
+            --user "$KEYCLOAK_PERMANENT_ADMIN_USERNAME" \
+            --password "$KEYCLOAK_PERMANENT_ADMIN_PASSWORD"
+        '
+      register: kc_login_perm
+      changed_when: false
+
+  rescue:
+
+    - name: Login with bootstrap admin (uses container ENV)
+      shell: |
+        {{ KEYCLOAK_EXEC_CONTAINER }} sh -lc '
+          {{ KEYCLOAK_KCADM }} config credentials \
+            --server {{ KEYCLOAK_SERVER_INTERNAL_URL }} \
+            --realm master \
+            --user "$KC_BOOTSTRAP_ADMIN_USERNAME" \
+            --password "$KC_BOOTSTRAP_ADMIN_PASSWORD"
+        '
+      register: kc_login_bootstrap
+      changed_when: false
+
+    - name: Ensure permanent admin user exists (create if missing)
+      shell: |
+        {{ KEYCLOAK_EXEC_CONTAINER }} sh -lc '
+          {{ KEYCLOAK_KCADM }} create users -r master \
+            -s "username=$KEYCLOAK_PERMANENT_ADMIN_USERNAME" \
+            -s "enabled=true"
+        '
+      register: kc_create_perm_admin
+      failed_when: >
+        not (
+          kc_create_perm_admin.rc == 0 or
+          (kc_create_perm_admin.stderr is defined and
+          ('User exists with same username' in kc_create_perm_admin.stderr))
+        )
+      changed_when: kc_create_perm_admin.rc == 0
+
+    - name: Set permanent admin password (by username, no ID needed)
+      shell: |
+        {{ KEYCLOAK_EXEC_CONTAINER }} sh -lc '
+          {{ KEYCLOAK_KCADM }} set-password -r master \
+            --username "$KEYCLOAK_PERMANENT_ADMIN_USERNAME" \
+            --new-password "$KEYCLOAK_PERMANENT_ADMIN_PASSWORD"
+        '
+      changed_when: true
+
+    - name: Grant global admin via master realm role 'admin'
+      shell: |
+        {{ KEYCLOAK_EXEC_CONTAINER }} sh -lc '
+          {{ KEYCLOAK_KCADM }} add-roles -r master \
+            --uusername "$KEYCLOAK_PERMANENT_ADMIN_USERNAME" \
+            --rolename admin
+        '
+      register: kc_grant_master_admin
+      changed_when: (kc_grant_master_admin.stderr is defined and kc_grant_master_admin.stderr | length > 0) or
+                    (kc_grant_master_admin.stdout is defined and kc_grant_master_admin.stdout | length > 0)
+      failed_when: false
+
+    - name: Verify login with permanent admin (after creation)
+      shell: |
+        {{ KEYCLOAK_EXEC_CONTAINER }} sh -lc '
+          {{ KEYCLOAK_KCADM }} config credentials \
+            --server {{ KEYCLOAK_SERVER_INTERNAL_URL }} \
+            --realm master \
+            --user "$KEYCLOAK_PERMANENT_ADMIN_USERNAME" \
+            --password "$KEYCLOAK_PERMANENT_ADMIN_PASSWORD"
+        '
+      changed_when: false

roles/web-app-keycloak/tasks/main.yml

@@ -13,82 +13,21 @@
   include_tasks: 04_dependencies.yml
   when: KEYCLOAK_LOAD_DEPENDENCIES | bool
 
-- name: "Wait until '{{ KEYCLOAK_CONTAINER }}' container is healthy"
-  community.docker.docker_container_info:
-    name: "{{ KEYCLOAK_CONTAINER }}"
-  register: kc_info
-  retries: 60
-  delay: 5
-  until: >
-    kc_info is succeeded and
-    (kc_info.container | default({})) != {} and
-    (kc_info.container.State | default({})) != {} and
-    (kc_info.container.State.Health | default({})) != {} and
-    (kc_info.container.State.Health.Status | default('')) == 'healthy'
+- name: "Load Login routines for '{{ application_id }}'"
+  include_tasks: 05_login.yml
 
-- name: kcadm login (master)
-  no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
-  shell: >
-    {{ KEYCLOAK_EXEC_KCADM }} config credentials
-    --server {{ KEYCLOAK_SERVER_INTERNAL_URL }}
-    --realm master
-    --user {{ KEYCLOAK_MASTER_API_USER_NAME }}
-    --password {{ KEYCLOAK_MASTER_API_USER_PASSWORD }}
-  changed_when: false
+- name: "Load Client Update routines for '{{ application_id }}'"
+  include_tasks: update/01_client.yml
 
-- name: "Update Client settings"
-  vars:
-    kc_object_kind:  "client"
-    kc_lookup_value: "{{ KEYCLOAK_CLIENT_ID }}"
-    kc_desired: >-
-      {{
-        KEYCLOAK_DICTIONARY_REALM.clients
-          | selectattr('clientId','equalto', KEYCLOAK_CLIENT_ID)
-          | list | first
-      }}
-    kc_force_attrs:
-      publicClient: >-
-        {{
-          (KEYCLOAK_DICTIONARY_REALM.clients
-            | selectattr('clientId','equalto', KEYCLOAK_CLIENT_ID)
-            | map(attribute='publicClient')
-            | first)
-        }}
-      serviceAccountsEnabled: >-
-        {{
-          (KEYCLOAK_DICTIONARY_REALM.clients
-            | selectattr('clientId','equalto', KEYCLOAK_CLIENT_ID)
-            | map(attribute='serviceAccountsEnabled')
-            | first )
-        }}
-      frontchannelLogout:  >-
-        {{
-          (KEYCLOAK_DICTIONARY_REALM.clients
-            | selectattr('clientId','equalto', KEYCLOAK_CLIENT_ID)
-            | map(attribute='frontchannelLogout')
-            | first)
-        }}
-      attributes: >-
-        {{
-          ( (KEYCLOAK_DICTIONARY_REALM.clients
-              | selectattr('clientId','equalto', KEYCLOAK_CLIENT_ID)
-              | list | first | default({}) ).attributes | default({}) )
-          | combine({'frontchannel.logout.url': KEYCLOAK_FRONTCHANNEL_LOGOUT_URL}, recursive=True)
-        }}
-  include_tasks: _update.yml
+- name: "Load Mail Update routines for '{{ application_id }} - {{ KEYCLOAK_REALM }}'"
+  include_tasks: update/02_mail_realm.yml
 
-- name: "Update REALM mail settings from realm dictionary (SPOT)"
-  include_tasks: _update.yml
-  vars:
-    kc_object_kind:  "realm"
-    kc_lookup_field: "id"
-    kc_lookup_value: "{{ KEYCLOAK_REALM }}"
-    kc_desired:
-      smtpServer: "{{ KEYCLOAK_DICTIONARY_REALM.smtpServer | default({}, true) }}"
-    kc_merge_path:  "smtpServer"
-  no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
+- name: "Load Mail Update routines for '{{ application_id }} - master'"
+  include_tasks: update/03_mail_master.yml
 
-- include_tasks: 05_rbac_client_scope.yml
+- name: "Load RBAC Update routines for '{{ application_id }}'"
+  include_tasks: update/04_rbac_client_scope.yml
 
-- include_tasks: 06_ldap.yml
+- name: "Load LDAP Update routines for '{{ application_id }}'"
+  include_tasks: update/05_ldap.yml
   when: KEYCLOAK_LDAP_ENABLED | bool

roles/web-app-keycloak/tasks/update/01_client.yml

@@ -0,0 +1,40 @@
+- name: "Update Client settings"
+  vars:
+    kc_object_kind:  "client"
+    kc_lookup_value: "{{ KEYCLOAK_CLIENT_ID }}"
+    kc_desired: >-
+      {{
+        KEYCLOAK_DICTIONARY_REALM.clients
+          | selectattr('clientId','equalto', KEYCLOAK_CLIENT_ID)
+          | list | first
+      }}
+    kc_force_attrs:
+      publicClient: >-
+        {{
+          (KEYCLOAK_DICTIONARY_REALM.clients
+            | selectattr('clientId','equalto', KEYCLOAK_CLIENT_ID)
+            | map(attribute='publicClient')
+            | first)
+        }}
+      serviceAccountsEnabled: >-
+        {{
+          (KEYCLOAK_DICTIONARY_REALM.clients
+            | selectattr('clientId','equalto', KEYCLOAK_CLIENT_ID)
+            | map(attribute='serviceAccountsEnabled')
+            | first )
+        }}
+      frontchannelLogout:  >-
+        {{
+          (KEYCLOAK_DICTIONARY_REALM.clients
+            | selectattr('clientId','equalto', KEYCLOAK_CLIENT_ID)
+            | map(attribute='frontchannelLogout')
+            | first)
+        }}
+      attributes: >-
+        {{
+          ( (KEYCLOAK_DICTIONARY_REALM.clients
+              | selectattr('clientId','equalto', KEYCLOAK_CLIENT_ID)
+              | list | first | default({}) ).attributes | default({}) )
+          | combine({'frontchannel.logout.url': KEYCLOAK_FRONTCHANNEL_LOGOUT_URL}, recursive=True)
+        }}
+  include_tasks: _update.yml

roles/web-app-keycloak/tasks/update/02_mail_realm.yml

@@ -0,0 +1,10 @@
+- name: "Update {{ KEYCLOAK_REALM }} REALM mail settings from realm dictionary"
+  include_tasks: _update.yml
+  vars:
+    kc_object_kind:  "realm"
+    kc_lookup_field: "id"
+    kc_lookup_value: "{{ KEYCLOAK_REALM }}"
+    kc_desired:
+      smtpServer: "{{ KEYCLOAK_DICTIONARY_REALM.smtpServer | default({}, true) }}"
+    kc_merge_path:  "smtpServer"
+  no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"

roles/web-app-keycloak/tasks/update/03_mail_master.yml

@@ -0,0 +1,10 @@
+- name: "Update Master REALM mail settings from realm dictionary"
+  include_tasks: _update.yml
+  vars:
+    kc_object_kind:  "realm"
+    kc_lookup_field: "id"
+    kc_lookup_value: "master"
+    kc_desired:
+      smtpServer: "{{ KEYCLOAK_DICTIONARY_REALM.smtpServer | default({}, true) }}"
+    kc_merge_path:  "smtpServer"
+  no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"

roles/web-app-keycloak/tasks/update/04_rbac_client_scope.yml

@@ -1,4 +1,3 @@
-# --- Ensure RBAC client scope exists (idempotent) ---
 - name: Ensure RBAC client scope exists
   shell: |
     cat <<'JSON' | {{ KEYCLOAK_EXEC_KCADM }} create client-scopes -r {{ KEYCLOAK_REALM }} -f -
@@ -16,7 +15,6 @@
                ('already exists' not in (create_rbac_scope.stderr | lower))
   no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
 
-# --- Get the scope id we will attach to the client ---
 - name: Get all client scopes
   shell: "{{ KEYCLOAK_EXEC_KCADM }} get client-scopes -r {{ KEYCLOAK_REALM }} --format json"
   register: all_scopes

roles/web-app-keycloak/templates/env.j2

@@ -10,19 +10,21 @@ KC_HTTP_ENABLED=                true
 KC_HEALTH_ENABLED=              {{ KEYCLOAK_HEALTH_ENABLED | lower }}
 KC_METRICS_ENABLED=             true
 
-# Administrator
-KEYCLOAK_ADMIN=                 "{{ KEYCLOAK_ADMIN }}"
-KEYCLOAK_ADMIN_PASSWORD=        "{{ KEYCLOAK_ADMIN_PASSWORD }}"
-
 # Database
 KC_DB=                          {{ database_type }}
 KC_DB_URL=                      {{ database_url_jdbc }}
 KC_DB_USERNAME=                 {{ database_username }}
 KC_DB_PASSWORD=                 {{ database_password }}
 
-# If the initial administrator already exists and the environment variables are still present at startup, an error message stating the failed creation of the initial administrator is shown in the logs. Keycloak ignores the values and starts up correctly.
-KC_BOOTSTRAP_ADMIN_USERNAME=    "{{ KEYCLOAK_ADMIN }}"
-KC_BOOTSTRAP_ADMIN_PASSWORD=    "{{ KEYCLOAK_ADMIN_PASSWORD }}"
+# Credentials
+
+## Bootstrap
+KC_BOOTSTRAP_ADMIN_USERNAME="{{ KEYCLOAK_BOOTSTRAP_ADMIN_USERNAME }}"
+KC_BOOTSTRAP_ADMIN_PASSWORD="{{ KEYCLOAK_BOOTSTRAP_ADMIN_PASSWORD }}"
+
+## Permanent
+KEYCLOAK_PERMANENT_ADMIN_USERNAME="{{ KEYCLOAK_PERMANENT_ADMIN_USERNAME }}"
+KEYCLOAK_PERMANENT_ADMIN_PASSWORD="{{ KEYCLOAK_PERMANENT_ADMIN_PASSWORD }}"
 
 # Enable detailed logs
 {% if MODE_DEBUG | bool %}

roles/web-app-keycloak/users/main.yml

@@ -1,3 +0,0 @@
-users:
-  administrator:
-    username:         "administrator"

roles/web-app-keycloak/vars/main.yml

@@ -29,14 +29,22 @@ KEYCLOAK_REALM_IMPORT_FILE_SRC:     "import/realm.json.j2"
 KEYCLOAK_REALM_IMPORT_FILE_DST:     "{{ [KEYCLOAK_REALM_IMPORT_DIR_HOST,'realm.json'] | path_join }}"
 
 ## Credentials
-KEYCLOAK_ADMIN:                     "{{ applications | get_app_conf(application_id, 'users.administrator.username') }}"
-KEYCLOAK_ADMIN_PASSWORD:            "{{ applications | get_app_conf(application_id, 'credentials.administrator_password') }}"
+
+### Bootstrap
+KEYCLOAK_BOOTSTRAP_ADMIN_USERNAME:  "{{ applications | get_app_conf(application_id, 'accounts.bootstrap.username') }}"
+KEYCLOAK_BOOTSTRAP_ADMIN_PASSWORD:  "{{ applications | get_app_conf(application_id, 'credentials.administrator_password') }}"
+
+### Permanent
+KEYCLOAK_PERMANENT_ADMIN_USERNAME:  "{{ applications | get_app_conf(application_id, 'accounts.system.username') }}"
+KEYCLOAK_PERMANENT_ADMIN_PASSWORD:  "{{ applications | get_app_conf(application_id, 'credentials.administrator_password') }}"
 
 ## Docker
-KEYCLOAK_CONTAINER:                 "{{ applications | get_app_conf(application_id, 'docker.services.keycloak.name') }}"      # Name of the keycloak docker container
-KEYCLOAK_EXEC_KCADM:                "docker exec -i {{ KEYCLOAK_CONTAINER }} /opt/keycloak/bin/kcadm.sh"                      # Init script for keycloak
-KEYCLOAK_IMAGE:                     "{{ applications | get_app_conf(application_id, 'docker.services.keycloak.image') }}"     # Keycloak docker image
-KEYCLOAK_VERSION:                   "{{ applications | get_app_conf(application_id, 'docker.services.keycloak.version') }}"   # Keycloak docker version
+KEYCLOAK_CONTAINER:                 "{{ applications | get_app_conf(application_id, 'docker.services.keycloak.name') }}"
+KEYCLOAK_EXEC_CONTAINER:            "docker exec -i {{ KEYCLOAK_CONTAINER }}"
+KEYCLOAK_KCADM:                     "/opt/keycloak/bin/kcadm.sh"
+KEYCLOAK_EXEC_KCADM:                "{{ KEYCLOAK_EXEC_CONTAINER }} {{ KEYCLOAK_KCADM }}"
+KEYCLOAK_IMAGE:                     "{{ applications | get_app_conf(application_id, 'docker.services.keycloak.image') }}"
+KEYCLOAK_VERSION:                   "{{ applications | get_app_conf(application_id, 'docker.services.keycloak.version') }}"
 
 ## Server
 KEYCLOAK_SERVER_HOST:               "127.0.0.1:{{ ports.localhost.http[application_id] }}"
@@ -69,11 +77,6 @@ KEYCLOAK_LDAP_USER_OBJECT_CLASSES: >
       ) | join(', ') 
   }}
 
-## API
-KEYCLOAK_MASTER_API_USER:           "{{ applications | get_app_conf(application_id, 'users.administrator') }}" # Master Administrator
-KEYCLOAK_MASTER_API_USER_NAME:      "{{ KEYCLOAK_MASTER_API_USER.username }}"                                  # Master Administrator Username
-KEYCLOAK_MASTER_API_USER_PASSWORD:  "{{ KEYCLOAK_MASTER_API_USER.password }}"                                  # Master Administrator Password
-
 # Dictionaries
 KEYCLOAK_DICTIONARY_REALM_RAW: "{{ lookup('template', 'import/realm.json.j2') }}"
 KEYCLOAK_DICTIONARY_REALM: >-

roles/web-app-mailu/tasks/01_core.yml

@@ -41,7 +41,7 @@
   meta: flush_handlers
 
 - name: "Create Mailu accounts"
-  include_tasks: 02_create-user.yml
+  include_tasks: 02_manage_user.yml
   vars:
     MAILU_DOCKER_DIR:        "{{ docker_compose.directories.instance }}"
     mailu_api_base_url:       "http://127.0.0.1:8080/api/v1"
@@ -55,7 +55,8 @@
     mailu_user_key:           "{{ item.key }}"
     mailu_user_name:          "{{ item.value.username }}"
     mailu_password:           "{{ item.value.password }}"
-    mailu_token_ip:           "{{ item.value.ip | default('') }}"
+    mailu_token_ip:           "{{ item.value.ip | default(networks.internet.ip4) }}"
+    mailu_token_name:         "{{ SOFTWARE_NAME ~ ' Token for ' ~ item.value.username }}"
   loop:                       "{{ users | dict2items }}"
   loop_control:
     loop_var: item
@@ -66,3 +67,5 @@
 
 - name: Set Mailu DNS records
   include_tasks: 05_dns-records.yml
+
+- include_tasks: utils/run_once.yml

roles/web-app-mailu/tasks/02_manage_user.yml

@@ -25,5 +25,5 @@
   no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
 
 - name: "Create Mailu API Token for {{ mailu_user_name }}"
-  include_tasks: 03_create-token.yml
-  when: "{{ 'mail-bot' in item.value.roles }}"
+  include_tasks: 03a_manage_user_token.yml
+  when: "'mail-bot' in item.value.roles"

roles/web-app-mailu/tasks/03a_manage_user_token.yml

@@ -0,0 +1,26 @@
+
+- name: "Fetch existing API tokens via curl inside admin container"
+  command: >-
+    {{ docker_compose_command_exec }} -T admin \
+      curl -s -X GET {{ mailu_api_base_url }}/token \
+        -H "Authorization: Bearer {{ MAILU_API_TOKEN }}"
+  args:
+    chdir: "{{ MAILU_DOCKER_DIR }}"
+  register: mailu_tokens_cli
+  changed_when: false
+  no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
+
+- name: "Extract existing token info for '{{ mailu_user_key }};{{ mailu_user_name }}'"
+  set_fact:
+    mailu_user_existing_token: >-
+      {{ (
+           mailu_tokens_cli.stdout
+           | default('[]')
+           | from_json
+           | selectattr('comment','equalto', mailu_token_name)
+           | list
+         ).0 | default(None) }}
+
+- name:           "Start Mailu token procedures for undefined tokens"
+  when:           users[mailu_user_key].mailu_token is not defined
+  include_tasks:  03b_create_user_token.yml

roles/web-app-mailu/tasks/03b_create_user_token.yml

@@ -1,26 +1,3 @@
-
-- name: "Fetch existing API tokens via curl inside admin container"
-  command: >-
-    {{ docker_compose_command_exec }} -T admin \
-      curl -s -X GET {{ mailu_api_base_url }}/token \
-        -H "Authorization: Bearer {{ MAILU_API_TOKEN }}"
-  args:
-    chdir: "{{ MAILU_DOCKER_DIR }}"
-  register: mailu_tokens_cli
-  changed_when: false
-  no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
-
-- name: "Extract existing token info for '{{ mailu_user_key }};{{ mailu_user_name }}'"
-  set_fact:
-    mailu_user_existing_token: >-
-      {{ (
-           mailu_tokens_cli.stdout
-           | default('[]')
-           | from_json
-           | selectattr('comment','equalto', mailu_user_key ~ " - ansible.infinito")
-           | list
-         ).0 | default(None) }}
-
 - name: "Delete existing API token for '{{ mailu_user_key }};{{ mailu_user_name }}' if local token missing but remote exists"
   command: >-
     {{ docker_compose_command_exec }} -T admin \
@@ -29,7 +6,6 @@
   args:
     chdir: "{{ MAILU_DOCKER_DIR }}"
   when:
-    - users[mailu_user_key].mailu_token is not defined
     - mailu_user_existing_token is not none
     - mailu_user_existing_token.id is defined
   register: mailu_token_delete
@@ -43,13 +19,12 @@
       -H "Authorization: Bearer {{ MAILU_API_TOKEN }}"
       -H "Content-Type: application/json"
       -d '{{ {
-            "comment": mailu_user_key ~ " - ansible.infinito",
+            "comment": mailu_token_name,
             "email": users[mailu_user_key].email,
             "ip": mailu_token_ip
           } | to_json }}'
   args:
     chdir: "{{ MAILU_DOCKER_DIR }}"
-  when: users[mailu_user_key].mailu_token is not defined
   register: mailu_token_creation
   # If curl sees 4xx/5xx it returns non-zero due to -f → fail the task.
   failed_when:
@@ -75,5 +50,16 @@
             )
           }, recursive=True)
       }}
-  when: users[mailu_user_key].mailu_token is not defined
   no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
+
+- name: "Reset MSMTP Configuration if No-Reply User Token changed"
+  when: users['no-reply'].username == mailu_user_name
+  block:
+    - name: "Set MSMTP run-once fact false"
+      set_fact:
+        run_once_sys_svc_msmtp: false
+      changed_when: false
+
+    - name: Reload MSMTP role
+      include_role:
+        name: "sys-svc-msmtp"

roles/web-app-mailu/tasks/main.yml

@@ -1,5 +1,3 @@
 ---
-- block:
 - include_tasks: 01_core.yml
-    - include_tasks: utils/run_once.yml
   when: run_once_web_app_mailu is not defined

roles/web-app-mastodon/tasks/01_wait.yml

@@ -0,0 +1,19 @@
+- name: Check health status of '{{ item }}' container
+  shell: |
+    cid=$(docker compose ps -q {{ item }})
+    docker inspect \
+      --format '{{ "{{.State.Health.Status}}" }}' \
+      $cid
+  args:
+    chdir: "{{ docker_compose.directories.instance }}"
+  register: healthcheck
+  retries: 60
+  delay: 5
+  until: healthcheck.stdout == "healthy"
+  loop:
+    - mastodon
+    - streaming
+    - sidekiq
+  loop_control:
+    label: "{{ item }}"
+  changed_when: false

roles/web-app-mastodon/tasks/02_cleanup.yml

@@ -0,0 +1,9 @@
+---
+# Cleanup routine for Mastodon
+# Removes cached remote media older than 14 days when MODE_CLEANUP is enabled.
+- name: "Cleanup Mastodon media cache older than 14 days"
+  command:
+    cmd: "docker exec -u root {{ MASTODON_CONTAINER }} bin/tootctl media remove --days=14"
+  register: mastodon_cleanup
+  changed_when: mastodon_cleanup.rc == 0
+  failed_when: mastodon_cleanup.rc != 0

roles/web-app-mastodon/tasks/03_setup.yml

@@ -1,6 +1,3 @@
 - name: "Execute migration for '{{ application_id }}'"
   command:
     cmd: "docker exec {{ MASTODON_CONTAINER }} bundle exec rails db:migrate"
-
-- name: "Include administrator routines for '{{ application_id }}'"
-  include_tasks: 02_administrator.yml

roles/web-app-mastodon/tasks/04_administrator.yml

@@ -1,26 +1,5 @@
 # Routines to create the administrator account
 # @see https://chatgpt.com/share/67b9b12c-064c-800f-9354-8e42e6459764
-
-- name: Check health status of '{{ item }}' container
-  shell: |
-    cid=$(docker compose ps -q {{ item }})
-    docker inspect \
-      --format '{{ "{{.State.Health.Status}}" }}' \
-      $cid
-  args:
-    chdir: "{{ docker_compose.directories.instance }}"
-  register: healthcheck
-  retries: 60
-  delay: 5
-  until: healthcheck.stdout == "healthy"
-  loop:
-    - mastodon
-    - streaming
-    - sidekiq
-  loop_control:
-    label: "{{ item }}"
-  changed_when: false
-
 - name: Remove line containing "- administrator" from config/settings.yml to allow creating administrator account
   command: 
     cmd:  "docker exec -u root {{ MASTODON_CONTAINER }} sed -i '/- administrator/d' config/settings.yml"

roles/web-app-mastodon/tasks/main.yml

@@ -18,5 +18,15 @@
   vars:
     docker_compose_flush_handlers: true
 
+- name: "Wait for Mastodon"
+  include_tasks: 01_wait.yml
+
+- name: "Cleanup Mastodon caches when MODE_CLEANUP is true"
+  include_tasks: 02_cleanup.yml
+  when: MODE_CLEANUP | bool
+
 - name: "start setup procedures for mastodon"
-  include_tasks: 01_setup.yml
+  include_tasks: 03_setup.yml
+
+- name: "Include administrator routines for '{{ application_id }}'"
+  include_tasks: 04_administrator.yml

roles/web-app-navigator/config/main.yml

@@ -16,7 +16,8 @@ server:
       font-src:
         - https://cdnjs.cloudflare.com
       frame-src:
-        - "{{ WEB_PROTOCOL }}://*.{{ PRIMARY_DOMAIN }}" # Makes sense that all of the website content is available in the navigator
+         # Makes sense that all of the website content is available in the navigator
+        - "{{ WEB_PROTOCOL }}://*.{{ PRIMARY_DOMAIN }}"
     flags:
       style-src:
         unsafe-inline:  true

roles/web-app-navigator/templates/docker-compose.yml.j2

@@ -13,4 +13,3 @@
 {% include 'roles/docker-container/templates/networks.yml.j2' %}
 
 {% include 'roles/docker-compose/templates/networks.yml.j2' %}
-

roles/web-app-nextcloud/config/main.yml

@@ -91,7 +91,7 @@ docker:
       mem_reservation:      "128m"
       mem_limit:            "512m"
       pids_limit:           256
-  enabled:  "{{ applications | get_app_conf('web-app-nextcloud', 'features.oidc', False) }}"   # Activate OIDC for Nextcloud
+  enabled:  "{{ applications | get_app_conf('web-app-nextcloud', 'features.oidc', False, True, True) }}"   # Activate OIDC for Nextcloud
   # floavor decides which OICD plugin should be used. 
   # Available options: oidc_login, sociallogin
   # @see https://apps.nextcloud.com/apps/oidc_login
@@ -194,7 +194,7 @@ plugins:
     enabled: false
   fileslibreofficeedit:
     # Nextcloud LibreOffice integration: allows online editing of documents with LibreOffice (https://apps.nextcloud.com/apps/fileslibreofficeedit)
-    enabled: "{{ not (applications | get_app_conf('web-app-nextcloud', 'plugins.richdocuments.enabled', False, True)) }}"
+    enabled: "{{ not (applications | get_app_conf('web-app-nextcloud', 'plugins.richdocuments.enabled', False, True, True)) }}"
   forms:
     # Nextcloud forms: facilitates creation of forms and surveys (https://apps.nextcloud.com/apps/forms)
     enabled: true
@@ -292,13 +292,13 @@ plugins:
   #  enabled: false
   twofactor_nextcloud_notification:
     # Nextcloud two-factor notification: sends notifications for two-factor authentication events (https://apps.nextcloud.com/apps/twofactor_nextcloud_notification)
-    enabled: "{{ not applications | get_app_conf('web-app-nextcloud', 'features.oidc', False, True) }}" # Deactivate 2FA if oidc is active
+    enabled: "{{ not applications | get_app_conf('web-app-nextcloud', 'features.oidc', False, True, True) }}" # Deactivate 2FA if oidc is active
   twofactor_totp:
     # Nextcloud two-factor TOTP: provides time-based one-time password authentication (https://apps.nextcloud.com/apps/twofactor_totp)
-    enabled: "{{ not applications | get_app_conf('web-app-nextcloud', 'features.oidc', False, True) }}" # Deactivate 2FA if oidc is active
+    enabled: "{{ not applications | get_app_conf('web-app-nextcloud', 'features.oidc', False, True, True) }}" # Deactivate 2FA if oidc is active
   user_ldap:
     # Nextcloud user LDAP: integrates LDAP for user management and authentication (https://apps.nextcloud.com/apps/user_ldap)
-    enabled: "{{ applications | get_app_conf('web-app-nextcloud', 'features.ldap', False, True) }}"
+    enabled: "{{ applications | get_app_conf('web-app-nextcloud', 'features.ldap', False, True, True) }}"
     user_directory:
       enabled:  true # Enables the LDAP User Directory Search
   user_oidc:

roles/web-svc-collabora/config/main.yml

@@ -20,6 +20,10 @@ docker:
       image:            collabora/code
       version:          latest
       name:             collabora
+      cpus:             2
+      mem_reservation:  1g
+      mem_limit:        2g
+      pids_limit:       2048
 features:
   logout:   false 
   desktop:  true # Just set to allow the iframe to load it

roles/web-svc-collabora/templates/docker-compose.yml.j2

@@ -4,6 +4,15 @@
     {% include 'roles/docker-container/templates/base.yml.j2' %}
     image: "{{ COLLABORA_IMAGE }}:{{ COLLABORA_VERSION }}"
     container_name: {{ COLLABORA_CONTAINER }}
+    security_opt:
+      - seccomp=unconfined
+      - apparmor=unconfined
+    cap_add:
+      - MKNOD
+      - SYS_CHROOT
+      - SETUID
+      - SETGID
+      - FOWNER
     ports:
       - "127.0.0.1:{{ ports.localhost.http[application_id] }}:{{ container_port }}"
 {% include 'roles/docker-container/templates/healthcheck/curl.yml.j2' %}

roles/web-svc-logout/config/main.yml

@@ -19,10 +19,11 @@ server:
       connect-src:
         - "{{ WEB_PROTOCOL }}://*.{{ PRIMARY_DOMAIN }}"
         - "{{ WEB_PROTOCOL }}://{{ PRIMARY_DOMAIN }}"
+        - "https://cdn.jsdelivr.net"
       script-src-elem:
-        - https://cdn.jsdelivr.net
+        - "https://cdn.jsdelivr.net"
       style-src-elem:
-        - https://cdn.jsdelivr.net
+        - "https://cdn.jsdelivr.net"
       frame-ancestors:
         - "{{ WEB_PROTOCOL }}://<< defaults_applications[web-app-keycloak].server.domains.canonical[0] >>"
 

roles/web-svc-logout/tasks/01_core.yml

@@ -21,11 +21,6 @@
 - name: "load docker, proxy for '{{ application_id }}'"
   include_role:
     name: sys-stk-full-stateless
-  vars:
-      aca_origin:       "'{{ domains | get_url('web-svc-logout', WEB_PROTOCOL) }}' always"
-      aca_credentials:  "'true' always"
-      aca_methods:      "'GET, OPTIONS' always"
-      aca_headers:      "'Accept, Authorization' always"
 
 - name: Create symbolic link from .env file to repository
   file:

roles/web-svc-logout/templates/logout-proxy.conf.j2

@@ -8,7 +8,11 @@ location = /logout {
     proxy_http_version 1.1;
 
     {# CORS headers – allow your central page to call this #}
-    {% include 'roles/sys-svc-proxy/templates/headers/access_control_allow.conf.j2' %}
+    {%- set aca_origin = "'{{ domains | get_url('web-svc-logout', WEB_PROTOCOL) }}' always" -%}
+    {%- set aca_credentials = "'true' always" -%}
+    {%- set aca_methods = "'GET, OPTIONS' always" -%}
+    {%- set aca_headers = "'Accept, Authorization' always" -%}
+    {%- include 'roles/sys-svc-proxy/templates/headers/access_control_allow.conf.j2' -%}
 
     {# Disable caching absolutely #}
     add_header Cache-Control "no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0" always;

tasks/stages/01_constructor.yml

@@ -16,6 +16,10 @@
       users: "{{ default_users | combine(users| default({}), recursive=True) }}"
     no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
 
+  - name: Merge networks definitions
+    set_fact:
+      networks: "{{ defaults_networks | combine(networks | default({}, true), recursive=True) }}"
+
   - name: Merge application definitions
     set_fact:
       applications: "{{ defaults_applications | merge_with_defaults(applications | default({}, true)) }}"
@@ -93,10 +97,6 @@
           generate_all_domains(WWW_REDIRECT_ENABLED | bool)
         }}
   
-  - name: Merge networks definitions
-    set_fact:
-      networks: "{{ defaults_networks | combine(networks | default({}, true), recursive=True) }}"
-  
   - name: Merge OIDC configuration
     set_fact:
       OIDC: "{{ defaults_oidc | combine(OIDC | default({}, true), recursive=True) }}"
@@ -128,6 +128,7 @@
     - svc-net     # 3. Load network roles
     - svc-db      # 4. Load database roles
     - svc-prx     # 5. Load proxy roles
-    - svc-ai      # 6. Load ai roles
+    - svc-ai      # 6. Load AI roles
+    - svc-bkp     # 7. Load Backup Roles
   loop_control:
     label: "{{ item }}-roles.yml"

tests/integration/test_filters_are_defined.py

@@ -28,14 +28,17 @@ BUILTIN_FILTERS: Set[str] = {
     "int", "join", "last", "length", "list", "lower", "map", "min", "max", "random",
     "reject", "rejectattr", "replace", "reverse", "round", "safe", "select",
     "selectattr", "slice", "sort", "string", "striptags", "sum", "title", "trim",
-    "truncate", "unique", "upper", "urlencode", "urlize", "wordcount", "xmlattr",
+    "truncate", "unique", "upper", "urlencode", "urlize", "wordcount", "xmlattr","contains",
 
     # Common Ansible filters (subset, extend as needed)
     "b64decode", "b64encode", "basename", "dirname", "from_json", "to_json",
     "from_yaml", "to_yaml", "combine", "difference", "intersect",
     "flatten", "zip", "regex_search", "regex_replace", "bool",
     "type_debug", "json_query", "mandatory", "hash", "checksum",
-    "lower", "upper", "capitalize", "unique", "dict2items", "items2dict", "password_hash", "path_join", "product", "quote", "split", "ternary", "to_nice_yaml", "tojson",
+    "lower", "upper", "capitalize", "unique", "dict2items", "items2dict", 
+    "password_hash", "path_join", "product", "quote", "split", "ternary", "to_nice_yaml", 
+    "tojson", "to_nice_json",
+
 
     # Date/time-ish
     "strftime",

tests/unit/module_utils/test_config_utils.py

@@ -0,0 +1,125 @@
+import os
+import shutil
+import tempfile
+import unittest
+
+from module_utils.config_utils import (
+    get_app_conf,
+    AppConfigKeyError,
+    ConfigEntryNotSetError,
+)
+
+class TestGetAppConf(unittest.TestCase):
+    def setUp(self):
+        # Isolate working directory so that schema files can be discovered
+        self._cwd = os.getcwd()
+        self.tmpdir = tempfile.mkdtemp(prefix="cfgutilstest_")
+        os.chdir(self.tmpdir)
+
+        # Minimal schema structure:
+        # roles/web-app-demo/schema/main.yml
+        os.makedirs(os.path.join("roles", "web-app-demo", "schema"), exist_ok=True)
+        with open(os.path.join("roles", "web-app-demo", "schema", "main.yml"), "w") as f:
+            f.write(
+                # Defines 'features.defined_but_unset' in schema (without a value in applications),
+                # plus 'features.oidc' and 'features.nested.list'
+                "features:\n"
+                "  oidc: {}\n"
+                "  defined_but_unset: {}\n"
+                "  nested:\n"
+                "    list:\n"
+                "      - {}\n"
+            )
+
+        # Example configuration with actual values
+        self.applications = {
+            "web-app-demo": {
+                "features": {
+                    "oidc": True,
+                    "nested": {
+                        "list": ["first", "second"]
+                    }
+                }
+            }
+        }
+
+    def tearDown(self):
+        os.chdir(self._cwd)
+        shutil.rmtree(self.tmpdir, ignore_errors=True)
+
+    # --- Tests ---
+
+    def test_missing_app_with_skip_missing_app_returns_default_true(self):
+        """If app ID is missing and skip_missing_app=True, it should return the default (True)."""
+        apps = {"some-other-app": {}}
+        val = get_app_conf(apps, "web-app-nextcloud", "features.oidc",
+                           strict=True, default=True, skip_missing_app=True)
+        self.assertTrue(val)
+
+    def test_missing_app_with_skip_missing_app_returns_default_false(self):
+        """If app ID is missing and skip_missing_app=True, it should return the default (False)."""
+        apps = {"svc-bkp-rmt-2-loc": {}}
+        val = get_app_conf(apps, "web-app-nextcloud", "features.oidc",
+                           strict=True, default=False, skip_missing_app=True)
+        self.assertFalse(val)
+
+    def test_missing_app_without_skip_missing_app_and_strict_true_raises(self):
+        """Missing app ID without skip_missing_app and strict=True should raise."""
+        apps = {}
+        with self.assertRaises(AppConfigKeyError):
+            get_app_conf(apps, "web-app-nextcloud", "features.oidc",
+                         strict=True, default=True, skip_missing_app=False)
+
+    def test_missing_app_without_skip_missing_app_and_strict_false_raises(self):
+        apps = {}
+        with self.assertRaises(AppConfigKeyError):
+            get_app_conf(apps, "web-app-nextcloud", "features.oidc",
+                        strict=False, default=True, skip_missing_app=False)
+
+    def test_existing_app_returns_expected_value(self):
+        """Existing app and key should return the configured value."""
+        val = get_app_conf(self.applications, "web-app-demo", "features.oidc",
+                           strict=True, default=False, skip_missing_app=False)
+        self.assertTrue(val)
+
+    def test_nested_list_index_access(self):
+        """Accessing list indices should work correctly."""
+        val0 = get_app_conf(self.applications, "web-app-demo", "features.nested.list[0]",
+                            strict=True, default=None, skip_missing_app=False)
+        val1 = get_app_conf(self.applications, "web-app-demo", "features.nested.list[1]",
+                            strict=True, default=None, skip_missing_app=False)
+        self.assertEqual(val0, "first")
+        self.assertEqual(val1, "second")
+
+    def test_schema_defined_but_unset_raises_in_strict_mode(self):
+        """Schema-defined but unset value should raise in strict mode."""
+        with self.assertRaises(ConfigEntryNotSetError):
+            get_app_conf(self.applications, "web-app-demo", "features.defined_but_unset",
+                         strict=True, default=False, skip_missing_app=False)
+
+    def test_schema_defined_but_unset_strict_false_returns_default(self):
+        """Schema-defined but unset value should return default when strict=False."""
+        val = get_app_conf(self.applications, "web-app-demo", "features.defined_but_unset",
+                           strict=False, default=True, skip_missing_app=False)
+        self.assertTrue(val)
+
+    def test_invalid_key_format_raises(self):
+        """Invalid key format in path should raise AppConfigKeyError."""
+        with self.assertRaises(AppConfigKeyError):
+            get_app_conf(self.applications, "web-app-demo", "features.nested.list[not-an-int]",
+                         strict=True, default=None, skip_missing_app=False)
+
+    def test_index_out_of_range_respects_strict(self):
+        """Out-of-range index should respect strict parameter."""
+        # strict=False returns default
+        val = get_app_conf(self.applications, "web-app-demo", "features.nested.list[99]",
+                           strict=False, default="fallback", skip_missing_app=False)
+        self.assertEqual(val, "fallback")
+        # strict=True raises
+        with self.assertRaises(AppConfigKeyError):
+            get_app_conf(self.applications, "web-app-demo", "features.nested.list[99]",
+                         strict=True, default=None, skip_missing_app=False)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/roles/svc-bkp-rmt-2-loc/files/test_pull_specific_host.py

@@ -0,0 +1,122 @@
+import unittest
+import sys
+import types
+from pathlib import Path
+from unittest.mock import patch, MagicMock
+import subprocess
+import time
+import os
+
+
+def load_module():
+    """
+    Dynamically load the target script:
+    roles/svc-bkp-rmt-2-loc/files/pull-specific-host.py
+    relative to this test file.
+    """
+    here = Path(__file__).resolve()
+    # tests/unit/roles/svc-bkp-rmt-2-loc/files -> up 5 levels to repo root
+    repo_root = here.parents[5]
+    target_path = repo_root / "roles" / "svc-bkp-rmt-2-loc" / "files" / "pull-specific-host.py"
+    if not target_path.exists():
+        raise FileNotFoundError(f"Cannot find script at {target_path}")
+    spec = types.ModuleType("pull_specific_host_module")
+    code = target_path.read_text(encoding="utf-8")
+    exec(compile(code, str(target_path), "exec"), spec.__dict__)
+    return spec
+
+
+class PullSpecificHostTests(unittest.TestCase):
+    def setUp(self):
+        self.mod = load_module()
+        self.hash64 = "a" * 64
+        self.host = "1.2.3.4"
+        self.remote = f"backup@{self.host}"
+        self.base = f"/Backups/{self.hash64}/"
+        self.backup_type = "backup-docker-to-local"
+        self.type_dir = f"{self.base}{self.backup_type}/"
+        self.last_local = f"{self.type_dir}20250101000000"
+        self.last_remote = f"{self.type_dir}20250202000000"
+
+    def _completed(self, stdout="", returncode=0):
+        return subprocess.CompletedProcess(args="mock", returncode=returncode, stdout=stdout, stderr="")
+
+    def _run_side_effect_success(self, command, capture_output=True, shell=True, text=True, check=False):
+        cmd = command if isinstance(command, str) else " ".join(command)
+        if cmd.startswith(f'ssh "{self.remote}" sha256sum /etc/machine-id'):
+            return self._completed(stdout=f"{self.hash64}  /etc/machine-id\n")
+        if cmd.startswith(f'ssh "{self.remote}" "find {self.base} -maxdepth 1 -type d -execdir basename {{}} ;"'):
+            return self._completed(stdout=f"{self.hash64}\n{self.backup_type}\n")
+        if cmd.startswith(f"ls -d {self.type_dir}* | tail -1"):
+            return self._completed(stdout=self.last_local)
+        if cmd.startswith(f'ssh "{self.remote}" "ls -d {self.type_dir}*'):
+            return self._completed(stdout=f"{self.last_remote}\n")
+        return self._completed(stdout="")
+
+    def _run_side_effect_find_fail(self, command, capture_output=True, shell=True, text=True, check=False):
+        cmd = command if isinstance(command, str) else " ".join(command)
+        if cmd.startswith(f'ssh "backup@{self.host}" "find {self.base} -maxdepth 1 -type d -execdir basename {{}} ;"'):
+            raise subprocess.CalledProcessError(returncode=1, cmd=cmd, output="", stderr="find: error")
+        if cmd.startswith(f'ssh "backup@{self.host}" sha256sum /etc/machine-id'):
+            return self._completed(stdout=f"{self.hash64}  /etc/machine-id\n")
+        return self._completed(stdout="")
+
+    def _run_side_effect_no_types(self, command, capture_output=True, shell=True, text=True, check=False):
+        cmd = command if isinstance(command, str) else " ".join(command)
+        if cmd.startswith(f'ssh "{self.remote}" sha256sum /etc/machine-id'):
+            return self._completed(stdout=f"{self.hash64}  /etc/machine-id\n")
+        if cmd.startswith(f'ssh "{self.remote}" "find {self.base} -maxdepth 1 -type d -execdir basename {{}} ;"'):
+            return self._completed(stdout="")
+        return self._completed(stdout="")
+
+    @patch("time.sleep", new=lambda *a, **k: None)
+    @patch.object(os, "makedirs")
+    @patch.object(os, "system")
+    @patch.object(subprocess, "run")
+    def test_success_rsync_zero_exit(self, mock_run, mock_system, _mkd):
+        mock_run.side_effect = self._run_side_effect_success
+        mock_system.return_value = 0
+        with self.assertRaises(SystemExit) as cm:
+            self.mod.pull_backups(self.host)
+        self.assertEqual(cm.exception.code, 0)
+        self.assertTrue(mock_system.called, "rsync (os.system) should be called")
+
+    @patch("time.sleep", new=lambda *a, **k: None)
+    @patch.object(os, "makedirs")
+    @patch.object(os, "system")
+    @patch.object(subprocess, "run")
+    def test_no_backup_types_exit_zero(self, mock_run, mock_system, _mkd):
+        mock_run.side_effect = self._run_side_effect_no_types
+        mock_system.return_value = 0
+        with self.assertRaises(SystemExit) as cm:
+            self.mod.pull_backups(self.host)
+        self.assertEqual(cm.exception.code, 0)
+        self.assertFalse(mock_system.called, "rsync should not be called when no types found")
+
+    @patch("time.sleep", new=lambda *a, **k: None)
+    @patch.object(os, "makedirs")
+    @patch.object(os, "system")
+    @patch.object(subprocess, "run")
+    def test_find_failure_exits_one(self, mock_run, mock_system, _mkd):
+        mock_run.side_effect = self._run_side_effect_find_fail
+        mock_system.return_value = 0
+        with self.assertRaises(SystemExit) as cm:
+            self.mod.pull_backups(self.host)
+        self.assertEqual(cm.exception.code, 1)
+        self.assertFalse(mock_system.called, "rsync should not be called when find fails")
+
+    @patch("time.sleep", new=lambda *a, **k: None)
+    @patch.object(os, "makedirs")
+    @patch.object(os, "system")
+    @patch.object(subprocess, "run")
+    def test_rsync_fails_after_retries_exit_nonzero(self, mock_run, mock_system, _mkd):
+        mock_run.side_effect = self._run_side_effect_success
+        mock_system.side_effect = [1] * 12  # 12 retries in the script
+        with self.assertRaises(SystemExit) as cm:
+            self.mod.pull_backups(self.host)
+        self.assertEqual(cm.exception.code, 1)
+        self.assertEqual(mock_system.call_count, 12, "rsync should have retried 12 times")
+
+
+if __name__ == "__main__":
+    unittest.main()