Refactor run-once orchestration and bootstrap Mailu/Mastodon in a single deploy

- Replace legacy utils/run_once.yml with the new helpers utils/once_flag.yml and utils/once_finalize.yml
- Introduce utils/compose_up.yml to ensure docker-compose stacks are up and to flush handlers safely without coupling to run-once flags
- Migrate all affected roles (desk-*, dev-*, sys-ctl-*, sys-svc-*, web-app-*, web-svc-*, util-*) to the new run-once helpers
- Rework sys-svc-msmtp to auto-load Mailu once per deploy, check reachability, and reuse the running stack instead of requiring multiple playbook passes
- Adjust web-app-mailu to integrate cert deployment, handler flushing, and run-once handling so Mailu is fully initialized in a single deploy
- Improve Matomo, CDN, logout and CSP/health-check related roles to cooperate with the new compose_up / once_* pattern
- Simplify alarm/backup/timer/service orchestration (sys-ctl-alm-*, sys-bkp-provider, sys-timer-cln-bkps, etc.) by moving run-once logic into dedicated 01_core.yml files
- Update integration tests so utils/once_flag.yml and utils/once_finalize.yml are recognised as valid run-once providers, keeping the global run_once_* guarantees consistent
- Align frontend injection and service dependencies so Mastodon- and Mailu-related services can be brought up coherently within a single deployment cycle rather than several iterations

roles/desk-git/tasks/main.yml

@@ -16,6 +16,6 @@
       command: gitconfig --merge-option rebase --name "{{users.client.full_name}}" --email "{{users.client.email}}" --website "{{users.client.website}}" --signing gpg --gpg-key "{{users.client.gpg}}"
       become: false
   
-    - include_tasks: utils/run_once.yml
+    - include_tasks: utils/once_finalize.yml
       vars:
         flush_handlers: false

roles/desk-gnome-caffeine/tasks/01_core.yml

@@ -20,4 +20,4 @@
     src: caffeine.desktop.j2
     dest: "{{auto_start_directory}}caffeine.desktop"
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/desk-qbittorrent/tasks/main.yml

@@ -10,5 +10,5 @@
       use: yay
       name:
       - qbittorrent
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_desk_qbittorrent is not defined

roles/desk-spotify/tasks/main.yml

@@ -9,5 +9,5 @@
       use: yay
       name:
       - spotify
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_desk_spotify is not defined

roles/desk-ssh/tasks/01_core.yml

@@ -50,4 +50,4 @@
     mode: "0644"
   become: false
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/desk-zoom/tasks/main.yml

@@ -9,5 +9,5 @@
       name:
       - zoom
     become: false
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_desk_zoom is not defined

roles/dev-fakeroot/tasks/main.yml

@@ -6,6 +6,6 @@
         name: fakeroot
         state: present
 
-    - include_tasks: utils/run_once.yml
+    - include_tasks: utils/once_finalize.yml
       vars:
         flush_handlers: false

roles/dev-git/tasks/main.yml

@@ -4,7 +4,7 @@
         name: git
         state:  present
 
-    - include_tasks: utils/run_once.yml
+    - include_tasks: utils/once_finalize.yml
       vars:
         flush_handlers: false
   when: run_once_dev_git is not defined

roles/dev-python-pip/tasks/main.yml

@@ -9,7 +9,7 @@
       name: python-pip
       state: present
 
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
     vars:
       flush_handlers: false
   when: run_once_dev_python_pip is not defined

roles/dev-python-yaml/tasks/main.yml

@@ -6,6 +6,6 @@
         name: python-yaml
         state: present
       
-    - include_tasks: utils/run_once.yml
+    - include_tasks: utils/once_finalize.yml
       vars:
         flush_handlers: false

roles/dev-shell/tasks/01_core.yml

@@ -0,0 +1,21 @@
+- name: Ensure ~/.bash_profile sources ~/.profile
+  lineinfile:
+    path: "$HOME/.bash_profile"
+    line: '[ -f ~/.profile ] && . ~/.profile'
+    insertafter: EOF
+    state: present
+    create: yes
+    mode: "0644"
+  become: false
+
+- name: Ensure ~/.zprofile sources ~/.profile
+  lineinfile:
+    path: "$HOME/.zprofile"
+    line: '[ -f ~/.profile ] && . ~/.profile'
+    insertafter: EOF
+    state: present
+    create: yes
+    mode: "0644"
+  become: false
+
+- include_tasks: utils/once_flag.yml

roles/dev-shell/tasks/main.yml

@@ -1,25 +1,2 @@
----
-- block:
-  - name: Ensure ~/.bash_profile sources ~/.profile
-    lineinfile:
-      path: "$HOME/.bash_profile"
-      line: '[ -f ~/.profile ] && . ~/.profile'
-      insertafter: EOF
-      state: present
-      create: yes
-      mode: "0644"
-    become: false
-
-  - name: Ensure ~/.zprofile sources ~/.profile
-    lineinfile:
-      path: "$HOME/.zprofile"
-      line: '[ -f ~/.profile ] && . ~/.profile'
-      insertafter: EOF
-      state: present
-      create: yes
-      mode: "0644"
-    become: false
-
-  - set_fact:
-      run_once_dev_shell: true
+- include_tasks: 01_core.yml
   when: run_once_dev_shell is not defined

roles/dev-yay/tasks/01_core.yml

@@ -55,4 +55,4 @@
     aur_only: yes
   when: MODE_UPDATE | bool
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/docker-compose/tasks/01_core.yml

@@ -1,3 +1,5 @@
+- include_tasks: utils/once_flag.yml
+
 - name: Remove all docker compose pull locks
   file:
     path: "{{ PATH_DOCKER_COMPOSE_PULL_LOCK_DIR }}"
@@ -19,5 +21,3 @@
     mode:   0700
     owner:  root
     group:  root
-
-- include_tasks: utils/run_once.yml

roles/docker-container/tasks/01_core.yml

@@ -2,4 +2,4 @@
     name: sys-svc-docker
   when: run_once_sys_svc_docker is not defined
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/drv-epson-multiprinter/tasks/01_core.yml

@@ -17,3 +17,5 @@
   community.general.pacman:
     name: imagescan
     state: present
+
+- include_tasks: utils/once_flag.yml

roles/drv-epson-multiprinter/tasks/main.yml

@@ -1,5 +1,2 @@
-- block:
-  - include_tasks: 01_core.yml
-  - set_fact:
-      run_once_drv_epson_multiprinter: true
+- include_tasks: 01_core.yml
   when: run_once_drv_epson_multiprinter is not defined

roles/pkgmgr-install/tasks/01_core.yml

@@ -9,3 +9,5 @@
     pkgmgr update pkgmgr
   register: pkgmgr_update
   changed_when: "'already up to date' not in (pkgmgr_update.stdout | lower)"
+
+- include_tasks: utils/once_flag.yml

roles/pkgmgr-install/tasks/main.yml

@@ -1,7 +1,4 @@
-- block:
-  - include_tasks: 01_core.yml
-  - set_fact:
-      run_once_pkgmgr_install: true
+- include_tasks: 01_core.yml
   when: run_once_pkgmgr_install is not defined
 
 - name: "update {{ package_name }}"

roles/pkgmgr/tasks/01_core.yml

@@ -48,4 +48,4 @@
   command: "pkgmgr pull --all"
   when: MODE_UPDATE | bool
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/svc-ai-ollama/tasks/01_core.yml

@@ -35,4 +35,4 @@
     (pull_result.rc | default(0)) != 0 and
     ('up to date' not in (pull_result.stdout | default('')))
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/svc-ai-ollama/tasks/main.yml

@@ -1,5 +1,4 @@
-- block:
-  - include_tasks: 01_core.yml
-    vars:
-      flush_handlers: true
+- include_tasks: 01_core.yml
+  vars:
+    flush_handlers: true
   when: run_once_svc_ai_ollama is not defined

roles/svc-bkp-loc-2-usb/tasks/main.yml

@@ -5,7 +5,7 @@
     loop:
     - sys-ctl-cln-bkps
     - sys-lock
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_svc_bkp_loc_2_usb is not defined
 
 - name: Fail if any backup_to_usb variable is empty

roles/svc-bkp-rmt-2-loc/tasks/main.yml

@@ -7,7 +7,7 @@
     - sys-ctl-alm-compose
     - sys-lock
     - sys-timer-cln-bkps
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_svc_bkp_rmt_2_loc is not defined
 
 - name: "Create Directory '{{ DOCKER_BACKUP_REMOTE_2_LOCAL_DIR }}'"

roles/svc-db-mariadb/tasks/01_core.yml

@@ -47,4 +47,4 @@
     - setup_mariadb_container_result is defined
     - setup_mariadb_container_result.changed
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/svc-db-postgres/tasks/01_core.yml

@@ -26,4 +26,4 @@
     name: python-psycopg2
     state: present
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/svc-opt-keyboard-color/tasks/01_core.yml

@@ -16,3 +16,5 @@
     system_service_on_calendar:   "{{ SYS_SCHEDULE_ANIMATION_KEYBOARD_COLOR }}"
     system_service_timer_enabled: true
     persistent:                   true
+
+- include_tasks: utils/once_flag.yml

roles/svc-opt-keyboard-color/tasks/main.yml

@@ -1,5 +1,2 @@
-- block:
-  - include_tasks: 01_core.yml
-  - set_fact:
-      run_once_svc_opt_keyboard_color: true
+- include_tasks: 01_core.yml
   when: run_once_svc_opt_keyboard_color is not defined

roles/svc-opt-swapfile/tasks/01_core.yml

@@ -11,4 +11,4 @@
   async: "{{ ASYNC_TIME if ASYNC_ENABLED | bool else omit }}"
   poll:  "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}"
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/svc-prx-openresty/tasks/main.yml

@@ -5,5 +5,5 @@
     vars:
       docker_compose_flush_handlers:  true
       docker_pull_git_repository:     false # Deactivated here to deactivate inhirement
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_svc_prx_openresty is not defined

roles/sys-bkp-provider-user/tasks/01_core.yml

@@ -16,4 +16,4 @@
 
 - include_tasks: 03_permissions_folders.yml
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-bkp-provider/tasks/01_core.yml

@@ -0,0 +1,7 @@
+- name: Include dependencies
+  include_role:
+    name: '{{ item }}'
+  loop:
+  - sys-bkp-provider-user
+  - sys-timer-cln-bkps
+- include_tasks: utils/once_finalize.yml

roles/sys-bkp-provider/tasks/main.yml

@@ -1,9 +1,2 @@
-- block:
-  - name: Include dependencies
-    include_role:
-      name: '{{ item }}'
-    loop:
-    - sys-bkp-provider-user
-    - sys-timer-cln-bkps
-  - include_tasks: utils/run_once.yml
+- include_tasks: 01_core.yml
   when: run_once_sys_bkp_provider is not defined

roles/sys-cli/tasks/main.yml

@@ -3,7 +3,7 @@
     include_role:
       name: dev-yay
     when: run_once_dev_yay is not defined
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_sys_cli is not defined
 
 - name: "pkgmgr install infinito"

roles/sys-ctl-alm-compose/tasks/01_core.yml

@@ -1,3 +1,5 @@
+- include_tasks: utils/once_flag.yml
+
 - name: "Include dependent services for '{{ system_service_id  }}'"
   include_role:
     name: '{{ item }}'
@@ -37,5 +39,3 @@
       name: "{{ system_service_id | get_service_name(SOFTWARE_NAME, False) ~ escaped_name.stdout }}.service"
       state: started
   when: MODE_ASSERT | bool
-
-- include_tasks: utils/run_once.yml

roles/sys-ctl-alm-compose/vars/main.yml

@@ -1,16 +1,7 @@
 system_service_id: sys-ctl-alm-compose@
 
-SYSTEMCTL_ALARM_COMPOSER_SUBSERVICES_BASE:
+SYSTEMCTL_ALARM_COMPOSER_SUBSERVICES:
   - 'sys-ctl-alm-email'
   - 'sys-ctl-alm-telegram'
 
-SYSTEMCTL_ALARM_EMAIL_ENABLED: "{{ users['no-reply'].mailu_token | default(false) and not MODE_RESET }}"  
-
-SYSTEMCTL_ALARM_COMPOSER_SUBSERVICES: >-
-  {{
-    SYSTEMCTL_ALARM_COMPOSER_SUBSERVICES_BASE 
-    if SYSTEMCTL_ALARM_EMAIL_ENABLED else
-    SYSTEMCTL_ALARM_COMPOSER_SUBSERVICES_BASE | reject('equalto', 'sys-ctl-alm-email') | list
-  }}
-
 SYSTEMCTL_ALARM_COMPOSER_DUMMY_MESSAGE: "[Info] Dummy Message: No Failure; Ansible is initializing {{ SOFTWARE_NAME }} on {{ inventory_hostname }}."

roles/sys-ctl-alm-email/tasks/01_core.yml

@@ -1,4 +1,4 @@
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_flag.yml
 
 - name: Include dependencies
   include_role:

roles/sys-ctl-alm-telegram/tasks/01_core.yml

@@ -22,4 +22,4 @@
     name: curl
     state: present
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-bkp-docker-2-loc/tasks/01_core.yml

@@ -28,4 +28,4 @@
     system_service_tpl_on_failure:      "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
     # system_service_tpl_exec_start_post: "/usr/bin/systemctl start {{ SYS_SERVICE_CLEANUP_BACKUPS }}" # Not possible to use it because it's a deathlock. Keep this line for documentation purposes
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-cln-anon-volumes/tasks/main.yml

@@ -14,6 +14,6 @@
       system_service_copy_files:          false
       system_service_force_linear_sync:   false
 
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when:
     - run_once_sys_ctl_cln_anon_volumes is not defined

roles/sys-ctl-cln-bkps/tasks/01_core.yml

@@ -22,6 +22,6 @@
     system_service_copy_files:          true
     system_service_force_linear_sync:   false
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml
   vars:
     flush_handlers: true

roles/sys-ctl-cln-certs/tasks/01_core.yml

@@ -19,4 +19,4 @@
     system_service_copy_files:        false
     system_service_force_linear_sync: false
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-cln-disc-space/tasks/01_core.yml

@@ -16,4 +16,4 @@
     system_service_tpl_exec_start_pre:  '/usr/bin/python {{ PATH_SYSTEM_LOCK_SCRIPT }} {{ SYS_SERVICE_GROUP_MANIPULATION | join(" ")  }} --ignore {{ SYS_SERVICE_GROUP_CLEANUP | join(" ") }} --timeout "{{ SYS_TIMEOUT_BACKUP_SERVICES }}"'
     system_service_force_linear_sync:   false
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-cln-docker/tasks/main.yml

@@ -19,5 +19,5 @@
         system_service_force_linear_sync:   false
         system_service_force_flush:         "{{ MODE_CLEANUP }}"
 
-    - include_tasks: utils/run_once.yml
+    - include_tasks: utils/once_finalize.yml
   when: run_once_sys_ctl_cln_docker is not defined

roles/sys-ctl-cln-faild-bkps/tasks/01_core.yml

@@ -22,4 +22,4 @@
     system_service_tpl_exec_start_pre:  '/usr/bin/python {{ PATH_SYSTEM_LOCK_SCRIPT }} {{ SYS_SERVICE_GROUP_MANIPULATION | join(" ")  }} --ignore {{ SYS_SERVICE_GROUP_CLEANUP| join(" ") }} --timeout "{{ SYS_TIMEOUT_CLEANUP_SERVICES }}"'
     system_service_tpl_exec_start:      '/bin/sh -c "{{ CLEANUP_FAILED_BACKUPS_PKG }} --all --workers {{ CLEANUP_FAILED_BACKUPS_WORKERS }} --yes"'
     system_service_force_linear_sync:   false
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-hlth-btrfs/tasks/01_core.yml

@@ -10,4 +10,4 @@
     system_service_timer_enabled:  true
     system_service_tpl_on_failure: "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-hlth-csp/tasks/01_core.yml

@@ -23,4 +23,4 @@
       --nginx-config-dir={{ NGINX.DIRECTORIES.HTTP.SERVERS }}
       --ignore-network-blocks-from {{ HEALTH_CSP_IGNORE_NETWORK_BLOCKS_FROM | join(' ') }}
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-hlth-disc-space/tasks/01_core.yml

@@ -11,4 +11,4 @@
     system_service_tpl_exec_start: "{{ system_service_script_exec }} {{ SIZE_PERCENT_CLEANUP_DISC_SPACE }}"
     system_service_tpl_on_failure: "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }} {{ SYS_SERVICE_CLEANUP_DISC_SPACE }}"
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-hlth-docker-container/tasks/01_core.yml

@@ -15,4 +15,4 @@
     system_service_on_calendar:     "{{ SYS_SCHEDULE_HEALTH_DOCKER_CONTAINER }}"
     system_service_tpl_on_failure:  "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }} {{ SYS_SERVICE_REPAIR_DOCKER_SOFT }}"
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-hlth-docker-volumes/tasks/01_core.yml

@@ -11,4 +11,4 @@
     system_service_tpl_on_failure:      "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }} {{ SYS_SERVICE_CLEANUP_ANONYMOUS_VOLUMES }}"
     system_service_tpl_exec_start:      '{{ system_service_script_exec }} "{{ DOCKER_WHITELISTET_ANON_VOLUMES | join(" ") }}"'
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-hlth-journalctl/tasks/01_core.yml

@@ -11,4 +11,4 @@
     system_service_tpl_on_failure:  "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
     system_service_suppress_flush:  true # There are almost allways errors in the journalctl logs so suppression is neccessary to let playbook run
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-hlth-msmtp/tasks/01_core.yml

@@ -13,4 +13,4 @@
     - not MODE_RESET | bool
     - users['no-reply'].mailu_token | default(false)
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-hlth-webserver/tasks/01_core.yml

@@ -26,4 +26,4 @@
       --expectations '{{ applications | web_health_expectations(www_enabled=WWW_REDIRECT_ENABLED | bool, group_names=group_names) | to_json }}'
     system_service_suppress_flush:        true # The healthcheck will just work after all routines passed
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-mtn-cert-deploy/files/script.sh

@@ -10,45 +10,71 @@ fi
 ssl_cert_folder="$1"
 docker_compose_instance_directory="$2"
 letsencrypt_live_path="$3"
-docker_compose_cert_directory="$docker_compose_instance_directory/volumes/certs"
+docker_compose_cert_directory="${docker_compose_instance_directory}volumes/certs"
 
-# Copy certificates
-cp -RvL "$letsencrypt_live_path/$ssl_cert_folder/"* "$docker_compose_cert_directory" || exit 1
+# Ensure the target cert directory exists
+if [ ! -d "$docker_compose_cert_directory" ]; then
+    echo "Creating certs directory: $docker_compose_cert_directory"
+    mkdir -p "$docker_compose_cert_directory" || exit 1
+fi
 
-# This code is optimized for mailu
-cp -v "$letsencrypt_live_path/$ssl_cert_folder/privkey.pem" "$docker_compose_cert_directory/key.pem" || exit 1
-cp -v "$letsencrypt_live_path/$ssl_cert_folder/fullchain.pem" "$docker_compose_cert_directory/cert.pem" || exit 1
+# Copy all certificates (generic)
+cp -RvL "${letsencrypt_live_path}/${ssl_cert_folder}/"* "$docker_compose_cert_directory" || exit 1
+
+# Mailu optimization: explicit key/cert mapping
+cp -v "${letsencrypt_live_path}/${ssl_cert_folder}/privkey.pem"   "${docker_compose_cert_directory}/key.pem"  || exit 1
+cp -v "${letsencrypt_live_path}/${ssl_cert_folder}/fullchain.pem" "${docker_compose_cert_directory}/cert.pem" || exit 1
 
 # Set correct reading rights
-chmod a+r -v "$docker_compose_cert_directory/"*
+chmod a+r -v "${docker_compose_cert_directory}/"* || exit 1
 
-# Flag to track if any Nginx reload was successful
+# Flags to track Nginx reload status
 nginx_reload_successful=false
+nginx_reload_failed=false
+failed_services=""
 
 # Reload Nginx in all containers within the Docker Compose setup
 cd "$docker_compose_instance_directory" || exit 1
 
+echo "Wait for 5 minutes to prevent interuption of setup procedures"
+sleep 300
+
 # Iterate over all services
 for service in $(docker compose ps --services); do
     echo "Checking service: $service"
+
     # Check if Nginx exists in the container
     if docker compose exec -T "$service" which nginx > /dev/null 2>&1; then
+        echo "Testing Nginx config for service: $service"
+        if ! docker compose exec -T "$service" nginx -t; then
+            echo "Nginx config test FAILED for service: $service" >&2
+            nginx_reload_failed=true
+            failed_services="$failed_services $service"
+            continue
+        fi
+
         echo "Reloading Nginx for service: $service"
         if docker compose exec -T "$service" nginx -s reload; then
             nginx_reload_successful=true
             echo "Successfully reloaded Nginx for service: $service"
         else
             echo "Failed to reload Nginx for service: $service" >&2
+            nginx_reload_failed=true
+            failed_services="$failed_services $service"
         fi
     else
         echo "Nginx not found in service: $service, skipping."
     fi
 done
 
-# Restart all containers if no Nginx reload was successful
-if [ "$nginx_reload_successful" = false ]; then
-    echo "No Nginx reload was successful. Restarting all Docker containers."
-    docker compose restart || exit 1
-else
+# Optional auto-healing: restart only the services whose reload failed
+if [ "$nginx_reload_failed" = true ]; then
+    echo "At least one Nginx reload failed. Affected services:${failed_services}"
+    echo "Restarting affected services to apply the new certificates..."
+    # shellcheck disable=SC2086
+    (sleep 120 && docker compose restart $failed_services) || (sleep 120 && docker compose restart) || exit 1
+elif [ "$nginx_reload_successful" = true ]; then
     echo "At least one Nginx reload was successful. No restart needed."
+else
+    echo "No Nginx instances found in any service. Nothing to reload."
 fi

roles/sys-ctl-mtn-cert-deploy/tasks/01_core.yml

@@ -5,6 +5,12 @@
     flush_handlers: true
   when: run_once_sys_ctl_alm_compose is not defined
 
+- name: "Ensure cert deploy target directory exists"
+  file:
+    path: "{{ docker_compose.directories.volumes }}certs"
+    state: directory
+    mode: "0755"
+
 - include_role:
     name: sys-service
   vars:
@@ -13,4 +19,6 @@
     persistent:                       "true"
     system_service_timer_enabled:     true
     system_service_tpl_on_failure:    "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
-    system_service_force_linear_sync: false
+    system_service_force_linear_sync: false
+
+- include_tasks: utils/once_flag.yml

roles/sys-ctl-mtn-cert-deploy/tasks/main.yml

@@ -1,7 +1,4 @@
-- block:
-  - include_tasks: 01_core.yml
-  - set_fact:
-      run_once_sys_ctl_mtn_cert_deploy: true
+- include_tasks: 01_core.yml
   when: run_once_sys_ctl_mtn_cert_deploy is not defined
 
 

roles/sys-ctl-mtn-cert-deploy/vars/main.yml

@@ -1 +1 @@
-system_service_id:                        "sys-ctl-mtn-cert-deploy"
+system_service_id: "sys-ctl-mtn-cert-deploy"

roles/sys-ctl-mtn-cert-renew/tasks/01_core.yml

@@ -22,4 +22,4 @@
     system_service_tpl_on_failure:    "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
     system_service_force_linear_sync: false
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-rpr-btrfs-balancer/tasks/01_core.yml

@@ -20,4 +20,4 @@
     system_service_tpl_exec_start:    "/bin/sh -c 'btrfs-auto-balancer 90 10'"
     system_service_force_linear_sync: true
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-rpr-docker-hard/tasks/01_core.yml

@@ -14,4 +14,4 @@
     system_service_tpl_on_failure:      "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
     system_service_force_linear_sync:   true
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-ctl-rpr-docker-soft/tasks/01_core.yml

@@ -12,4 +12,4 @@
       /bin/sh -c '{{ system_service_script_exec }} --manipulation-string "{{ SYS_SERVICE_GROUP_MANIPULATION | join(" ") }}" {{ PATH_DOCKER_COMPOSE_INSTANCES }}'
     system_service_force_linear_sync:   true
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-daemon/tasks/main.yml

@@ -4,5 +4,5 @@
     when: MODE_RESET | bool and run_once_sys_daemon is not defined
   - name: Apply systemd manager defaults
     include_tasks: 02_defaults.yml
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_sys_daemon is not defined

roles/sys-dns-wildcards/tasks/01_core.yml

@@ -9,4 +9,4 @@
     cloudflare_async_poll:    "{{ ASYNC_POLL }}"
   when: DNS_PROVIDER == 'cloudflare'
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-front-inj-all/tasks/01_dependencies.yml

@@ -14,7 +14,7 @@
     - application_id != 'web-app-matomo'
     - run_once_web_app_matomo is not defined
 
-- name: "Setup web-app-matomo because endpoint was not reachable"
+- name: "Setup web-app-matomo"
   include_role:
     name: web-app-matomo
     public: false

roles/sys-front-inj-all/tasks/main.yml

@@ -3,7 +3,7 @@
       include_role:
         name: sys-svc-webserver-core
       when: run_once_sys_svc_webserver_core is not defined
-    - include_tasks: utils/run_once.yml
+    - include_tasks: utils/once_finalize.yml
   when: run_once_sys_front_inj_all is not defined
 
 - name: Build inj_enabled for '{{ domain }}'"
@@ -13,7 +13,8 @@
 - name: "Included dependent services"
   include_tasks: 01_dependencies.yml
   vars:
-    proxy_extra_configuration: ""
+    proxy_extra_configuration:     ""
+    docker_compose_flush_handlers: true
 
 - name: Reinitialize 'inj_enabled' for '{{ domain }}', after loading the required webservices
   set_fact:

roles/sys-front-inj-css/tasks/01_core.yml

@@ -29,4 +29,4 @@
     mode:   '0644'
   loop: "{{ CSS_FILES }}"
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-front-inj-desktop/tasks/01_core.yml

@@ -6,4 +6,4 @@
     group:  "{{ NGINX.USER }}"
     mode:   '0644'
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-lock/tasks/main.yml

@@ -5,7 +5,7 @@
         src: sys-lock.py
         dest: "{{ PATH_SYSTEM_LOCK_SCRIPT }}"
 
-    - include_tasks: utils/run_once.yml
+    - include_tasks: utils/once_finalize.yml
       vars:
         flush_handlers: false
   when: run_once_sys_lock is not defined

roles/sys-postfix/tasks/main.yml

@@ -3,7 +3,7 @@
     include_role:
       name: user-administrator
     when: run_once_user_administrator is not defined
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_sys_postfix is not defined
 
 - name: install postfix

roles/sys-service/tasks/01_core.yml

@@ -7,4 +7,4 @@
   include_tasks: 02_reset.yml
   when: MODE_RESET | bool
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-stk-front-base/tasks/main.yml

@@ -3,7 +3,7 @@
     include_role:
       name: sys-svc-webserver-https
     when: run_once_sys_svc_webserver_https is not defined
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_sys_stk_front_base is not defined
 
 - include_tasks: "01_cloudflare.yml"

roles/sys-svc-cdn/tasks/main.yml

@@ -8,7 +8,7 @@
         group: "{{ NGINX.USER }}"
         mode:  "0755"
       loop: "{{ CDN_DIRS_GLOBAL }}"
-    - include_tasks: utils/run_once.yml
+    - include_tasks: utils/once_finalize.yml
   when:
     - run_once_sys_svc_cdn is not defined
 

roles/sys-svc-certbot/tasks/01_core.yml

@@ -7,4 +7,4 @@
   include_tasks: 02_no_webroot.yml
   when: CERTBOT_ACME_CHALLENGE_METHOD != 'webroot'
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-svc-certs/tasks/main.yml

@@ -3,7 +3,7 @@
     include_role:
       name: sys-svc-webserver-https
     when: run_once_sys_svc_webserver_https is not defined
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_sys_svc_certs is not defined
 
 - name: "Include flavor '{{ CERTBOT_FLAVOR }}' for '{{ domain }}'"

roles/sys-svc-cln-domains/tasks/main.yml

@@ -50,5 +50,5 @@
 #    'No certificate found with name' not in certbot_delete_result.stderr
 #  changed_when: >
 #    certbot_delete_result.rc == 0
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_sys_svc_cln_domains is not defined

roles/sys-svc-dns/tasks/01_core.yml

@@ -36,4 +36,4 @@
     parent_dns_proxied: false
   when: run_once_sys_dns_wildcards is not defined
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-svc-docker/tasks/01_core.yml

@@ -1,3 +1,5 @@
+- include_tasks: utils/once_flag.yml
+
 - name: docker & docker compose install
   community.general.pacman:
     name: 
@@ -26,5 +28,3 @@
     - sys-ctl-hlth-docker-volumes
     - sys-ctl-rpr-docker-hard
   when: SYS_SVC_DOCKER_LOAD_SERVICES | bool
-
-- include_tasks: utils/run_once.yml

roles/sys-svc-journalctl/tasks/main.yml

@@ -3,7 +3,7 @@
     include_role:
       name: sys-ctl-hlth-journalctl
     when: run_once_sys_ctl_hlth_journalctl is not defined
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_sys_svc_journalctl is not defined
 
 - name: copy journald.conf

roles/sys-svc-letsencrypt/tasks/01_core.yml

@@ -9,4 +9,4 @@
     dest: "{{ [ NGINX.DIRECTORIES.HTTP.GLOBAL, 'letsencrypt.conf' ] | path_join }}"
   notify: restart openresty
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-svc-msmtp/tasks/01_core.yml

@@ -1,3 +1,25 @@
+- include_tasks: utils/once_flag.yml
+
+- name: "Check if Mail Host is reachable"
+  uri:
+    url: "{{ WEB_PROTOCOL ~ '://' ~ SYSTEM_EMAIL.HOST }}"
+    method: HEAD
+    validate_certs: yes
+    status_code: [200, 301, 302]
+  register: mail_host_reachability
+  failed_when: false
+  changed_when: false
+  no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
+  when: 
+    - run_once_web_app_mailu is not defined
+    - "{{ 'web-app-mailu' in group_names }}"
+    - SYSTEM_EMAIL.HOST == domains | get_domain('web-app-mailu')
+
+- name: "Load Mailu Routines for '{{ role_name }}'"
+  include_tasks: 02_mailu.yml
+  when: 
+    - mail_host_reachability is defined
+    - mail_host_reachability.status | default(0) not in [200, 301, 302]
 
 - name: install msmtp msmtp-mta
   community.general.pacman:
@@ -16,5 +38,4 @@
     name: sys-ctl-hlth-msmtp
   when: run_once_sys_ctl_hlth_msmtp is not defined
 
-- set_fact:
-    run_once_sys_svc_msmtp: true
+- include_tasks: utils/compose_up.yml

roles/sys-svc-msmtp/tasks/02_mailu.yml

@@ -0,0 +1,11 @@
+- name: "Load Mailu before MSMTP config, to guaranty that server is up"
+  include_role:
+    name: web-app-mailu
+    public: false
+  vars:
+    flush_handlers: true
+
+- name: "Reset compose handlers after Mailu include for MSMTP"
+  include_tasks: "{{ [ playbook_dir, 'tasks/utils/load_handlers.yml' ] | path_join }}"
+  vars:
+    handler_role_name: "docker-compose"

roles/sys-svc-msmtp/tasks/main.yml

@@ -1,6 +1,3 @@
 - name: "Load MSMTP Core Once"
   include_tasks: 01_core.yml
-  when:
-  - run_once_sys_svc_msmtp is not defined or run_once_sys_svc_msmtp is false
-  - users['no-reply'].mailu_token is defined
-  - not MODE_RESET | bool
+  when: not run_once_sys_svc_msmtp | default(false)

roles/sys-svc-sshd/tasks/main.yml

@@ -11,5 +11,5 @@
       group: root
       mode: '0644'
     notify: sshd restart
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_sys_svc_sshd is not defined

roles/sys-svc-webserver-core/tasks/01_core.yml

@@ -53,4 +53,4 @@
   vars:
     flush_handlers: false
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/sys-svc-webserver-https/tasks/main.yml

@@ -7,5 +7,5 @@
     - sys-svc-cln-domains
     - sys-svc-letsencrypt
     - sys-svc-dns
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_sys_svc_webserver_https is not defined

roles/sys-timer-cln-bkps/tasks/01_core.yml

@@ -1,3 +1,5 @@
+- include_tasks: utils/once_flag.yml
+
 - name: Include dependencies
   include_role:
     name: '{{ item }}'

roles/sys-timer-cln-bkps/tasks/main.yml

@@ -1,6 +1,2 @@
-- block:
-  - include_tasks: 01_core.yml
-  - name: run the cleanup_backups_timer tasks once
-    set_fact:
-      run_once_sys_timer_cln_bkps: true
+- include_tasks: 01_core.yml
   when: run_once_sys_timer_cln_bkps is not defined

roles/update-apt/tasks/main.yml

@@ -5,7 +5,7 @@
         upgrade: dist
         force_apt_get: yes
 
-    - include_tasks: utils/run_once.yml
+    - include_tasks: utils/once_finalize.yml
       vars:
         flush_handlers: false
   when: run_once_update_apt is not defined

roles/update-compose/tasks/01_core.yml

@@ -18,4 +18,4 @@
     - ansible_distribution == "Debian"
     - run_once_update_apt is not defined
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/update-pacman/tasks/main.yml

@@ -4,7 +4,7 @@
         update_cache: yes
         upgrade: yes
 
-    - include_tasks: utils/run_once.yml
+    - include_tasks: utils/once_finalize.yml
       vars:
         flush_handlers: false
   when: run_once_update_pacman is not defined

roles/user-administrator/tasks/01_core.yml

@@ -53,4 +53,4 @@
   vars:
     user_name: "administrator"
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_flag.yml

roles/user-root/tasks/01_core.yml

@@ -25,6 +25,4 @@
   vars:
     user_name:  "root"
 
-- include_tasks: utils/run_once.yml
-  vars:
-    flush_handlers: false
+- include_tasks: utils/once_flag.yml

roles/util-desk-design/tasks/01_core.yml

@@ -15,4 +15,6 @@
     use: yay
     name:
     - drawio-desktop
-  become: false
+  become: false
+
+- include_tasks: utils/once_flag.yml

roles/util-desk-design/tasks/main.yml

@@ -1,6 +1,3 @@
-- block:
-  - include_tasks: 01_core.yml
-  - set_fact:
-      run_once_util_desk_design: true
+- include_tasks: 01_core.yml
   when: run_once_util_desk_design is not defined
 

roles/util-desk-dev-core/tasks/main.yml

@@ -9,5 +9,5 @@
       name:
       - code
       state: present
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_util_desk_dev_core is not defined

roles/util-desk-dev-python/tasks/main.yml

@@ -3,5 +3,5 @@
     include_role:
       name: dev-python-pip
     when: run_once_dev_python_pip is not defined
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_util_desk_dev_python is not defined

roles/util-dev-admin/tasks/main.yml

@@ -14,5 +14,5 @@
       - fdupes
       - p7zip
       state: present
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_util_dev_admin is not defined

roles/web-app-bookwyrm/tasks/main.yml

@@ -3,5 +3,5 @@
   - name: "load docker, db/redis and proxy for {{ application_id }}"
     include_role:
       name: sys-stk-full-stateful
-  - include_tasks: utils/run_once.yml
+  - include_tasks: utils/once_finalize.yml
   when: run_once_web_app_bookwyrm is not defined

roles/web-app-bridgy-fed/tasks/01_core.yml

@@ -9,4 +9,4 @@
     domain:     "{{ domains | get_domain(application_id) }}"
     http_port:  "{{ ports.localhost.http[application_id] }}"
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml

roles/web-app-chess/tasks/01_core.yml

@@ -9,4 +9,4 @@
   notify: 
     - docker compose build
 
-- include_tasks: utils/run_once.yml
+- include_tasks: utils/once_finalize.yml