feat(proxy): add configurable client_max_body_size for HTML and upload locations

This commit introduces a unified mechanism to configure client_max_body_size
for both HTML and upload locations in the sys-svc-proxy role. The directive
is now injected early in html.conf.j2 and moved to a dedicated block in
upload.conf.j2 to ensure consistent behavior for large file uploads such as
OpenProject attachments.

Additionally:
- Added client_max_body_size variable override from web-app-openproject (set to 30m).
- Reordered header includes to avoid duplicate injection.
- Improved comments and structure for better clarity.

Reference: https://chatgpt.com/share/691d873e-9b50-800f-ae70-baf8bf1e5454

ansible.cfg

@@ -1,5 +1,6 @@
 [defaults]
 # --- Performance & Behavior ---
+pipelining = True
 forks = 25
 strategy = linear
 gathering = smart
@@ -14,19 +15,14 @@ stdout_callback = yaml
 callbacks_enabled = profile_tasks,timer
 
 # --- Plugin paths ---
-filter_plugins = ./filter_plugins
+filter_plugins  = ./filter_plugins
 lookup_plugins  = ./lookup_plugins
 module_utils    = ./module_utils
 
 [ssh_connection]
-# Multiplexing: safer socket path in HOME instead of /tmp
-ssh_args = -o ControlMaster=auto -o ControlPersist=20s -o ControlPath=~/.ssh/ansible-%h-%p-%r \
-           -o ServerAliveInterval=15 -o ServerAliveCountMax=3 -o StrictHostKeyChecking=accept-new \
-           -o PreferredAuthentications=publickey,password,keyboard-interactive
-
-# Pipelining boosts speed; works fine if sudoers does not enforce "requiretty"
+ssh_args = -o ControlMaster=auto -o ControlPersist=20s -o ControlPath=~/.ssh/ansible-%h-%p-%r -o ServerAliveInterval=15 -o ServerAliveCountMax=3 -o StrictHostKeyChecking=accept-new -o PreferredAuthentications=publickey,password,keyboard-interactive
 pipelining = True
-scp_if_ssh = smart
+transfer_method = smart
 
 [persistent_connection]
 connect_timeout = 30

cli/build/defaults/applications.py

@@ -83,6 +83,13 @@ class DefaultsGenerator:
             print(f"Error during rendering: {e}", file=sys.stderr)
             sys.exit(1)
 
+        # Sort applications by application key for stable output
+        apps = result.get("defaults_applications", {})
+        if isinstance(apps, dict) and apps:
+            result["defaults_applications"] = {
+                k: apps[k] for k in sorted(apps.keys())
+            }
+
         # Write output
         self.output_file.parent.mkdir(parents=True, exist_ok=True)
         with self.output_file.open("w", encoding="utf-8") as f:

cli/build/defaults/users.py

@@ -220,6 +220,10 @@ def main():
         print(f"Error building user entries: {e}", file=sys.stderr)
         sys.exit(1)
 
+    # Sort users by key for deterministic output
+    if isinstance(users, dict) and users:
+        users = OrderedDict(sorted(users.items()))
+
     # Convert OrderedDict into plain dict for YAML
     default_users = {'default_users': users}
     plain_data = dictify(default_users)

filter_plugins/csp_filters.py

@@ -10,9 +10,23 @@ from module_utils.config_utils import get_app_conf
 from module_utils.get_url import get_url
 
 
+def _dedup_preserve(seq):
+    """Return a list with stable order and unique items."""
+    seen = set()
+    out = []
+    for x in seq:
+        if x not in seen:
+            seen.add(x)
+            out.append(x)
+    return out
+
+
 class FilterModule(object):
     """
-    Custom filters for Content Security Policy generation and CSP-related utilities.
+    Jinja filters for building a robust, CSP3-aware Content-Security-Policy header.
+    Safari/CSP2 compatibility is ensured by merging the -elem/-attr variants into the base
+    directives (style-src, script-src). We intentionally do NOT mirror back into -elem/-attr
+    to allow true CSP3 granularity on modern browsers.
     """
 
     def filters(self):
@@ -61,11 +75,14 @@ class FilterModule(object):
         """
         Returns CSP flag tokens (e.g., "'unsafe-eval'", "'unsafe-inline'") for a directive,
         merging sane defaults with app config.
-        Default: 'unsafe-inline' is enabled for style-src and style-src-elem.
+
+        Defaults:
+          - For styles we enable 'unsafe-inline' by default (style-src, style-src-elem, style-src-attr),
+            because many apps rely on inline styles / style attributes.
+          - For scripts we do NOT enable 'unsafe-inline' by default.
         """
-        # Defaults that apply to all apps
         default_flags = {}
-        if directive in ('style-src', 'style-src-elem'):
+        if directive in ('style-src', 'style-src-elem', 'style-src-attr'):
             default_flags = {'unsafe-inline': True}
 
         configured = get_app_conf(
@@ -76,7 +93,6 @@ class FilterModule(object):
             {}
         )
 
-        # Merge defaults with configured flags (configured overrides defaults)
         merged = {**default_flags, **configured}
 
         tokens = []
@@ -131,77 +147,148 @@ class FilterModule(object):
     ):
         """
         Builds the Content-Security-Policy header value dynamically based on application settings.
-        - Flags (e.g., 'unsafe-eval', 'unsafe-inline') are read from server.csp.flags.<directive>,
-          with sane defaults applied in get_csp_flags (always 'unsafe-inline' for style-src and style-src-elem).
-        - Inline hashes are read from server.csp.hashes.<directive>.
-        - Whitelists are read from server.csp.whitelist.<directive>.
-        - Inline hashes are added only if the final tokens do NOT include 'unsafe-inline'.
+
+        Key points:
+          - CSP3-aware: supports base/elem/attr for styles and scripts.
+          - Safari/CSP2 fallback: base directives (style-src, script-src) always include
+            the union of their -elem/-attr variants.
+          - We do NOT mirror back into -elem/-attr; finer CSP3 rules remain effective
+            on modern browsers if you choose to use them.
+          - If the app explicitly disables a token on the *base* (e.g. style-src.unsafe-inline: false),
+            that token is removed from the merged base even if present in elem/attr.
+          - Inline hashes are added ONLY if that directive does NOT include 'unsafe-inline'.
+          - Whitelists/flags/hashes read from:
+              server.csp.whitelist.<directive>
+              server.csp.flags.<directive>
+              server.csp.hashes.<directive>
+          - “Smart defaults”:
+              * internal CDN for style/script elem and connect
+              * Matomo endpoints (if feature enabled) for script-elem/connect
+              * Simpleicons (if feature enabled) for connect
+              * reCAPTCHA (if feature enabled) for script-elem/frame-src
+              * frame-ancestors extended for desktop/logout/keycloak if enabled
         """
         try:
             directives = [
-                'default-src',      # Fallback source list for content types not explicitly listed
-                'connect-src',      # Allowed URLs for XHR, WebSockets, EventSource, fetch()
-                'frame-ancestors',  # Who may embed this page
-                'frame-src',        # Sources for nested browsing contexts (e.g., <iframe>)
-                'script-src',       # Sources for script execution
-                'script-src-elem',  # Sources for <script> elements
-                'style-src',        # Sources for inline styles and <style>/<link> elements
-                'style-src-elem',   # Sources for <style> and <link rel="stylesheet">
-                'font-src',         # Sources for fonts
-                'worker-src',       # Sources for workers
-                'manifest-src',     # Sources for web app manifests
-                'media-src',        # Sources for audio and video
+                'default-src',
+                'connect-src',
+                'frame-ancestors',
+                'frame-src',
+                'script-src',
+                'script-src-elem',
+                'script-src-attr',
+                'style-src',
+                'style-src-elem',
+                'style-src-attr',
+                'font-src',
+                'worker-src',
+                'manifest-src',
+                'media-src',
             ]
 
-            parts = []
+            tokens_by_dir = {}
+            explicit_flags_by_dir = {}
 
             for directive in directives:
+                # Collect explicit flags (to later respect explicit "False" on base during merge)
+                explicit_flags = get_app_conf(
+                    applications,
+                    application_id,
+                    'server.csp.flags.' + directive,
+                    False,
+                    {}
+                )
+                explicit_flags_by_dir[directive] = explicit_flags
+
                 tokens = ["'self'"]
 
-                # 1) Load flags (includes defaults from get_csp_flags)
+                # 1) Flags (with sane defaults)
                 flags = self.get_csp_flags(applications, application_id, directive)
                 tokens += flags
 
-                # 2) Allow fetching from internal CDN by default for selected directives
-                if directive in ['script-src-elem', 'connect-src', 'style-src-elem']:
+                # 2) Internal CDN defaults for selected directives
+                if directive in ('script-src-elem', 'connect-src', 'style-src-elem', 'style-src'):
                     tokens.append(get_url(domains, 'web-svc-cdn', web_protocol))
 
-                # 3) Matomo integration if feature is enabled
-                if directive in ['script-src-elem', 'connect-src']:
+                # 3) Matomo (if enabled)
+                if directive in ('script-src-elem', 'connect-src'):
                     if self.is_feature_enabled(applications, matomo_feature_name, application_id):
                         tokens.append(get_url(domains, 'web-app-matomo', web_protocol))
 
-                # 4) ReCaptcha integration (scripts + frames) if feature is enabled
+                # 4) Simpleicons (if enabled) – typically used via connect-src (fetch)
+                if directive == 'connect-src':
+                    if self.is_feature_enabled(applications, 'simpleicons', application_id):
+                        tokens.append(get_url(domains, 'web-svc-simpleicons', web_protocol))
+
+                # 5) reCAPTCHA (if enabled) – scripts + frames
                 if self.is_feature_enabled(applications, 'recaptcha', application_id):
-                    if directive in ['script-src-elem', 'frame-src']:
+                    if directive in ('script-src-elem', 'frame-src'):
                         tokens.append('https://www.gstatic.com')
                         tokens.append('https://www.google.com')
 
-                # 5) Frame ancestors handling (desktop + logout support)
+                # 6) Frame ancestors (desktop + logout)
                 if directive == 'frame-ancestors':
                     if self.is_feature_enabled(applications, 'desktop', application_id):
-                        # Allow being embedded by the desktop app domain (and potentially its parent)
+                        # Allow being embedded by the desktop app domain's site
                         domain = domains.get('web-app-desktop')[0]
                         sld_tld = ".".join(domain.split(".")[-2:])  # e.g., example.com
                         tokens.append(f"{sld_tld}")
                     if self.is_feature_enabled(applications, 'logout', application_id):
-                        # Allow embedding via logout proxy and Keycloak app
                         tokens.append(get_url(domains, 'web-svc-logout', web_protocol))
                         tokens.append(get_url(domains, 'web-app-keycloak', web_protocol))
 
-                # 6) Custom whitelist entries
+                # 7) Custom whitelist
                 tokens += self.get_csp_whitelist(applications, application_id, directive)
 
-                # 7) Add inline content hashes ONLY if final tokens do NOT include 'unsafe-inline'
-                #    (Check tokens, not flags, to include defaults and later modifications.)
+                # 8) Inline hashes (only if this directive does NOT include 'unsafe-inline')
                 if "'unsafe-inline'" not in tokens:
                     for snippet in self.get_csp_inline_content(applications, application_id, directive):
                         tokens.append(self.get_csp_hash(snippet))
 
-                # Append directive
-                parts.append(f"{directive} {' '.join(tokens)};")
+                tokens_by_dir[directive] = _dedup_preserve(tokens)
 
-            # 8) Static img-src directive (kept permissive for data/blob and any host)
+            # ----------------------------------------------------------
+            # CSP3 families → ensure CSP2 fallback (Safari-safe)
+            # Merge style/script families so base contains union of elem/attr.
+            # Respect explicit disables on the base (e.g. unsafe-inline=False).
+            # Do NOT mirror back into elem/attr (keep granularity).
+            # ----------------------------------------------------------
+            def _strip_if_disabled(unioned_tokens, explicit_flags, name):
+                """
+                Remove a token (e.g. 'unsafe-inline') from the unioned token list
+                if it is explicitly disabled in the base directive flags.
+                """
+                if isinstance(explicit_flags, dict) and explicit_flags.get(name) is False:
+                    tok = f"'{name}'"
+                    return [t for t in unioned_tokens if t != tok]
+                return unioned_tokens
+
+            def merge_family(base_key, elem_key, attr_key):
+                base = tokens_by_dir.get(base_key, [])
+                elem = tokens_by_dir.get(elem_key, [])
+                attr = tokens_by_dir.get(attr_key, [])
+                union = _dedup_preserve(base + elem + attr)
+
+                # Respect explicit disables on the base
+                explicit_base = explicit_flags_by_dir.get(base_key, {})
+                # The most relevant flags for script/style:
+                for flag_name in ('unsafe-inline', 'unsafe-eval'):
+                    union = _strip_if_disabled(union, explicit_base, flag_name)
+
+                tokens_by_dir[base_key] = union  # write back only to base
+
+            merge_family('style-src',  'style-src-elem',  'style-src-attr')
+            merge_family('script-src', 'script-src-elem', 'script-src-attr')
+
+            # ----------------------------------------------------------
+            # Assemble header
+            # ----------------------------------------------------------
+            parts = []
+            for directive in directives:
+                if directive in tokens_by_dir:
+                    parts.append(f"{directive} {' '.join(tokens_by_dir[directive])};")
+
+            # Keep permissive img-src for data/blob + any host (as before)
             parts.append("img-src * data: blob:;")
 
             return ' '.join(parts)

filter_plugins/get_docker_paths.py

@@ -20,9 +20,10 @@ def get_docker_paths(application_id: str, path_docker_compose_instances: str) ->
             'config':   f"{base}config/",
         },
         'files': {
-            'env':            f"{base}.env/env",
-            'docker_compose': f"{base}docker-compose.yml",
-            'dockerfile':     f"{base}Dockerfile",
+            'env':                      f"{base}.env/env",
+            'docker_compose':           f"{base}docker-compose.yml",
+            'docker_compose_override':  f"{base}docker-compose.override.yml",
+            'dockerfile':               f"{base}Dockerfile",
         }
     }
 

filter_plugins/jvm_filters.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import sys, os, re
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from ansible.errors import AnsibleFilterError
+from module_utils.config_utils import get_app_conf
+from module_utils.entity_name_utils import get_entity_name
+
+_UNIT_RE = re.compile(r'^\s*(\d+(?:\.\d+)?)\s*([kKmMgGtT]?[bB]?)?\s*$')
+_FACTORS = {
+    '': 1, 'b': 1,
+    'k': 1024, 'kb': 1024,
+    'm': 1024**2, 'mb': 1024**2,
+    'g': 1024**3, 'gb': 1024**3,
+    't': 1024**4, 'tb': 1024**4,
+}
+
+def _to_bytes(v: str) -> int:
+    if v is None:
+        raise AnsibleFilterError("jvm_filters: size value is None")
+    s = str(v).strip()
+    m = _UNIT_RE.match(s)
+    if not m:
+        raise AnsibleFilterError(f"jvm_filters: invalid size '{v}'")
+    num, unit = m.group(1), (m.group(2) or '').lower()
+    try:
+        val = float(num)
+    except ValueError as e:
+        raise AnsibleFilterError(f"jvm_filters: invalid numeric size '{v}'") from e
+    factor = _FACTORS.get(unit)
+    if factor is None:
+        raise AnsibleFilterError(f"jvm_filters: unknown unit in '{v}'")
+    return int(val * factor)
+
+def _to_mb(v: str) -> int:
+    return max(0, _to_bytes(v) // (1024 * 1024))
+
+def _svc(app_id: str) -> str:
+    return get_entity_name(app_id)
+
+def _mem_limit_mb(apps: dict, app_id: str) -> int:
+    svc = _svc(app_id)
+    raw = get_app_conf(apps, app_id, f"docker.services.{svc}.mem_limit")
+    mb = _to_mb(raw)
+    if mb <= 0:
+        raise AnsibleFilterError(f"jvm_filters: mem_limit for '{svc}' must be > 0 MB (got '{raw}')")
+    return mb
+
+def _mem_res_mb(apps: dict, app_id: str) -> int:
+    svc = _svc(app_id)
+    raw = get_app_conf(apps, app_id, f"docker.services.{svc}.mem_reservation")
+    mb = _to_mb(raw)
+    if mb <= 0:
+        raise AnsibleFilterError(f"jvm_filters: mem_reservation for '{svc}' must be > 0 MB (got '{raw}')")
+    return mb
+
+def jvm_max_mb(apps: dict, app_id: str) -> int:
+    """Xmx = min( floor(0.7*limit), limit-1024, 12288 ) with floor at 1024 MB."""
+    limit_mb = _mem_limit_mb(apps, app_id)
+    c1 = (limit_mb * 7) // 10
+    c2 = max(0, limit_mb - 1024)
+    c3 = 12288
+    return max(1024, min(c1, c2, c3))
+
+def jvm_min_mb(apps: dict, app_id: str) -> int:
+    """Xms = min( floor(Xmx/2), mem_reservation, Xmx ) with floor at 512 MB."""
+    xmx = jvm_max_mb(apps, app_id)
+    res = _mem_res_mb(apps, app_id)
+    return max(512, min(xmx // 2, res, xmx))
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            "jvm_max_mb": jvm_max_mb,
+            "jvm_min_mb": jvm_min_mb,
+        }

filter_plugins/node_autosize.py

@@ -0,0 +1,141 @@
+# filter_plugins/node_autosize.py
+# Reuse app config to derive sensible Node.js heap sizes for containers.
+#
+# Usage example (Jinja):
+#   {{ applications | node_max_old_space_size('web-app-nextcloud', 'whiteboard') }}
+#
+# Heuristics (defaults):
+#   - candidate = 35% of mem_limit
+#   - min       = 768 MB (required minimum)
+#   - cap       = min(3072 MB, 60% of mem_limit)
+#
+# NEW: If mem_limit (container cgroup RAM) is smaller than min_mb, we raise an
+# exception — to prevent a misconfiguration where Node's heap could exceed the cgroup
+# and be OOM-killed.
+
+from __future__ import annotations
+import re
+from ansible.errors import AnsibleFilterError
+
+# Import the shared config resolver from module_utils
+try:
+    from module_utils.config_utils import get_app_conf, AppConfigKeyError
+except Exception as e:
+    raise AnsibleFilterError(
+        f"Failed to import get_app_conf from module_utils.config_utils: {e}"
+    )
+
+_SIZE_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([kmgtp]?i?b?)?\s*$", re.IGNORECASE)
+_MULT = {
+    "": 1,
+    "b": 1,
+    "k": 10**3, "kb": 10**3,
+    "m": 10**6, "mb": 10**6,
+    "g": 10**9, "gb": 10**9,
+    "t": 10**12, "tb": 10**12,
+    "p": 10**15, "pb": 10**15,
+    "kib": 1024,
+    "mib": 1024**2,
+    "gib": 1024**3,
+    "tib": 1024**4,
+    "pib": 1024**5,
+}
+
+
+def _to_bytes(val):
+    """Convert numeric or string memory limits (e.g. '512m', '2GiB') to bytes."""
+    if val is None or val == "":
+        return None
+    if isinstance(val, (int, float)):
+        return int(val)
+    if not isinstance(val, str):
+        raise AnsibleFilterError(f"Unsupported mem_limit type: {type(val).__name__}")
+    m = _SIZE_RE.match(val)
+    if not m:
+        raise AnsibleFilterError(f"Unrecognized mem_limit string: {val!r}")
+    num = float(m.group(1))
+    unit = (m.group(2) or "").lower()
+    if unit not in _MULT:
+        raise AnsibleFilterError(f"Unknown unit in mem_limit: {unit!r}")
+    return int(num * _MULT[unit])
+
+
+def _mb(bytes_val: int) -> int:
+    """Return decimal MB (10^6) as integer — Node expects MB units."""
+    return int(round(bytes_val / 10**6))
+
+
+def _compute_old_space_mb(
+    total_mb: int, pct: float, min_mb: int, hardcap_mb: int, safety_cap_pct: float
+) -> int:
+    """
+    Compute Node.js old-space heap (MB) with safe minimum and cap handling.
+
+    NOTE: The calling function ensures total_mb >= min_mb; here we only
+    apply the sizing heuristics and caps.
+    """
+    candidate = int(total_mb * float(pct))
+    safety_cap = int(total_mb * float(safety_cap_pct))
+    final_cap = min(int(hardcap_mb), safety_cap)
+
+    # Enforce minimum first; only apply cap if it's above the minimum
+    candidate = max(candidate, int(min_mb))
+    if final_cap >= int(min_mb):
+        candidate = min(candidate, final_cap)
+
+    # Never below a tiny hard floor
+    return max(candidate, 128)
+
+
+def node_max_old_space_size(
+    applications: dict,
+    application_id: str,
+    service_name: str,
+    pct: float = 0.35,
+    min_mb: int = 768,
+    hardcap_mb: int = 3072,
+    safety_cap_pct: float = 0.60,
+) -> int:
+    """
+    Derive Node.js --max-old-space-size (MB) from the service's mem_limit in app config.
+
+    Looks up: docker.services.<service_name>.mem_limit for the given application_id.
+
+    Raises:
+        AnsibleFilterError if mem_limit is missing/invalid OR if mem_limit (MB) < min_mb.
+    """
+    try:
+        mem_limit = get_app_conf(
+            applications=applications,
+            application_id=application_id,
+            config_path=f"docker.services.{service_name}.mem_limit",
+            strict=True,
+            default=None,
+        )
+    except AppConfigKeyError as e:
+        raise AnsibleFilterError(str(e))
+
+    if mem_limit in (None, False, ""):
+        raise AnsibleFilterError(
+            f"mem_limit not set for application '{application_id}', service '{service_name}'"
+        )
+
+    total_bytes = _to_bytes(mem_limit)
+    total_mb = _mb(total_bytes)
+
+    # NEW: guardrail — refuse to size a heap larger than the cgroup limit
+    if total_mb < int(min_mb):
+        raise AnsibleFilterError(
+            f"mem_limit ({total_mb} MB) is below the required minimum heap ({int(min_mb)} MB) "
+            f"for application '{application_id}', service '{service_name}'. "
+            f"Increase mem_limit or lower min_mb."
+        )
+
+    return _compute_old_space_mb(total_mb, pct, min_mb, hardcap_mb, safety_cap_pct)
+
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            "node_max_old_space_size": node_max_old_space_size,
+        }

group_vars/all/00_general.yml

@@ -76,8 +76,9 @@ _applications_nextcloud_oidc_flavor: >-
           False,
           'oidc_login'
           if applications
-            | get_app_conf('web-app-nextcloud','features.ldap',False, True)
-          else 'sociallogin'
+            | get_app_conf('web-app-nextcloud','features.ldap',False, True, True)
+          else 'sociallogin',
+          True
         )
   }}
 

group_vars/all/01_modes.yml

@@ -5,5 +5,6 @@ MODE_DUMMY:   false                       # Executes dummy/test routines instead
 MODE_UPDATE:  true                        # Executes updates
 MODE_DEBUG:   false                       # This enables debugging in ansible and in the apps, You SHOULD NOT enable this on production servers
 MODE_RESET:   false                       # Cleans up all Infinito.Nexus files. It's necessary to run to whole playbook and not particial roles when using this function.
-MODE_CLEANUP: "{{ MODE_DEBUG  | bool }}"  # Cleanup unused files and configurations
+MODE_CLEANUP: true                        # Cleanup unused files and configurations
 MODE_ASSERT:  "{{ MODE_DEBUG  | bool }}"  # Executes validation tasks during the run.
+MODE_BACKUP:  true                        # Executes the Backup before the deployment

group_vars/all/08_schedule.yml

@@ -1,4 +1,3 @@
-
 # Service Timers
 
 ## Meta
@@ -6,12 +5,12 @@ SYS_TIMER_ALL_ENABLED:                        "{{ MODE_DEBUG }}"   # Runtime Var
 
 ## Server Tact Variables 
 
-HOURS_SERVER_AWAKE:                           "0..23" # Ours in which the server is "awake" (100% working). Rest of the time is reserved for maintanance
+HOURS_SERVER_AWAKE:                           "6..23" # Ours in which the server is "awake" (100% working). Rest of the time is reserved for maintanance
 RANDOMIZED_DELAY_SEC:                         "5min"  # Random delay for systemd timers to avoid peak loads.
 
 ## Timeouts for all services
 SYS_TIMEOUT_DOCKER_RPR_HARD:                  "10min"
-SYS_TIMEOUT_DOCKER_RPR_SOFT:                 "{{ SYS_TIMEOUT_DOCKER_RPR_HARD }}"
+SYS_TIMEOUT_DOCKER_RPR_SOFT:                  "{{ SYS_TIMEOUT_DOCKER_RPR_HARD }}"
 SYS_TIMEOUT_CLEANUP_SERVICES:                 "15min"
 SYS_TIMEOUT_DOCKER_UPDATE:                    "20min"
 SYS_TIMEOUT_STORAGE_OPTIMIZER:                "{{ SYS_TIMEOUT_DOCKER_UPDATE }}"
@@ -24,29 +23,29 @@ SYS_SCHEDULE_HEALTH_BTRFS:                    "*-*-* 00:00:00"
 SYS_SCHEDULE_HEALTH_JOURNALCTL:               "*-*-* 00:00:00"                        # Check once per day the journalctl for errors
 SYS_SCHEDULE_HEALTH_DISC_SPACE:               "*-*-* 06,12,18,00:00:00"               # Check four times per day if there is sufficient disc space 
 SYS_SCHEDULE_HEALTH_DOCKER_CONTAINER:         "*-*-* {{ HOURS_SERVER_AWAKE }}:00:00"  # Check once per hour if the docker containers are healthy
-SYS_SCHEDULE_HEALTH_DOCKER_VOLUMES:           "*-*-* {{ HOURS_SERVER_AWAKE }}:15:00"  # Check once per hour if the docker volumes are healthy
-SYS_SCHEDULE_HEALTH_CSP_CRAWLER:              "*-*-* {{ HOURS_SERVER_AWAKE }}:30:00"  # Check once per hour if all CSP are fullfilled available
-SYS_SCHEDULE_HEALTH_NGINX:                    "*-*-* {{ HOURS_SERVER_AWAKE }}:45:00"  # Check once per hour if all webservices are available
+SYS_SCHEDULE_HEALTH_DOCKER_VOLUMES:           "*-*-* {{ HOURS_SERVER_AWAKE }}:00:00"  # Check once per hour if the docker volumes are healthy
+SYS_SCHEDULE_HEALTH_CSP_CRAWLER:              "*-*-* {{ HOURS_SERVER_AWAKE }}:00:00"  # Check once per hour if all CSP are fullfilled available
+SYS_SCHEDULE_HEALTH_NGINX:                    "*-*-* {{ HOURS_SERVER_AWAKE }}:00:00"  # Check once per hour if all webservices are available
 SYS_SCHEDULE_HEALTH_MSMTP:                    "*-*-* 00:00:00"                        # Check once per day SMTP Server              
 
 ### Schedule for cleanup tasks
-SYS_SCHEDULE_CLEANUP_BACKUPS:                 "*-*-* 00,06,12,18:30:00"               # Cleanup backups every 6 hours, MUST be called before disc space cleanup
-SYS_SCHEDULE_CLEANUP_DISC_SPACE:              "*-*-* 07,13,19,01:30:00"               # Cleanup disc space every 6 hours
-SYS_SCHEDULE_CLEANUP_CERTS:                   "*-*-* 12,00:45:00"                     # Deletes and revokes unused certs
-SYS_SCHEDULE_CLEANUP_FAILED_BACKUPS:          "*-*-* 12:00:00"                        # Clean up failed docker backups every noon
+SYS_SCHEDULE_CLEANUP_CERTS:                   "*-*-* 20:00"                           # Deletes and revokes unused certs once per day
+SYS_SCHEDULE_CLEANUP_FAILED_BACKUPS:          "*-*-* 21:00"                           # Clean up failed docker backups once per day
+SYS_SCHEDULE_CLEANUP_BACKUPS:                 "*-*-* 22:00"                           # Cleanup backups once per day, MUST be called before disc space cleanup
+SYS_SCHEDULE_CLEANUP_DISC_SPACE:              "*-*-* 23:00"                           # Cleanup disc space once per day
 
 ### Schedule for repair services
 SYS_SCHEDULE_REPAIR_BTRFS_AUTO_BALANCER:      "Sat *-*-01..07 00:00:00"               # Execute btrfs auto balancer every first Saturday of a month
-SYS_SCHEDULE_REPAIR_DOCKER_HARD:              "Sun *-*-* 08:00:00"                    # Restart docker instances every Sunday at 8:00 AM
+SYS_SCHEDULE_REPAIR_DOCKER_HARD:              "Sun *-*-* 00:00:00"                    # Restart docker instances every Sunday
 
 ### Schedule for backup tasks
-SYS_SCHEDULE_BACKUP_DOCKER_TO_LOCAL:          "*-*-* 03:30:00"
-SYS_SCHEDULE_BACKUP_REMOTE_TO_LOCAL:          "*-*-* 21:30:00"
+SYS_SCHEDULE_BACKUP_REMOTE_TO_LOCAL:          "*-*-* 00:30:00"                        # Pull Backup of the previous day
+SYS_SCHEDULE_BACKUP_DOCKER_TO_LOCAL:          "*-*-* 01:00:00"                        # Backup the current day
 
 ### Schedule for Maintenance Tasks
-SYS_SCHEDULE_MAINTANANCE_LETSENCRYPT_RENEW:   "*-*-* 12,00:30:00"                     # Renew Mailu certificates twice per day
-SYS_SCHEDULE_MAINTANANCE_LETSENCRYPT_DEPLOY:  "*-*-* 13,01:30:00"                     # Deploy letsencrypt certificates twice per day to docker containers
-SYS_SCHEDULE_MAINTANANCE_NEXTCLOUD:           "22"                                    # Do nextcloud maintanace between 22:00 and 02:00
+SYS_SCHEDULE_MAINTANANCE_LETSENCRYPT_RENEW:   "*-*-* 10,22:00:00"                     # Renew Mailu certificates twice per day
+SYS_SCHEDULE_MAINTANANCE_LETSENCRYPT_DEPLOY:  "*-*-* 11,23:00:00"                     # Deploy letsencrypt certificates twice per day to docker containers
+SYS_SCHEDULE_MAINTANANCE_NEXTCLOUD:           "21"                                    # Do nextcloud maintanace between 21:00 and 01:00
 
 ### Animation
 SYS_SCHEDULE_ANIMATION_KEYBOARD_COLOR:        "*-*-* *:*:00"                          # Change the keyboard color every minute

group_vars/all/09_networks.yml

@@ -110,6 +110,12 @@ defaults_networks:
       subnet: 192.168.104.16/28
     web-app-minio:
       subnet: 192.168.104.32/28
+    web-svc-coturn:
+      subnet: 192.168.104.48/28
+    web-app-mini-qr:
+      subnet: 192.168.104.64/28
+    web-app-shopware:
+      subnet: 192.168.104.80/28
 
     # /24 Networks / 254 Usable Clients
     web-app-bigbluebutton:

group_vars/all/10_ports.yml

@@ -80,20 +80,31 @@ ports:
       web-app-flowise: 8056
       web-app-minio_api: 8057
       web-app-minio_console: 8058
+      web-app-mini-qr: 8059
+      web-app-shopware: 8060
       web-app-bigbluebutton: 48087    # This port is predefined by bbb. @todo Try to change this to a 8XXX port
   public:
     # The following ports should be changed to 22 on the subdomain via stream mapping
     ssh:
-      web-app-gitea: 2201
-      web-app-gitlab: 2202
+      web-app-gitea:          2201
+      web-app-gitlab:         2202
     ldaps:
-      svc-db-openldap: 636
-    stun:
-      web-app-bigbluebutton: 3478    # Not sure if it's right placed here or if it should be moved to localhost section
-      # Occupied by BBB:     3479
-      web-app-nextcloud:     3480
-    turn:
-      web-app-bigbluebutton: 5349    # Not sure if it's right placed here or if it should be moved to localhost section
-      web-app-nextcloud:     5350        # Not used yet
+      svc-db-openldap:        636
+    stun_turn:
+      web-app-bigbluebutton:  3478  # Not sure if it's right placed here or if it should be moved to localhost section
+      # Occupied by BBB:      3479
+      web-app-nextcloud:      3480
+      web-svc-coturn:         3481
+    stun_turn_tls:
+      web-app-bigbluebutton:  5349  # Not sure if it's right placed here or if it should be moved to localhost section
+      web-app-nextcloud:      5350  # Not used yet
+      web-svc-coturn:         5351
     federation:
       web-app-matrix_synapse: 8448
+    relay_port_ranges:
+      web-svc-coturn_start:         20000
+      web-svc-coturn_end:           39999
+      web-app-bigbluebutton_start:  40000
+      web-app-bigbluebutton_end:    49999
+      web-app-nextcloud_start:      50000
+      web-app-nextcloud_end:        59999

group_vars/all/18_resource.yml

@@ -20,9 +20,15 @@ RESOURCE_ACTIVE_DOCKER_CONTAINER_COUNT: >-
 # Per-container fair share (numbers!), later we append 'g' only for the string fields in compose
 RESOURCE_CPUS_NUM: >-
   {{
-    ((RESOURCE_AVAIL_CPUS | float) / (RESOURCE_ACTIVE_DOCKER_CONTAINER_COUNT | float))
-    | round(2)
+    [
+      (
+        ((RESOURCE_AVAIL_CPUS | float) / (RESOURCE_ACTIVE_DOCKER_CONTAINER_COUNT | float))
+        | round(2)
+      ),
+      0.5
+    ] | max
   }}
+
 RESOURCE_MEM_RESERVATION_NUM: >-
   {{
     (((RESOURCE_AVAIL_MEM  | float) / (RESOURCE_ACTIVE_DOCKER_CONTAINER_COUNT | float)) * 0.7)
@@ -38,4 +44,4 @@ RESOURCE_MEM_LIMIT_NUM: >-
 RESOURCE_CPUS:            "{{ RESOURCE_CPUS_NUM }}"
 RESOURCE_MEM_RESERVATION: "{{ RESOURCE_MEM_RESERVATION_NUM }}g"
 RESOURCE_MEM_LIMIT:       "{{ RESOURCE_MEM_LIMIT_NUM }}g"
-RESOURCE_PIDS_LIMIT: 512
+RESOURCE_PIDS_LIMIT:      512

module_utils/cert_utils.py

@@ -6,6 +6,7 @@ __metaclass__ = type
 import os
 import subprocess
 import time
+from datetime import datetime
 
 class CertUtils:
     _domain_cert_mapping = None
@@ -22,6 +23,30 @@ class CertUtils:
         except subprocess.CalledProcessError:
             return ""
 
+    @staticmethod
+    def run_openssl_dates(cert_path):
+        """
+        Returns (not_before_ts, not_after_ts) as POSIX timestamps or (None, None) on failure.
+        """
+        try:
+            output = subprocess.check_output(
+                ['openssl', 'x509', '-in', cert_path, '-noout', '-startdate', '-enddate'],
+                universal_newlines=True
+            )
+            nb, na = None, None
+            for line in output.splitlines():
+                line = line.strip()
+                if line.startswith('notBefore='):
+                    nb = line.split('=', 1)[1].strip()
+                elif line.startswith('notAfter='):
+                    na = line.split('=', 1)[1].strip()
+            def _parse(openssl_dt):
+                # OpenSSL format example: "Oct 10 12:34:56 2025 GMT"
+                return int(datetime.strptime(openssl_dt, "%b %d %H:%M:%S %Y %Z").timestamp())
+            return (_parse(nb) if nb else None, _parse(na) if na else None)
+        except Exception:
+            return (None, None)
+
     @staticmethod
     def extract_sans(cert_text):
         dns_entries = []
@@ -59,7 +84,6 @@ class CertUtils:
         else:
             return domain == san
 
-
     @classmethod
     def build_snapshot(cls, cert_base_path):
         snapshot = []
@@ -82,6 +106,17 @@ class CertUtils:
 
     @classmethod
     def refresh_cert_mapping(cls, cert_base_path, debug=False):
+        """
+        Build mapping: SAN -> list of entries
+        entry = {
+            'folder': str,
+            'cert_path': str,
+            'mtime': float,
+            'not_before': int|None,
+            'not_after': int|None,
+            'is_wildcard': bool
+        }
+        """
         cert_files = cls.list_cert_files(cert_base_path)
         mapping = {}
         for cert_path in cert_files:
@@ -90,46 +125,82 @@ class CertUtils:
                 continue
             sans = cls.extract_sans(cert_text)
             folder = os.path.basename(os.path.dirname(cert_path))
+            try:
+                mtime = os.stat(cert_path).st_mtime
+            except FileNotFoundError:
+                mtime = 0.0
+            nb, na = cls.run_openssl_dates(cert_path)
+
             for san in sans:
-                if san not in mapping:
-                    mapping[san] = folder
+                entry = {
+                    'folder': folder,
+                    'cert_path': cert_path,
+                    'mtime': mtime,
+                    'not_before': nb,
+                    'not_after': na,
+                    'is_wildcard': san.startswith('*.'),
+                }
+                mapping.setdefault(san, []).append(entry)
+
         cls._domain_cert_mapping = mapping
         if debug:
-            print(f"[DEBUG] Refreshed domain-to-cert mapping: {mapping}")
+            print(f"[DEBUG] Refreshed domain-to-cert mapping (counts): "
+                  f"{ {k: len(v) for k, v in mapping.items()} }")
 
     @classmethod
     def ensure_cert_mapping(cls, cert_base_path, debug=False):
         if cls._domain_cert_mapping is None or cls.snapshot_changed(cert_base_path):
             cls.refresh_cert_mapping(cert_base_path, debug)
 
+    @staticmethod
+    def _score_entry(entry):
+        """
+        Return tuple used for sorting newest-first:
+        (not_before or -inf, mtime)
+        """
+        nb = entry.get('not_before')
+        mtime = entry.get('mtime', 0.0)
+        return (nb if nb is not None else -1, mtime)
+
     @classmethod
     def find_cert_for_domain(cls, domain, cert_base_path, debug=False):
         cls.ensure_cert_mapping(cert_base_path, debug)
 
-        exact_match = None
-        wildcard_match = None
+        candidates_exact = []
+        candidates_wild = []
 
-        for san, folder in cls._domain_cert_mapping.items():
+        for san, entries in cls._domain_cert_mapping.items():
             if san == domain:
-                exact_match = folder
-                break
-            if san.startswith('*.'):
+                candidates_exact.extend(entries)
+            elif san.startswith('*.'):
                 base = san[2:]
                 if domain.count('.') == base.count('.') + 1 and domain.endswith('.' + base):
-                    wildcard_match = folder
+                    candidates_wild.extend(entries)
 
-        if exact_match:
-            if debug:
-                print(f"[DEBUG] Exact match for {domain} found in {exact_match}")
-            return exact_match
+        def _pick_newest(entries):
+            if not entries:
+                return None
+            # newest by (not_before, mtime)
+            best = max(entries, key=cls._score_entry)
+            return best
 
-        if wildcard_match:
-            if debug:
-                print(f"[DEBUG] Wildcard match for {domain} found in {wildcard_match}")
-            return wildcard_match
+        best_exact = _pick_newest(candidates_exact)
+        best_wild = _pick_newest(candidates_wild)
+
+        if best_exact and debug:
+            print(f"[DEBUG] Best exact match for {domain}: {best_exact['folder']} "
+                  f"(not_before={best_exact['not_before']}, mtime={best_exact['mtime']})")
+        if best_wild and debug:
+            print(f"[DEBUG] Best wildcard match for {domain}: {best_wild['folder']} "
+                  f"(not_before={best_wild['not_before']}, mtime={best_wild['mtime']})")
+
+        # Prefer exact if it exists; otherwise wildcard
+        chosen = best_exact or best_wild
+
+        if chosen:
+            return chosen['folder']
 
         if debug:
             print(f"[DEBUG] No certificate folder found for {domain}")
 
         return None
-

module_utils/config_utils.py

@@ -24,7 +24,7 @@ class ConfigEntryNotSetError(AppConfigKeyError):
     pass
 
 
-def get_app_conf(applications, application_id, config_path, strict=True, default=None):
+def get_app_conf(applications, application_id, config_path, strict=True, default=None, skip_missing_app=False):
     # Path to the schema file for this application
     schema_path = os.path.join('roles', application_id, 'schema', 'main.yml')
 
@@ -133,6 +133,9 @@ def get_app_conf(applications, application_id, config_path, strict=True, default
     try:
         obj = applications[application_id]
     except KeyError:
+        if skip_missing_app:
+            # Simply return default instead of failing
+            return default if default is not None else False
         raise AppConfigKeyError(
             f"Application ID '{application_id}' not found in applications dict.\n"
             f"path_trace: {path_trace}\n"

module_utils/manager/inventory.py

@@ -142,7 +142,8 @@ class InventoryManager:
         """
         if algorithm == "random_hex":
             return secrets.token_hex(64)
-        
+        if algorithm == "random_hex_32":
+            return secrets.token_hex(32)
         if algorithm == "sha256":
             return hashlib.sha256(secrets.token_bytes(32)).hexdigest()
         if algorithm == "sha1":

requirements.yml

@@ -3,4 +3,7 @@ collections:
   - name: community.general
   - name: hetzner.hcloud
 yay:
-  - python-simpleaudio
+  - python-simpleaudio
+  - python-numpy
+pacman:
+  - ansible

roles/categories.yml

@@ -153,6 +153,11 @@ roles:
       description: "Core AI building blocks—model serving, OpenAI-compatible gateways, vector databases, orchestration, and chat UIs."
       icon: "fas fa-brain"
       invokable: true
+    bkp:
+      title: "Backup Services"
+      description: "Service-level backup and recovery components—handling automated data snapshots, remote backups, synchronization services, and backup orchestration across databases, files, and containers."
+      icon: "fas fa-database"
+      invokable: true
   user:
     title: "Users & Access"
     description: "User accounts & access control"

roles/dev-locales/templates/locale.gen.j2

@@ -127,7 +127,7 @@
 #de_BE@euro ISO-8859-15  
 #de_CH.UTF-8 UTF-8  
 #de_CH ISO-8859-1  
-de_DE.UTF-8 UTF-8  
+#de_DE.UTF-8 UTF-8
 #de_DE ISO-8859-1  
 #de_DE@euro ISO-8859-15  
 #de_IT.UTF-8 UTF-8  

roles/dev-yay/defaults/main.yml

@@ -0,0 +1,4 @@
+AUR_HELPER:               yay
+AUR_BUILDER_USER:         aur_builder
+AUR_BUILDER_GROUP:        wheel
+AUR_BUILDER_SUDOERS_PATH: /etc/sudoers.d/11-install-aur_builder

roles/dev-yay/tasks/01_core.yml

@@ -6,42 +6,53 @@
   - dev-git
   - dev-base-devel
 
-- name: install yay
+- name: Install yay build prerequisites
   community.general.pacman:
     name:
     - base-devel
     - patch
     state: present
 
-- name: Create the `aur_builder` user
+- name: Create the AUR builder user
   become: true
   ansible.builtin.user:
-    name: aur_builder
+    name: "{{ AUR_BUILDER_USER }}"
     create_home: yes
-    group: wheel
+    group: "{{ AUR_BUILDER_GROUP }}"
 
-- name: Allow the `aur_builder` user to run `sudo pacman` without a password
+- name: Allow AUR builder to run pacman without password
   become: true
   ansible.builtin.lineinfile:
-    path: /etc/sudoers.d/11-install-aur_builder
-    line: 'aur_builder ALL=(ALL) NOPASSWD: /usr/bin/pacman'
+    path: "{{ AUR_BUILDER_SUDOERS_PATH }}"
+    line: '{{ AUR_BUILDER_USER }} ALL=(ALL) NOPASSWD: /usr/bin/pacman'
     create: yes
     validate: 'visudo -cf %s'
 
 - name: Clone yay from AUR
   become: true
-  become_user: aur_builder
+  become_user: "{{ AUR_BUILDER_USER }}"
   git:
     repo: https://aur.archlinux.org/yay.git
-    dest: /home/aur_builder/yay
+    dest: "/home/{{ AUR_BUILDER_USER }}/yay"
     clone: yes
     update: yes
 
 - name: Build and install yay
   become: true
-  become_user: aur_builder
+  become_user: "{{ AUR_BUILDER_USER }}"
   shell: |
-    cd /home/aur_builder/yay
+    cd /home/{{ AUR_BUILDER_USER }}/yay
     makepkg -si --noconfirm
   args:
     creates: /usr/bin/yay
+
+- name: upgrade the system using yay, only act on AUR packages.
+  become: true
+  become_user: "{{ AUR_BUILDER_USER }}"
+  kewlfft.aur.aur:
+    upgrade: yes
+    use: "{{ AUR_HELPER }}"
+    aur_only: yes
+  when: MODE_UPDATE | bool
+
+- include_tasks: utils/run_once.yml

roles/dev-yay/tasks/main.yml

@@ -1,5 +1,3 @@
 - block:
   - include_tasks: 01_core.yml
-  - set_fact:
-      run_once_dev_yay: true
   when: run_once_dev_yay is not defined

roles/docker-compose/tasks/03_repository.yml

@@ -1,15 +1,18 @@
 - name: Set default docker_repository_path
   set_fact:
-    docker_repository_path: "{{docker_compose.directories.services}}repository/"
+    docker_repository_path: "{{ [ docker_compose.directories.services, 'repository/' ] | path_join }}"
 
 - name: pull docker repository
   git:
-    repo:       "{{ docker_repository_address }}"
-    dest:       "{{ docker_repository_path }}"
-    version:    "{{ docker_repository_branch | default('main') }}"
-    depth:      1
-    update:     yes
-    recursive:  yes
+    repo:           "{{ docker_repository_address }}"
+    dest:           "{{ docker_repository_path }}"
+    version:        "{{ docker_repository_branch | default('main') }}"
+    single_branch:  yes
+    depth:          1
+    update:         yes
+    recursive:      yes
+    force:          yes
+    accept_hostkey: yes
   notify:
     - docker compose build
     - docker compose up

roles/docker-compose/tasks/04_files.yml

@@ -28,6 +28,21 @@
     - env_template is failed
     - "'Could not find or access' not in env_template.msg"
 
+- name:           "Create (optional) '{{ docker_compose.files.docker_compose_override }}'"
+  template:
+    src:          "{{ item }}"
+    dest:         "{{ docker_compose.files.docker_compose_override }}"
+    mode:         '770'
+    force:        yes
+  notify:         docker compose up
+  register:       docker_compose_override_template
+  loop:
+    - "{{ application_id | abs_role_path_by_application_id }}/templates/docker-compose.override.yml.j2"
+    - "{{ application_id | abs_role_path_by_application_id }}/files/docker-compose.override.yml"
+  failed_when:
+    - docker_compose_override_template is failed
+    - "'Could not find or access' not in docker_compose_override_template.msg"
+
 - name:           "Create (obligatoric) '{{ docker_compose.files.docker_compose }}'"
   template:
     src:          "docker-compose.yml.j2"

roles/docker-compose/users/main.yml

@@ -1,4 +1,6 @@
 users:
   blackhole:
     description: "Everything what will be send to this user will disapear"
-    username: "blackhole"
+    username: "blackhole"
+    roles:
+      - mail-bot

roles/docker-compose/vars/docker-compose.yml

@@ -1,2 +1,4 @@
 # @See https://chatgpt.com/share/67a23d18-fb54-800f-983c-d6d00752b0b4
-docker_compose: "{{ application_id | get_docker_paths(PATH_DOCKER_COMPOSE_INSTANCES) }}"
+docker_compose:               "{{ application_id | get_docker_paths(PATH_DOCKER_COMPOSE_INSTANCES) }}"
+docker_compose_command_base:  "docker compose --env-file {{ docker_compose.files.env }}"
+docker_compose_command_exec:  "{{ docker_compose_command_base }} exec"

roles/docker-container/templates/base.yml.j2

@@ -1,11 +1,13 @@
 {# Base for docker services #}
 
-    restart: {{ DOCKER_RESTART_POLICY }}
+    restart: {{ docker_restart_policy | default(DOCKER_RESTART_POLICY) }}
 {% if application_id | has_env  %}
     env_file:
       - "{{ docker_compose.files.env }}"
 {% endif %}
     logging:
       driver: journald
-    {{ lookup('template', 'roles/docker-container/templates/resource.yml.j2') | indent(4) }}
+{% filter indent(4) %}
+    {% include 'roles/docker-container/templates/resource.yml.j2' %}
+{% endfilter %}
 {{ "\n" }}

roles/docker-container/templates/healthcheck/http.yml.j2

@@ -0,0 +1,31 @@
+{# ------------------------------------------------------------------------------
+  Healthcheck: HTTP Local
+  ------------------------------------------------------------------------------
+  This template defines a generic HTTP healthcheck for containers exposing
+  a web service on a local port (e.g., Nginx, Apache, PHP-FPM, Shopware, etc.).
+
+  It uses `wget` or `curl` (as fallback) to test if the container responds on
+  http://127.0.0.1:{{ container_port }}/. If the request succeeds, Docker marks
+  the container as "healthy"; otherwise, as "unhealthy".
+
+  Parameters:
+    - container_port: The internal port the service listens on.
+
+  Timing:
+    - interval:      30s   → Check every 30 seconds
+    - timeout:        5s   → Each check must complete within 5 seconds
+    - retries:        5    → Mark unhealthy after 5 consecutive failures
+    - start_period:  20s   → Grace period before health checks begin
+
+  Usage:
+  {% filter indent(4) %}
+      {% include 'roles/docker-container/templates/healthcheck/http.yml.j2' %}
+  {% endfilter %}
+  ------------------------------------------------------------------------------
+#}
+healthcheck:
+  test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:{{ container_port }}/ >/dev/null || curl -fsS http://127.0.0.1:{{ container_port }}/ >/dev/null"]
+  interval: 30s
+  timeout: 5s
+  retries: 5
+  start_period: 20s

roles/pkgmgr-install/tasks/main.yml

@@ -4,7 +4,7 @@
       run_once_pkgmgr_install: true
   when: run_once_pkgmgr_install is not defined
 
-- name: update {{ package_name }}
+- name: "update {{ package_name }}"
   ansible.builtin.shell: |
     source ~/.venvs/pkgmgr/bin/activate
     pkgmgr update {{ package_name }} --dependencies --clone-mode https

roles/svc-bkp-rmt-2-loc/files/pull-specific-host.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import subprocess
+import time
+import sys
+
+
+def run_command(command, capture_output=True, check=False, shell=True):
+    """Run a shell command and return its output as string."""
+    try:
+        result = subprocess.run(
+            command,
+            capture_output=capture_output,
+            shell=shell,
+            text=True,
+            check=check
+        )
+        return result.stdout.strip()
+    except subprocess.CalledProcessError as e:
+        if capture_output:
+            print(e.stdout)
+            print(e.stderr)
+        raise
+
+
+def pull_backups(hostname: str):
+    print(f"pulling backups from: {hostname}")
+    errors = 0
+
+    print("loading meta data...")
+    remote_host = f"backup@{hostname}"
+    print(f"host address:         {remote_host}")
+
+    remote_machine_id = run_command(f'ssh "{remote_host}" sha256sum /etc/machine-id')[:64]
+    print(f"remote machine id:    {remote_machine_id}")
+
+    general_backup_machine_dir = f"/Backups/{remote_machine_id}/"
+    print(f"backup dir:           {general_backup_machine_dir}")
+
+    try:
+        remote_backup_types = run_command(
+            f'ssh "{remote_host}" "find {general_backup_machine_dir} -maxdepth 1 -type d -execdir basename {{}} ;"'
+        ).splitlines()
+        print(f"backup types:          {' '.join(remote_backup_types)}")
+    except subprocess.CalledProcessError:
+        sys.exit(1)
+
+    for backup_type in remote_backup_types:
+        if backup_type == remote_machine_id:
+            continue
+
+        print(f"backup type:              {backup_type}")
+
+        general_backup_type_dir = f"{general_backup_machine_dir}{backup_type}/"
+        general_versions_dir = general_backup_type_dir
+
+        # local previous version
+        try:
+            local_previous_version_dir = run_command(f"ls -d {general_versions_dir}* | tail -1")
+        except subprocess.CalledProcessError:
+            local_previous_version_dir = ""
+        print(f"last local backup:      {local_previous_version_dir}")
+
+        # remote versions
+        remote_backup_versions = run_command(
+            f'ssh "{remote_host}" "ls -d /Backups/{remote_machine_id}/backup-docker-to-local/*"'
+        ).splitlines()
+        print(f"remote backup versions:   {' '.join(remote_backup_versions)}")
+
+        remote_last_backup_dir = remote_backup_versions[-1] if remote_backup_versions else ""
+        print(f"last remote backup:       {remote_last_backup_dir}")
+
+        remote_source_path = f"{remote_host}:{remote_last_backup_dir}/"
+        print(f"source path:              {remote_source_path}")
+
+        local_backup_destination_path = remote_last_backup_dir
+        print(f"backup destination:       {local_backup_destination_path}")
+
+        print("creating local backup destination folder...")
+        os.makedirs(local_backup_destination_path, exist_ok=True)
+
+        rsync_command = (
+            f'rsync -abP --delete --delete-excluded --rsync-path="sudo rsync" '
+            f'--link-dest="{local_previous_version_dir}" "{remote_source_path}" "{local_backup_destination_path}"'
+        )
+        print("starting backup...")
+        print(f"executing:                {rsync_command}")
+
+        retry_count = 0
+        max_retries = 12
+        retry_delay = 300  # 5 minutes
+        last_retry_start = 0
+        max_retry_duration = 43200  # 12 hours
+
+        rsync_exit_code = 1
+        while retry_count < max_retries:
+            print(f"Retry attempt: {retry_count + 1}")
+            if retry_count > 0:
+                current_time = int(time.time())
+                last_retry_duration = current_time - last_retry_start
+                if last_retry_duration >= max_retry_duration:
+                    print("Last retry took more than 12 hours, increasing max retries to 12.")
+                    max_retries = 12
+            last_retry_start = int(time.time())
+            rsync_exit_code = os.system(rsync_command)
+            if rsync_exit_code == 0:
+                break
+            retry_count += 1
+            time.sleep(retry_delay)
+
+        if rsync_exit_code != 0:
+            print(f"Error: rsync failed after {max_retries} attempts")
+            errors += 1
+
+    sys.exit(errors)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Pull backups from a remote backup host via rsync."
+    )
+    parser.add_argument(
+        "hostname",
+        help="Hostname from which backup should be pulled"
+    )
+    args = parser.parse_args()
+    pull_backups(args.hostname)
+
+
+if __name__ == "__main__":
+    main()

roles/svc-bkp-rmt-2-loc/files/sys-bkp-rmt-2-loc.sh

@@ -1,85 +0,0 @@
-#!/bin/bash
-# @param $1 hostname from which backup should be pulled
-
-echo "pulling backups from: $1" &&
-
-# error counter
-errors=0 &&
-
-echo "loading meta data..." &&
-
-remote_host="backup@$1" &&
-echo "host address:         $remote_host" &&
-
-remote_machine_id="$( (ssh "$remote_host" sha256sum /etc/machine-id) | head -c 64 )" &&
-echo "remote machine id:    $remote_machine_id" &&
-
-general_backup_machine_dir="/Backups/$remote_machine_id/" &&
-echo "backup dir:           $general_backup_machine_dir" &&
-
-remote_backup_types="$(ssh "$remote_host" "find $general_backup_machine_dir -maxdepth 1 -type d -execdir basename {} ;")" &&
-echo "backup types:          $remote_backup_types" || exit 1
-
-for backup_type in $remote_backup_types; do
-  if [ "$backup_type" != "$remote_machine_id" ]; then
-    echo "backup type:              $backup_type" &&
-    
-    general_backup_type_dir="$general_backup_machine_dir""$backup_type/" &&
-    general_versions_dir="$general_backup_type_dir" &&
-    local_previous_version_dir="$(ls -d $general_versions_dir* | tail -1)" &&
-    echo "last local backup:      $local_previous_version_dir" &&
-
-    remote_backup_versions="$(ssh "$remote_host" ls -d "$general_backup_type_dir"\*)" &&
-    echo "remote backup versions:   $remote_backup_versions" &&
-
-
-    remote_last_backup_dir=$(echo "$remote_backup_versions" | tail -1) &&
-    echo "last remote backup:       $remote_last_backup_dir" &&
-
-    remote_source_path="$remote_host:$remote_last_backup_dir/" &&
-    echo "source path:              $remote_source_path" &&
-
-    local_backup_destination_path=$remote_last_backup_dir &&
-    echo "backup destination:       $local_backup_destination_path" &&
-
-    echo "creating local backup destination folder..." &&
-    mkdir -vp "$local_backup_destination_path" &&
-
-    echo "starting backup..."
-    rsync_command='rsync -abP --delete --delete-excluded --rsync-path="sudo rsync" --link-dest="'$local_previous_version_dir'" "'$remote_source_path'" "'$local_backup_destination_path'"'
-
-    echo "executing:                $rsync_command"
-
-    retry_count=0
-    max_retries=12
-    retry_delay=300  # Retry delay in seconds (5 minutes)
-    last_retry_start=0
-    max_retry_duration=43200  # Maximum duration for a single retry attempt (12 hours)
-
-    while [[ $retry_count -lt $max_retries ]]; do
-      echo "Retry attempt: $((retry_count + 1))"
-      if [[ $retry_count -gt 0 ]]; then
-        current_time=$(date +%s)
-        last_retry_duration=$((current_time - last_retry_start))
-        if [[ $last_retry_duration -ge $max_retry_duration ]]; then
-          echo "Last retry took more than 12 hours, increasing max retries to 12."
-          max_retries=12
-        fi
-      fi
-      last_retry_start=$(date +%s)
-      eval "$rsync_command"
-      rsync_exit_code=$?
-      if [[ $rsync_exit_code -eq 0 ]]; then
-        break
-      fi
-      retry_count=$((retry_count + 1))
-      sleep $retry_delay
-    done
-
-    if [[ $rsync_exit_code -ne 0 ]]; then
-      echo "Error: rsync failed after $max_retries attempts"
-      ((errors += 1))
-    fi
-  fi
-done
-exit $errors;

roles/svc-bkp-rmt-2-loc/tasks/main.yml

@@ -10,15 +10,15 @@
   - include_tasks: utils/run_once.yml
   when: run_once_svc_bkp_rmt_2_loc is not defined
 
-- name: "create {{ DOCKER_BACKUP_REMOTE_2_LOCAL_DIR }}"
+- name: "Create Directory '{{ DOCKER_BACKUP_REMOTE_2_LOCAL_DIR }}'"
   file:
     path: "{{ DOCKER_BACKUP_REMOTE_2_LOCAL_DIR }}"
     state: directory
     mode: "0755"
 
-- name: create svc-bkp-rmt-2-loc.sh
+- name: "Deploy '{{ DOCKER_BACKUP_REMOTE_2_LOCAL_SCRIPT }}'"
   copy:
-    src: svc-bkp-rmt-2-loc.sh
+    src:  "{{ DOCKER_BACKUP_REMOTE_2_LOCAL_FILE }}"
     dest: "{{ DOCKER_BACKUP_REMOTE_2_LOCAL_SCRIPT }}"
     mode: "0755"
 

roles/svc-bkp-rmt-2-loc/templates/script.sh.j2

@@ -3,6 +3,6 @@
 hosts="{{ DOCKER_BACKUP_REMOTE_2_LOCAL_BACKUP_PROVIDERS | join(' ') }}";
 errors=0
 for host in $hosts; do
-  bash {{ DOCKER_BACKUP_REMOTE_2_LOCAL_SCRIPT }} $host || ((errors+=1));
+  python {{ DOCKER_BACKUP_REMOTE_2_LOCAL_SCRIPT }} $host || ((errors+=1));
 done;
 exit $errors;

roles/svc-bkp-rmt-2-loc/vars/main.yml

@@ -1,5 +1,9 @@
+# General
 application_id:                                 svc-bkp-rmt-2-loc
-system_service_id:                                   "{{ application_id }}"  
+system_service_id:                              "{{ application_id }}" 
+
+# Role Specific
 DOCKER_BACKUP_REMOTE_2_LOCAL_DIR:               '{{ PATH_ADMINISTRATOR_SCRIPTS }}{{ application_id }}/'
-DOCKER_BACKUP_REMOTE_2_LOCAL_SCRIPT:            "{{ DOCKER_BACKUP_REMOTE_2_LOCAL_DIR }}svc-bkp-rmt-2-loc.sh"
+DOCKER_BACKUP_REMOTE_2_LOCAL_FILE:              'pull-specific-host.py'
+DOCKER_BACKUP_REMOTE_2_LOCAL_SCRIPT:            "{{ [ DOCKER_BACKUP_REMOTE_2_LOCAL_DIR , DOCKER_BACKUP_REMOTE_2_LOCAL_FILE ] | path_join }}"
 DOCKER_BACKUP_REMOTE_2_LOCAL_BACKUP_PROVIDERS:  "{{ applications | get_app_conf(application_id, 'backup_providers')  }}"

roles/svc-db-openldap/config/main.yml

@@ -5,9 +5,14 @@ network:
 docker: 
   services:
     openldap:     
-      image:      "bitnami/openldap"
+      image:      "bitnamilegacy/openldap"
       name:       "openldap"
       version:    "latest"
+      cpus: 1.25
+      # Optimized up to 5k user
+      mem_reservation: 1g
+      mem_limit: 1.5g
+      pids_limit: 1024
   network:        "openldap"
   volumes:
     data:         "openldap_data"

roles/svc-db-postgres/config/main.yml

@@ -6,7 +6,7 @@ docker:
       name:               postgres
       # Please set an version in your inventory file! 
       # Rolling release isn't recommended
-      version:            "latest"
+      version:            "17-3.5"
       backup:
         database_routine: true
       cpus:               "2.0"
@@ -14,5 +14,5 @@ docker:
       mem_limit:          "6g"
       pids_limit:         1024
   volumes:
-    data:     "postgres_data"
-  network:    "postgres"
+    data:                 "postgres_data"
+  network:                "postgres"

roles/svc-db-postgres/templates/Dockerfile.j2

@@ -5,7 +5,7 @@ RUN apt-get update \
  && apt-get install -y --no-install-recommends \
       build-essential \
       git \
-      postgresql-server-dev-all \
+      postgresql-server-dev-{{ POSTGRES_VERSION_MAJOR | default('all', true) }} \
  && git clone https://github.com/pgvector/pgvector.git /tmp/pgvector \
  && cd /tmp/pgvector \
  && make \

roles/svc-db-postgres/vars/main.yml

@@ -1,19 +1,21 @@
 # General
 application_id:                           svc-db-postgres
+entity_name:                              "{{ application_id | get_entity_name }}"
 
 # Docker
 docker_compose_flush_handlers:            true
 
 # Docker Compose
-database_type:                            "{{ application_id | get_entity_name }}"
+database_type:                            "{{ entity_name }}"
 
 ## Postgres
 POSTGRES_VOLUME:                          "{{ applications | get_app_conf(application_id, 'docker.volumes.data') }}"
-POSTGRES_CONTAINER:                       "{{ applications | get_app_conf(application_id, 'docker.services.postgres.name') }}"
-POSTGRES_IMAGE:                           "{{ applications | get_app_conf(application_id, 'docker.services.postgres.image') }}"
-POSTGRES_SUBNET:                          "{{ networks.local['svc-db-postgres'].subnet }}"
+POSTGRES_CONTAINER:                       "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.name') }}"
+POSTGRES_IMAGE:                           "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.image') }}"
+POSTGRES_VERSION:                         "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.version') }}"
+POSTGRES_VERSION_MAJOR:                   "{{ POSTGRES_VERSION | regex_replace('^([0-9]+).*', '\\1') }}"
 POSTGRES_NETWORK_NAME:                    "{{ applications | get_app_conf(application_id, 'docker.network') }}"
-POSTGRES_VERSION:                         "{{ applications | get_app_conf(application_id, 'docker.services.postgres.version') }}"
+POSTGRES_SUBNET:                          "{{ networks.local['svc-db-postgres'].subnet }}"
 POSTGRES_PASSWORD:                        "{{ applications | get_app_conf(application_id, 'credentials.POSTGRES_PASSWORD') }}"
 POSTGRES_PORT:                            "{{ database_port | default(ports.localhost.database[ application_id ]) }}"
 POSTGRES_INIT:                            "{{ database_username is defined and database_password is defined and database_name is defined }}"

roles/svc-db-redis/templates/service.yml.j2

@@ -16,4 +16,12 @@
       retries: 30
     networks:
       - default
+{% macro include_resource_for(svc, indent=4) -%}
+  {% set service_name = svc -%}
+  {%- set _snippet -%}
+{% include 'roles/docker-container/templates/resource.yml.j2' %}
+  {%- endset -%}
+{{ _snippet | indent(indent, true) }}
+{%- endmacro %}
+{{ include_resource_for('redis') }}
 {{ "\n" }}

roles/svc-opt-swapfile/tasks/01_core.yml

@@ -0,0 +1,14 @@
+- name: Install '
+  include_role:
+    name: pkgmgr-install
+  vars:
+    package_name: "{{ SWAPFILE_PKG }}"
+  when: run_once_pkgmgr_install is not defined
+
+- name: Execute create swapfile script
+  shell:  "{{ SWAPFILE_PKG }} '{{ SWAPFILE_SIZE }}'"
+  become: true
+  async: "{{ ASYNC_TIME if ASYNC_ENABLED | bool else omit }}"
+  poll:  "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}"
+
+- include_tasks: utils/run_once.yml

roles/svc-opt-swapfile/tasks/main.yml

@@ -1,19 +1,3 @@
 - block:
-  - name: Include dependency 'pkgmgr-install'
-    include_role:
-      name: pkgmgr-install
-    when: run_once_pkgmgr_install is not defined
-  - include_tasks: utils/run_once.yml
+    - include_tasks: 01_core.yml
   when: run_once_svc_opt_swapfile is not defined
-
-- name: "pkgmgr install"
-  include_role:
-    name: pkgmgr-install
-  vars:
-    package_name: swap-forge
-
-- name: Execute create swapfile script
-  shell: swap-forge "{{ SWAPFILE_SIZE }}"
-  become: true
-  async: "{{ ASYNC_TIME if ASYNC_ENABLED | bool else omit }}"
-  poll:  "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}"

roles/svc-opt-swapfile/vars/main.yml

@@ -1,3 +1,4 @@
 application_id: "svc-opt-swapfile"
 
-SWAPFILE_SIZE:  "{{ applications | get_app_conf(application_id, 'swapfile_size') }}"
+SWAPFILE_SIZE:  "{{ applications | get_app_conf(application_id, 'swapfile_size') }}"
+SWAPFILE_PKG:   "swap-forge"

roles/sys-bkp-provider-user/files/ssh-wrapper.sh

@@ -13,7 +13,7 @@ get_backup_types="find /Backups/$hashed_machine_id/ -maxdepth 1 -type d -execdir
 
 
 # @todo This configuration is not scalable yet. If other backup services then sys-ctl-bkp-docker-2-loc are integrated, this logic needs to be optimized
-get_version_directories="ls -d /Backups/$hashed_machine_id/sys-ctl-bkp-docker-2-loc/*"
+get_version_directories="ls -d /Backups/$hashed_machine_id/backup-docker-to-local/*"
 last_version_directory="$($get_version_directories | tail -1)"
 rsync_command="sudo rsync --server --sender -blogDtpre.iLsfxCIvu . $last_version_directory/"
 

roles/sys-bkp-provider-user/tasks/01_core.yml

@@ -3,30 +3,6 @@
     name: backup
     create_home: yes
 
-- name: create .ssh directory
-  file:
-    path: /home/backup/.ssh
-    state: directory
-    owner: backup
-    group: backup
-    mode: '0700'
-
-- name: create /home/backup/.ssh/authorized_keys
-  template:
-    src: "authorized_keys.j2"
-    dest: /home/backup/.ssh/authorized_keys
-    owner: backup
-    group: backup
-    mode: '0644'
-
-- name: create /home/backup/ssh-wrapper.sh
-  copy:
-    src: "ssh-wrapper.sh"
-    dest: /home/backup/ssh-wrapper.sh
-    owner: backup
-    group: backup
-    mode: '0700'
-
 - name: grant backup sudo rights
   copy:
     src: "backup"
@@ -35,3 +11,9 @@
     owner: root
     group: root
   notify: sshd restart
+
+- include_tasks: 02_permissions_ssh.yml
+
+- include_tasks: 03_permissions_folders.yml
+
+- include_tasks: utils/run_once.yml

roles/sys-bkp-provider-user/tasks/02_permissions_ssh.yml

@@ -0,0 +1,23 @@
+- name: create .ssh directory
+  file:
+    path: /home/backup/.ssh
+    state: directory
+    owner: backup
+    group: backup
+    mode: '0700'
+
+- name: create /home/backup/.ssh/authorized_keys
+  template:
+    src: "authorized_keys.j2"
+    dest: /home/backup/.ssh/authorized_keys
+    owner: backup
+    group: backup
+    mode: '0644'
+
+- name: create /home/backup/ssh-wrapper.sh
+  copy:
+    src: "ssh-wrapper.sh"
+    dest: /home/backup/ssh-wrapper.sh
+    owner: backup
+    group: backup
+    mode: '0700'

roles/sys-bkp-provider-user/tasks/03_permissions_folders.yml

@@ -0,0 +1,66 @@
+# Ensure the backups root exists and is owned by backup
+- name: Ensure backups root exists and owned by backup
+  file:
+    path: "{{ BACKUPS_FOLDER_PATH }}"
+    state: directory
+    owner: backup
+    group: backup
+    mode: "0700"
+
+# Explicit ACL so 'backup' has rwx, others none
+- name: Grant ACL rwx on backups root to backup user
+  ansible.posix.acl:
+    path: "{{ BACKUPS_FOLDER_PATH }}"
+    entity: backup
+    etype: user
+    permissions: rwx
+    state: present
+
+# Set default ACLs so new entries inherit rwx for backup and nothing for others
+- name: Set default ACL (inherit) for backup user under backups root
+  ansible.posix.acl:
+    path: "{{ BACKUPS_FOLDER_PATH }}"
+    entity: backup
+    etype: user
+    permissions: rwx
+    default: true
+    state: present
+
+# Remove default ACLs for group/others (defensive hardening)
+# Default ACLs so new entries inherit only backup's rwx
+- name: Default ACL for backup user (inherit)
+  ansible.posix.acl:
+    path: "{{ BACKUPS_FOLDER_PATH }}"
+    etype: user
+    entity: backup
+    permissions: rwx
+    default: true
+    state: present
+
+# Explicitly set default group/other to no permissions (instead of absent)
+- name: Default ACL for group -> none
+  ansible.posix.acl:
+    path: "{{ BACKUPS_FOLDER_PATH }}"
+    etype: group
+    permissions: '---'
+    default: true
+    state: present
+
+- name: Default ACL for other -> none
+  ansible.posix.acl:
+    path: "{{ BACKUPS_FOLDER_PATH }}"
+    etype: other
+    permissions: '---'
+    default: true
+    state: present
+
+- name: Fix ownership level 0..2 directories to backup:backup
+  ansible.builtin.shell: >
+    find "{{ BACKUPS_FOLDER_PATH }}" -mindepth 0 -maxdepth 2 -xdev -type d -exec chown backup:backup {} +
+  changed_when: false
+
+- name: Fix perms level 0..2 directories to 0700
+  ansible.builtin.shell: >
+    find "{{ BACKUPS_FOLDER_PATH }}" -mindepth 0 -maxdepth 2 -xdev -type d -exec chmod 700 {} +
+  changed_when: false
+

roles/sys-bkp-provider-user/tasks/main.yml

@@ -1,4 +1,2 @@
-- block:
-    - include_tasks: 01_core.yml
-    - include_tasks: utils/run_once.yml
+- include_tasks: 01_core.yml
   when: run_once_sys_bkp_provider_user is not defined

roles/sys-ctl-alm-compose/tasks/01_core.yml

@@ -5,21 +5,23 @@
     - sys-ctl-alm-telegram
     - sys-ctl-alm-email
   vars:
-    flush_handlers: true
-    system_service_timer_enabled: false
-    system_service_copy_files: true
-    system_service_tpl_exec_start: "{{ system_service_script_exec }} %I"
-    system_service_tpl_on_failure: ""
+    flush_handlers:                   true
+    system_service_timer_enabled:     false
+    system_service_copy_files:        true
+    system_service_tpl_exec_start:    "{{ system_service_script_exec }} %I"
+    system_service_tpl_on_failure:    ""
+    system_service_force_linear_sync: false
 
 - name: "Include core service for '{{ system_service_id  }}'"
   include_role:
     name: sys-service
   vars:
-    flush_handlers: true
-    system_service_timer_enabled: false
-    system_service_copy_files: true
-    system_service_tpl_exec_start: "{{ system_service_script_exec }} %I"
-    system_service_tpl_on_failure: "" # No on failure needed, because it's anyhow the default on failure procedure
+    flush_handlers:                   true
+    system_service_timer_enabled:     false
+    system_service_copy_files:        true
+    system_service_tpl_exec_start:    "{{ system_service_script_exec }} %I"
+    system_service_tpl_on_failure:    "" # No on failure needed, because it's anyhow the default on failure procedure
+    system_service_force_linear_sync: false
 
 - name: Assert '{{ system_service_id }}'
   block:

roles/sys-ctl-alm-email/tasks/01_core.yml

@@ -1,8 +1,7 @@
 - name: Include dependencies
   include_role:
-    name: '{{ item }}'
-  loop:
-  - sys-svc-msmtp
+    name: "sys-svc-msmtp"
+  when: run_once_sys_svc_msmtp is not defined or run_once_sys_svc_msmtp is false
 
 - include_role:
     name: sys-service

roles/sys-ctl-bkp-docker-2-loc/tasks/01_core.yml

@@ -19,6 +19,8 @@
   vars:
     system_service_copy_files:          false
     system_service_timer_enabled:       true
+    system_service_force_linear_sync:   true
+    system_service_force_flush:         "{{ MODE_BACKUP | bool }}"
     system_service_on_calendar:         "{{ SYS_SCHEDULE_BACKUP_DOCKER_TO_LOCAL }}"
     system_service_tpl_exec_start_pre:  '/usr/bin/python {{ PATH_SYSTEM_LOCK_SCRIPT }} {{ SYS_SERVICE_GROUP_MANIPULATION | join(" ")  }} --ignore {{ SYS_SERVICE_BACKUP_DOCKER_2_LOC }} --timeout "{{ SYS_TIMEOUT_BACKUP_SERVICES }}"'
     system_service_tpl_exec_start:      "/bin/sh -c '{{ BKP_DOCKER_2_LOC_EXEC }}'"

roles/sys-ctl-bkp-docker-2-loc/tasks/main.yml

@@ -1,9 +1,6 @@
-- block:
-    - include_tasks: 01_core.yml
-  when: 
-    - run_once_sys_ctl_bkp_docker_2_loc is not defined
+- include_tasks: 01_core.yml
+  when: run_once_sys_ctl_bkp_docker_2_loc is not defined
 
 - name: "include 04_seed-database-to-backup.yml"
   include_tasks: 04_seed-database-to-backup.yml
-  when:
-    - BKP_DOCKER_2_LOC_DB_ENABLED | bool
+  when: BKP_DOCKER_2_LOC_DB_ENABLED | bool

roles/sys-ctl-cln-anon-volumes/tasks/main.yml

@@ -12,6 +12,7 @@
       system_service_tpl_exec_start:      dockreap --no-confirmation
       system_service_tpl_exec_start_pre:  "" # Anonymous volumes can allways be removed. It isn't necessary to wait for any service to stop.
       system_service_copy_files:          false
+      system_service_force_linear_sync:   false
 
   - include_tasks: utils/run_once.yml
   when:

roles/sys-ctl-cln-bkps/tasks/01_core.yml

@@ -17,9 +17,10 @@
     name: sys-service
   vars:
     system_service_tpl_on_failure:      "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
-    system_service_tpl_exec_start:      "{{ system_service_script_exec }} --backups-folder-path {{ BACKUPS_FOLDER_PATH }} --maximum-backup-size-percent {{SIZE_PERCENT_MAXIMUM_BACKUP}}"
+    system_service_tpl_exec_start:      "{{ system_service_script_exec }} --backups-folder-path {{ BACKUPS_FOLDER_PATH }} --maximum-backup-size-percent {{ SIZE_PERCENT_MAXIMUM_BACKUP }}"
     system_service_tpl_exec_start_pre:  '/usr/bin/python {{ PATH_SYSTEM_LOCK_SCRIPT }} {{ SYS_SERVICE_GROUP_MANIPULATION | join(" ")  }} --ignore {{ SYS_SERVICE_GROUP_CLEANUP | join(" ") }} --timeout "{{ SYS_TIMEOUT_BACKUP_SERVICES }}"'
     system_service_copy_files:          true
+    system_service_force_linear_sync:   false
 
 - include_tasks: utils/run_once.yml
   vars:

roles/sys-ctl-cln-certs/tasks/01_core.yml

@@ -14,6 +14,7 @@
 - include_role:
     name: sys-service
   vars:
-    system_service_timer_enabled:  true
-    system_service_on_calendar:    "{{ SYS_SCHEDULE_CLEANUP_CERTS }}"
-    system_service_copy_files:     false
+    system_service_timer_enabled:     true
+    system_service_on_calendar:       "{{ SYS_SCHEDULE_CLEANUP_CERTS }}"
+    system_service_copy_files:        false
+    system_service_force_linear_sync: false

roles/sys-ctl-cln-disc-space/tasks/01_core.yml

@@ -14,3 +14,4 @@
     system_service_tpl_on_failure:      "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
     system_service_tpl_exec_start:      "{{ system_service_script_exec }} {{ SIZE_PERCENT_CLEANUP_DISC_SPACE }}"
     system_service_tpl_exec_start_pre:  '/usr/bin/python {{ PATH_SYSTEM_LOCK_SCRIPT }} {{ SYS_SERVICE_GROUP_MANIPULATION | join(" ")  }} --ignore {{ SYS_SERVICE_GROUP_CLEANUP | join(" ") }} --timeout "{{ SYS_TIMEOUT_BACKUP_SERVICES }}"'
+    system_service_force_linear_sync:   false

roles/sys-ctl-cln-disc-space/templates/script.sh.j2

@@ -39,6 +39,18 @@ if [ "$force_freeing" = true ]; then
       docker exec -u www-data $nextcloud_application_container /var/www/html/occ versions:cleanup || exit 6
     fi
 
+    # Mastodon cleanup (remote media cache)
+    mastodon_application_container="{{ applications | get_app_conf('web-app-mastodon', 'docker.services.mastodon.name') }}"
+    mastodon_cleanup_days="1"
+
+    if [ -n "$mastodon_application_container" ] && docker ps -a --format '{% raw %}{{.Names}}{% endraw %}' | grep -qw "$mastodon_application_container"; then
+      echo "Cleaning up Mastodon media cache (older than ${mastodon_cleanup_days} days)" &&
+      docker exec -u root "$mastodon_application_container" bash -lc "bin/tootctl media remove --days=${mastodon_cleanup_days}" || exit 8
+
+      # Optional: additionally remove local thumbnail/cache files older than X days
+      # Warning: these will be regenerated when accessed, which may cause extra CPU/I/O load
+      # docker exec -u root "$mastodon_application_container" bash -lc "find /mastodon/public/system/cache -type f -mtime +${mastodon_cleanup_days} -delete" || exit 9
+    fi
   fi
 
   if command -v pacman >/dev/null 2>&1 ; then

roles/sys-ctl-cln-faild-bkps/tasks/01_core.yml

@@ -21,5 +21,5 @@
     system_service_tpl_on_failure:      "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
     system_service_tpl_exec_start_pre:  '/usr/bin/python {{ PATH_SYSTEM_LOCK_SCRIPT }} {{ SYS_SERVICE_GROUP_MANIPULATION | join(" ")  }} --ignore {{ SYS_SERVICE_GROUP_CLEANUP| join(" ") }} --timeout "{{ SYS_TIMEOUT_CLEANUP_SERVICES }}"'
     system_service_tpl_exec_start:      '/bin/sh -c "{{ CLEANUP_FAILED_BACKUPS_PKG }} --all --workers {{ CLEANUP_FAILED_BACKUPS_WORKERS }} --yes"'
-
+    system_service_force_linear_sync:   false
 - include_tasks: utils/run_once.yml

roles/sys-ctl-hlth-csp/README.md

@@ -14,6 +14,32 @@ Designed for Archlinux systems, this role periodically checks whether web resour
 - **Domain Extraction:** Parses all `.conf` files in the NGINX config folder to determine the list of domains to check.
 - **Automated Execution:** Registers a systemd service and timer for recurring health checks.
 - **Error Notification:** Integrates with `sys-ctl-alm-compose` for alerting on failure.
+- **Ignore List Support:** Optional variable to suppress network block reports from specific external domains.
+
+## Configuration
+
+### Variables
+
+- **`HEALTH_CSP_IGNORE_NETWORK_BLOCKS_FROM`** (list, default: `[]`)  
+  Optional list of domains whose network block failures (e.g., ORB) should be ignored during CSP checks.
+
+Example:
+
+```yaml
+HEALTH_CSP_IGNORE_NETWORK_BLOCKS_FROM:
+  - pxscdn.com
+  - cdn.example.org
+```
+
+This will run the CSP checker with:
+
+```bash
+checkcsp start --short --ignore-network-blocks-from pxscdn.com -- cdn.example.org <domains...>
+```
+
+### Systemd Integration
+
+The role configures a systemd service and timer which executes the CSP crawler periodically against all NGINX domains.
 
 ## License
 
@@ -24,4 +50,4 @@ Infinito.Nexus NonCommercial License
 
 Kevin Veen-Birkenbach
 Consulting & Coaching Solutions
-[https://www.veen.world](https://www.veen.world)
+[https://www.veen.world](https://www.veen.world)

roles/sys-ctl-hlth-csp/defaults/main.yml

@@ -0,0 +1,5 @@
+# List of domains whose network block failures (e.g., ORB) should be ignored
+# during CSP checks. This is useful for suppressing known external resources
+# (e.g., third-party CDNs) that cannot be influenced but otherwise cause
+# unnecessary alerts in the crawler reports.
+HEALTH_CSP_IGNORE_NETWORK_BLOCKS_FROM: []

roles/sys-ctl-hlth-csp/files/script.py

@@ -21,11 +21,20 @@ def extract_domains(config_path):
         print(f"Directory {config_path} not found.", file=sys.stderr)
         return None
 
-def run_checkcsp(domains):
+def run_checkcsp(domains, ignore_network_blocks_from):
     """
-    Executes the 'checkcsp' command with the given domains.
+    Executes the 'checkcsp' command with the given domains and optional ignores.
     """
-    cmd = ["checkcsp", "start", "--short"] + domains
+    cmd = ["checkcsp", "start", "--short"]
+
+    # pass through ignore list only if not empty
+    if ignore_network_blocks_from:
+        cmd.append("--ignore-network-blocks-from")
+        cmd.extend(ignore_network_blocks_from)
+        cmd.append("--")
+
+    cmd += domains
+
     try:
         result = subprocess.run(cmd, check=True)
         return result.returncode
@@ -45,6 +54,12 @@ def main():
         required=True,
         help="Directory containing NGINX .conf files"
     )
+    parser.add_argument(
+        "--ignore-network-blocks-from",
+        nargs="*",
+        default=[],
+        help="Optional: one or more domains whose network block failures should be ignored"
+    )
     args = parser.parse_args()
 
     domains = extract_domains(args.nginx_config_dir)
@@ -55,7 +70,7 @@ def main():
         print("No domains found to check.")
         sys.exit(0)
 
-    rc = run_checkcsp(domains)
+    rc = run_checkcsp(domains, args.ignore_network_blocks_from)
     sys.exit(rc)
 
 if __name__ == "__main__":

roles/sys-ctl-hlth-csp/tasks/01_core.yml

@@ -18,6 +18,9 @@
     system_service_timer_enabled:         true
     system_service_tpl_on_failure:        "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
     system_service_tpl_timeout_start_sec: "{{ CURRENT_PLAY_DOMAINS_ALL | timeout_start_sec_for_domains }}"
-    system_service_tpl_exec_start:        "{{ system_service_script_exec }} --nginx-config-dir={{ NGINX.DIRECTORIES.HTTP.SERVERS }}"
+    system_service_tpl_exec_start: >-
+      {{ system_service_script_exec }}
+      --nginx-config-dir={{ NGINX.DIRECTORIES.HTTP.SERVERS }}
+      --ignore-network-blocks-from {{ HEALTH_CSP_IGNORE_NETWORK_BLOCKS_FROM | join(' ') }}
 
 - include_tasks: utils/run_once.yml

roles/sys-ctl-hlth-webserver/filter_plugins/web_health_expectations.py

@@ -1,4 +1,3 @@
-# roles/sys-ctl-hlth-webserver/filter_plugins/web_health_expectations.py
 import os
 import sys
 from collections.abc import Mapping
@@ -94,6 +93,26 @@ def _normalize_selection(group_names):
         raise ValueError("web_health_expectations: 'group_names' must be provided and non-empty")
     return sel
 
+def _normalize_codes(x):
+    """
+    Accepts:
+      - single code (int or str)
+      - list/tuple/set of codes
+    Returns a de-duplicated list of valid ints (100..599) in original order.
+    """
+    if x is None:
+        return []
+    if isinstance(x, (list, tuple, set)):
+        out = []
+        seen = set()
+        for v in x:
+            c = _valid_http_code(v)
+            if c is not None and c not in seen:
+                seen.add(c)
+                out.append(c)
+        return out
+    c = _valid_http_code(x)
+    return [c] if c is not None else []
 
 def web_health_expectations(applications, www_enabled: bool = False, group_names=None, redirect_maps=None):
     """Produce a **flat mapping**: domain -> [expected_status_codes].
@@ -138,17 +157,15 @@ def web_health_expectations(applications, www_enabled: bool = False, group_names
         sc_map = {}
         if isinstance(sc_raw, Mapping):
             for k, v in sc_raw.items():
-                code = _valid_http_code(v)
-                if code is not None:
-                    sc_map[str(k)] = code
+                codes = _normalize_codes(v)
+                if codes:
+                    sc_map[str(k)] = codes
 
         if isinstance(canonical_raw, Mapping) and canonical_raw:
             for key, domains in canonical_raw.items():
                 domains_list = _to_list(domains, allow_mapping=False)
-                code = _valid_http_code(sc_map.get(key))
-                if code is None:
-                    code = _valid_http_code(sc_map.get("default"))
-                expected = [code] if code is not None else list(DEFAULT_OK)
+                codes = sc_map.get(key) or sc_map.get("default")
+                expected = list(codes) if codes else list(DEFAULT_OK)
                 for d in domains_list:
                     if d:
                         expectations[d] = expected
@@ -156,8 +173,8 @@ def web_health_expectations(applications, www_enabled: bool = False, group_names
             for d in _to_list(canonical_raw, allow_mapping=True):
                 if not d:
                     continue
-                code = _valid_http_code(sc_map.get("default"))
-                expectations[d] = [code] if code is not None else list(DEFAULT_OK)
+                codes = sc_map.get("default")
+                expectations[d] = list(codes) if codes else list(DEFAULT_OK)
 
         for d in aliases:
             if d:

roles/sys-ctl-mtn-cert-deploy/tasks/01_core.yml

@@ -8,8 +8,9 @@
 - include_role:
     name: sys-service
   vars:
-    system_service_state:         restarted
-    system_service_on_calendar:   "{{ SYS_SCHEDULE_MAINTANANCE_LETSENCRYPT_DEPLOY }}"
-    persistent:                    "true"
-    system_service_timer_enabled:  true
-    system_service_tpl_on_failure: "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
+    system_service_state:             restarted
+    system_service_on_calendar:       "{{ SYS_SCHEDULE_MAINTANANCE_LETSENCRYPT_DEPLOY }}"
+    persistent:                       "true"
+    system_service_timer_enabled:     true
+    system_service_tpl_on_failure:    "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
+    system_service_force_linear_sync: false

roles/sys-ctl-mtn-cert-renew/tasks/01_core.yml

@@ -3,7 +3,7 @@
     name: '{{ item }}'
   loop:
   - sys-svc-certbot
-  - sys-svc-webserver
+  - sys-svc-webserver-core
   - sys-ctl-alm-compose
 
 - name: install certbot
@@ -15,8 +15,9 @@
 - include_role:
     name: sys-service
   vars:
-    system_service_copy_files:      false
-    system_service_on_calendar:     "{{ SYS_SCHEDULE_MAINTANANCE_LETSENCRYPT_RENEW }}"
-    persistent:                     true
-    system_service_timer_enabled:   true
-    system_service_tpl_on_failure:  "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
+    system_service_copy_files:        false
+    system_service_on_calendar:       "{{ SYS_SCHEDULE_MAINTANANCE_LETSENCRYPT_RENEW }}"
+    persistent:                       true
+    system_service_timer_enabled:     true
+    system_service_tpl_on_failure:    "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
+    system_service_force_linear_sync: false

roles/sys-ctl-rpr-btrfs-balancer/tasks/01_core.yml

@@ -12,9 +12,10 @@
 - include_role:
     name: sys-service
   vars:
-    system_service_suppress_flush:  true # It takes a super long time - Better wait for failure of timed service instead of executing it on every play
-    system_service_copy_files:      false
-    system_service_on_calendar:     "{{ SYS_SCHEDULE_REPAIR_BTRFS_AUTO_BALANCER }}"
-    system_service_timer_enabled:   true
-    system_service_tpl_on_failure:  "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
-    system_service_tpl_exec_start:  "/bin/sh -c 'btrfs-auto-balancer 90 10'"
+    system_service_suppress_flush:    true # It takes a super long time - Better wait for failure of timed service instead of executing it on every play
+    system_service_copy_files:        false
+    system_service_on_calendar:       "{{ SYS_SCHEDULE_REPAIR_BTRFS_AUTO_BALANCER }}"
+    system_service_timer_enabled:     true
+    system_service_tpl_on_failure:    "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
+    system_service_tpl_exec_start:    "/bin/sh -c 'btrfs-auto-balancer 90 10'"
+    system_service_force_linear_sync: true

roles/sys-ctl-rpr-docker-hard/tasks/01_core.yml

@@ -12,5 +12,6 @@
     system_service_tpl_exec_start:      '{{ system_service_script_exec }} {{ PATH_DOCKER_COMPOSE_INSTANCES }}'
     system_service_tpl_exec_start_post: "/usr/bin/systemctl start {{ SYS_SERVICE_CLEANUP_ANONYMOUS_VOLUMES }}"
     system_service_tpl_on_failure:      "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
+    system_service_force_linear_sync:   true
 
 - include_tasks: utils/run_once.yml

roles/sys-ctl-rpr-docker-soft/tasks/01_core.yml

@@ -10,5 +10,6 @@
     system_service_tpl_exec_start_pre:  "/usr/bin/python {{ PATH_SYSTEM_LOCK_SCRIPT }} {{ SYS_SERVICE_GROUP_MANIPULATION | join(' ')  }} --ignore {{ SYS_SERVICE_GROUP_CLEANUP| join(' ') }} {{ SYS_SERVICE_REPAIR_DOCKER_SOFT }} --timeout '{{ SYS_TIMEOUT_DOCKER_RPR_SOFT }}'"
     system_service_tpl_exec_start: >
       /bin/sh -c '{{ system_service_script_exec }} --manipulation-string "{{ SYS_SERVICE_GROUP_MANIPULATION | join(" ") }}" {{ PATH_DOCKER_COMPOSE_INSTANCES }}'
+    system_service_force_linear_sync:   true
 
 - include_tasks: utils/run_once.yml

roles/sys-front-inj-all/tasks/01_dependencies.yml

@@ -0,0 +1,16 @@
+- name: "Load CDN for '{{ domain }}'"
+  include_role:
+    name: web-svc-cdn
+    public: false
+  when:
+    - application_id != 'web-svc-cdn'
+    - run_once_web_svc_cdn is not defined
+
+- name: Load Logout for '{{ domain }}'
+  include_role:
+    name: web-svc-logout
+    public: false
+  when: 
+    - run_once_web_svc_logout is not defined
+    - application_id != 'web-svc-logout'
+    - inj_enabled.logout

roles/sys-front-inj-all/tasks/main.yml

@@ -1,22 +1,41 @@
+- block:
+    - name: Include dependency 'sys-svc-webserver-core'
+      include_role:
+        name: sys-svc-webserver-core
+      when: run_once_sys_svc_webserver_core is not defined
+    - include_tasks: utils/run_once.yml
+  when: run_once_sys_front_inj_all is not defined
+
 - name: Build inj_enabled
   set_fact:
     inj_enabled: "{{ applications | inj_enabled(application_id, SRV_WEB_INJ_COMP_FEATURES_ALL) }}"
 
-- name: "Load CDN Service for '{{ domain }}'"
-  include_role:
-    name: sys-svc-cdn
-    public: true # Expose variables so that they can be used in all injection roles
+- name: "Included dependent services"
+  include_tasks: 01_dependencies.yml
+  vars:
+    proxy_extra_configuration: ""
 
-- name: Reinitialize 'inj_enabled' for '{{ domain }}', after modification by CDN
+- name: Reinitialize 'inj_enabled' for '{{ domain }}', after loading the required webservices
   set_fact:
     inj_enabled: "{{ applications | inj_enabled(application_id, SRV_WEB_INJ_COMP_FEATURES_ALL) }}"
     inj_head_features: "{{ SRV_WEB_INJ_COMP_FEATURES_ALL | inj_features('head') }}"
     inj_body_features: "{{ SRV_WEB_INJ_COMP_FEATURES_ALL | inj_features('body') }}"
 
+- name: "Load CDN Service for '{{ domain }}'"
+  include_role:
+    name: sys-svc-cdn
+    public: true
+
+- name: "Activate logout proxy for '{{ domain }}'"
+  include_role:
+    name: sys-front-inj-logout
+    public: true
+  when: inj_enabled.logout
+
 - name: "Activate Desktop iFrame notifier for '{{ domain }}'"
   include_role:
     name:   sys-front-inj-desktop
-    public: true # Vars used in templates
+    public: true
   when: inj_enabled.desktop
 
 - name: "Activate Corporate CSS for '{{ domain }}'"
@@ -33,17 +52,3 @@
   include_role:
     name: sys-front-inj-javascript
   when: inj_enabled.javascript
-
-- name: "Activate logout proxy for '{{ domain }}'"
-  include_role:
-    name: sys-front-inj-logout
-    public: true # Vars used in templates
-  when: inj_enabled.logout
-
-- block:
-  - name: Include dependency 'sys-svc-webserver'
-    include_role:
-      name: sys-svc-webserver
-    when: run_once_sys_svc_webserver is not defined
-  - include_tasks: utils/run_once.yml
-  when: run_once_sys_front_inj_all is not defined

roles/sys-front-inj-all/templates/location.lua.j2

@@ -10,17 +10,6 @@
 
 lua_need_request_body on;
 
-header_filter_by_lua_block {
-    local ct = ngx.header.content_type or ""
-    if ct:lower():find("^text/html") then
-        ngx.ctx.is_html = true
-        -- IMPORTANT: body will be modified → drop Content-Length to avoid mismatches
-        ngx.header.content_length = nil
-    else
-        ngx.ctx.is_html = false
-    end
-}
-
 body_filter_by_lua_block {
     -- Only process HTML responses
     if not ngx.ctx.is_html then

roles/sys-front-inj-css/tasks/01_core.yml

@@ -1,8 +1,3 @@
-- name: Include dependency 'sys-svc-webserver'
-  include_role:
-    name: sys-svc-webserver
-  when: run_once_sys_svc_webserver is not defined
-
 - name: Generate color palette with colorscheme-generator
   set_fact:
     color_palette: "{{ lookup('colorscheme', CSS_BASE_COLOR, count=CSS_COUNT, shades=CSS_SHADES) }}"
@@ -19,3 +14,5 @@
     group:  "{{ NGINX.USER }}"
     mode:   '0644'
   loop: "{{ CSS_FILES }}"
+
+- include_tasks: utils/run_once.yml

roles/sys-front-inj-css/tasks/main.yml

@@ -1,6 +1,4 @@
-- block:
-    - include_tasks: 01_core.yml
-    - include_tasks: utils/run_once.yml
+- include_tasks: 01_core.yml
   when: run_once_sys_front_inj_css is not defined
 
 - name: "Resolve optional app style.css source for '{{ application_id }}'"

roles/sys-front-inj-css/templates/head_sub.j2

@@ -3,6 +3,6 @@
 {% for css_file in ['default.css','bootstrap.css'] %}
 <link rel="stylesheet" href="{{ [ cdn_urls.shared.css, css_file, lookup('local_mtime_qs', [__css_tpl_dir, css_file ~ '.j2'] | path_join)] | url_join }}">
 {% endfor %}
-{% if app_style_present | bool %}
+{% if app_style_present | default(false) | bool %}
 <link rel="stylesheet" href="{{ [ cdn_urls.role.release.css, 'style.css', lookup('local_mtime_qs', app_style_src)] | url_join }}">
 {% endif %}

roles/sys-front-inj-desktop/tasks/main.yml

@@ -1,8 +1,4 @@
 - block:
-  - name: Include dependency 'sys-svc-webserver'
-    include_role:
-      name: sys-svc-webserver
-    when: run_once_sys_svc_webserver is not defined
   - include_tasks: 01_deploy.yml
   - include_tasks: utils/run_once.yml
   when: run_once_sys_front_inj_desktop is not defined

roles/sys-front-inj-javascript/tasks/main.yml

@@ -1,11 +1,4 @@
-- block:
-
-  - name: Include dependency 'sys-svc-webserver'
-    include_role:
-      name: sys-svc-webserver
-    when: run_once_sys_svc_webserver is not defined
-  - include_tasks: utils/run_once.yml
-  when: run_once_sys_front_inj_javascript is not defined
+# run_once_sys_front_inj_javascript: deactivated
 
 - name: "Load JavaScript code for '{{ application_id }}'"
   set_fact:

roles/sys-front-inj-logout/tasks/01_core.yml

@@ -1,8 +1,6 @@
-- name: Include dependency 'sys-svc-webserver'
-  include_role:
-    name: sys-svc-webserver
-  when: 
-    - run_once_sys_svc_webserver is not defined
-  
 - name: "deploy the logout.js"
-  include_tasks: "02_deploy.yml"
+  include_tasks: "02_deploy.yml"
+
+- set_fact:
+    run_once_sys_front_inj_logout: true
+  changed_when: false

roles/sys-front-inj-logout/tasks/02_deploy.yml

@@ -1,10 +1,10 @@
 - name: Deploy logout.js
-  template:
-    src: logout.js.j2
-    dest: "{{ INJ_LOGOUT_JS_DESTINATION }}"
-    owner: "{{ NGINX.USER }}"
-    group: "{{ NGINX.USER }}"
-    mode: '0644'
+  copy:
+    src:    logout.js
+    dest:   "{{ INJ_LOGOUT_JS_DESTINATION }}"
+    owner:  "{{ NGINX.USER }}"
+    group:  "{{ NGINX.USER }}"
+    mode:   '0644'
 
 - name: Get stat for logout.js
   stat:

roles/sys-front-inj-logout/tasks/main.yml

@@ -1,16 +1,16 @@
-- block:
-  - include_tasks: 01_core.yml
-  - set_fact:
-      run_once_sys_front_inj_logout: true
+- name: "Load base for '{{ application_id }}'"
+  include_tasks: 01_core.yml
   when: run_once_sys_front_inj_logout is not defined
 
 - name: "Load logout code for '{{ application_id }}'"
   set_fact:
     logout_code: "{{ lookup('template', 'logout_one_liner.js.j2') }}"
+  changed_when: false
 
 - name: "Collapse logout code into one-liner for '{{ application_id }}'"
   set_fact:
     logout_code_one_liner: "{{ logout_code | to_one_liner }}"
+  changed_when: false
 
 - name: "Append logout CSP hash for '{{ application_id }}'"
   set_fact:

roles/sys-front-inj-logout/templates/head_sub.j2

@@ -1 +1 @@
-<script src="{{ cdn_urls.shared.js }}/{{ INJ_LOGOUT_JS_FILE_NAME }}{{ lookup('local_mtime_qs', [playbook_dir, 'roles', 'sys-front-inj-logout', 'templates', INJ_LOGOUT_JS_FILE_NAME ~ '.j2'] | path_join) }}"></script>
+<script src="{{ cdn_urls.shared.js }}/{{ INJ_LOGOUT_JS_FILE_NAME }}{{ lookup('local_mtime_qs', [playbook_dir, 'roles', 'sys-front-inj-logout', 'files', INJ_LOGOUT_JS_FILE_NAME] | path_join) }}"></script>

roles/sys-front-inj-matomo/tasks/main.yml

@@ -1,10 +1,4 @@
-- block:
-  - name: Include dependency 'sys-svc-webserver'
-    include_role:
-      name: sys-svc-webserver
-    when: run_once_sys_svc_webserver is not defined
-  - include_tasks: utils/run_once.yml
-  when: run_once_sys_front_inj_matomo is not defined
+# run_once_sys_front_inj_matomo: deactivated
 
 - name: "Relevant variables for role: {{ role_path | basename }}"
   debug:

roles/sys-service/handlers/main.yml

@@ -1,19 +1,19 @@
 - name: "Enable systemctl service"
   systemd:
-    name: "{{ system_service_id | get_service_name(SOFTWARE_NAME) }}"
+    name: "{{ system_service_name }}"
     enabled: yes
     daemon_reload: true
   become: true
-  async:  "{{ ASYNC_TIME if ASYNC_ENABLED | bool else omit }}"
-  poll:   "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}"
+  async:  "{{ system_service_async }}"
+  poll:   "{{ system_service_poll }}"
   listen: refresh systemctl service
 
 - name: "Set systemctl service state"
   systemd:
-    name: "{{ system_service_id | get_service_name(SOFTWARE_NAME) }}"
+    name: "{{ system_service_name }}"
     state: "{{ system_service_state }}"
   become: true
-  async:  "{{ ASYNC_TIME if ASYNC_ENABLED | bool else omit }}"
-  poll:   "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}"
+  async:  "{{ system_service_async }}"
+  poll:   "{{ system_service_poll }}"
   when:   not (system_service_suppress_flush | bool)
   listen: refresh systemctl service

roles/sys-service/tasks/05_service.yml

@@ -31,7 +31,7 @@
 - name: "setup systemctl '{{ system_service_id }}'"
   template:
     src:  "{{ system_service_template_src }}"
-    dest: "{{ [ PATH_SYSTEM_SERVICE_DIR, system_service_id | get_service_name(SOFTWARE_NAME) ] | path_join }}"
+    dest: "{{ [ PATH_SYSTEM_SERVICE_DIR, system_service_name ] | path_join }}"
     owner: root
     group: root
     mode: '0644'
@@ -46,5 +46,5 @@
     command: /bin/true
     notify: refresh systemctl service
     when: not system_service_uses_at
-  when: system_force_flush | bool
+  when: system_service_force_flush | bool
 

roles/sys-service/vars/main.yml

@@ -1,22 +1,28 @@
-UNIT_SUFFIX_REMOVER_PACKAGE:  "unsure"
+UNIT_SUFFIX_REMOVER_PACKAGE:          "unsure"
+system_service_name:                  "{{ system_service_id | get_service_name(SOFTWARE_NAME) }}"
 
 ## Paths
-system_service_role_name:      "{{ system_service_id | regex_replace('@','') }}"
-system_service_role_dir:       "{{ [ playbook_dir, 'roles', system_service_role_name ] | path_join }}"
-system_service_script_dir:     "{{ [ PATH_SYSTEMCTL_SCRIPTS, system_service_id ] | path_join }}"
+system_service_role_name:             "{{ system_service_id | regex_replace('@','') }}"
+system_service_role_dir:              "{{ [ playbook_dir, 'roles', system_service_role_name ] | path_join }}"
+system_service_script_dir:            "{{ [ PATH_SYSTEMCTL_SCRIPTS, system_service_id ] | path_join }}"
 
 ## Settings
-system_force_flush:            "{{ SYS_SERVICE_ALL_ENABLED | bool }}"                           # When set to true it activates the flushing of services. defaults to SYS_SERVICE_ALL_ENABLED
-system_service_suppress_flush: "{{ (system_service_id in SYS_SERVICE_SUPPRESS_FLUSH) | bool }}" # When set to true it suppresses the flushing of services
-system_service_copy_files:     true                                                             # When set to false file copying will be skipped
-system_service_timer_enabled:  false                                                            # When set to true timer will be loaded
-system_service_state:          "{{ SYS_SERVICE_DEFAULT_STATE }}"
+system_service_force_linear_sync:     "{{ system_service_name in SYS_SERVICE_GROUP_MANIPULATION }}"    # Disables automatic async
+system_service_force_flush:           "{{ SYS_SERVICE_ALL_ENABLED | bool }}"                           # When set to true it activates the flushing of services. defaults to SYS_SERVICE_ALL_ENABLED
+system_service_suppress_flush:        "{{ (system_service_id in SYS_SERVICE_SUPPRESS_FLUSH) | bool }}" # When set to true it suppresses the flushing of services
+system_service_copy_files:            true                                                             # When set to false file copying will be skipped
+system_service_timer_enabled:         false                                                            # When set to true timer will be loaded
+system_service_state:                 "{{ SYS_SERVICE_DEFAULT_STATE }}"
+
+## ASYNC Settings
+system_service_async:                 "{{ omit if (system_service_force_linear_sync | bool or not ASYNC_ENABLED | bool) else ASYNC_TIME }}"
+system_service_poll:                  "{{ omit if (system_service_force_linear_sync | bool or not ASYNC_ENABLED | bool) else ASYNC_POLL }}"
 
 # Dynamic Loaded ( Just available when dependencies are loaded )
-system_service_script_base:   "{{ system_service_script_src | basename | regex_replace('\\.j2$', '') }}"
-system_service_script_type:   "{{ system_service_script_base | filetype }}"
-system_service_script_inter:  "/bin/{{ 'bash' if system_service_script_type == 'sh' else 'python3'}}"
-system_service_script_exec:   "{{ system_service_script_inter }} {{ system_service_id | get_service_script_path( system_service_script_type ) }}"
+system_service_script_base:           "{{ system_service_script_src | basename | regex_replace('\\.j2$', '') }}"
+system_service_script_type:           "{{ system_service_script_base | filetype }}"
+system_service_script_inter:          "/bin/{{ 'bash' if system_service_script_type == 'sh' else 'python3'}}"
+system_service_script_exec:           "{{ system_service_script_inter }} {{ system_service_id | get_service_script_path( system_service_script_type ) }}"
 
 # Service template
 system_service_tpl_on_failure:        "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"

roles/sys-stk-front-base/README.md

@@ -0,0 +1,21 @@
+# Front Base (HTTPS + Cloudflare + Handlers) 🚀
+
+## Description
+**sys-stk-front-base** bootstraps the front layer that most web-facing apps need:
+- Ensures the HTTPS base via `sys-svc-webserver-https`
+- (Optional) Cloudflare bootstrap (zone lookup, dev mode, purge)
+- Wires OpenResty/Nginx handlers
+- Leaves per-domain certificate issuance to consumer roles (or pass-through vars to `sys-util-csp-cert` if needed)
+
+> This role is intentionally small and reusable. It prepares the ground so app roles can just render their vHost.
+
+## Responsibilities
+- Include `sys-svc-webserver-https` (once per host)
+- Include Cloudflare tasks when `DNS_PROVIDER == "cloudflare"`
+- Load handler utilities (e.g., `svc-prx-openresty`)
+- Stay domain-agnostic: expect `domain` to be provided by the consumer
+
+## Outputs
+- Handler wiring completed
+- HTTPS base ready (Nginx, ACME webroot)
+- Cloudflare prepared (optional)

roles/sys-stk-front-base/meta/main.yml

@@ -1,6 +1,6 @@
 galaxy_info:
   author: "Kevin Veen-Birkenbach"
-  description: "Updates AUR packages on Arch Linux systems using yay. This role automates the upgrade process for AUR packages, ensuring that the system remains up-to-date with the latest versions available in the Arch User Repository."
+  description: "Front bootstrap for web apps: HTTPS base, optional Cloudflare setup, and handler wiring."
   license: "Infinito.Nexus NonCommercial License"
   license_url: "https://s.infinito.nexus/license"
   company: |
@@ -9,16 +9,16 @@ galaxy_info:
     https://www.veen.world
   min_ansible_version: "2.9"
   platforms:
-  - name: Archlinux
-    versions:
-    - rolling
+    - name: Archlinux
+      versions:
+        - rolling
   galaxy_tags:
-  - aur
-  - update
-  - archlinux
-  - yay
-  - system
-  - maintenance
+    - nginx
+    - https
+    - cloudflare
+    - automation
+    - web
   repository: "https://s.infinito.nexus/code"
   issue_tracker_url: "https://s.infinito.nexus/issues"
-  documentation: "https://docs.infinito.nexus"
+  documentation: "https://docs.infinito.nexus/"
+dependencies: []

roles/sys-stk-front-base/tasks/main.yml

@@ -0,0 +1,14 @@
+- block:
+  - name: Include dependency 'sys-svc-webserver-https'
+    include_role:
+      name: sys-svc-webserver-https
+    when: run_once_sys_svc_webserver_https is not defined
+  - include_tasks: utils/run_once.yml
+  when: run_once_sys_stk_front_base is not defined
+
+- include_tasks: "01_cloudflare.yml"
+  when: DNS_PROVIDER == "cloudflare"
+
+- include_tasks: "{{ [ playbook_dir, 'tasks/utils/load_handlers.yml' ] | path_join }}"
+  vars:
+    handler_role_name: "svc-prx-openresty"

roles/sys-stk-front-proxy/defaults/main.yml

@@ -1,8 +1,5 @@
 # default vhost flavour
-vhost_flavour:        "basic"               # valid: basic, ws_generic
-
-# build the full template path from the flavour
-vhost_template_src:   "roles/sys-svc-proxy/templates/vhost/{{ vhost_flavour }}.conf.j2"
+vhost_flavour:              "basic" # valid: basic, ws_generic
 
 # Enable / Disable Proxy during development, for faster Debugging
 SYS_STK_FRONT_PROXY_ENABLED: true

roles/sys-stk-front-proxy/tasks/01_base.yml

@@ -1,26 +1,15 @@
-- block:
-  - name: Include dependency 'sys-svc-proxy'
-    include_role:
-      name: sys-svc-proxy
-    when: run_once_sys_svc_proxy is not defined
-  - include_tasks: utils/run_once.yml
-  when: run_once_sys_stk_front_proxy is not defined
+- name: Front bootstrap
+  include_role:
+    name: sys-stk-front-base
 
-- include_tasks: "02_cloudflare.yml"
-  when: DNS_PROVIDER == "cloudflare"
-
-- include_tasks: "{{ [ playbook_dir, 'tasks/utils/load_handlers.yml' ] | path_join }}"
-  vars:
-    handler_role_name: "svc-prx-openresty"
-
-- name: "include role for {{ domain }} to receive certificates and do the modification routines"
+- name: "include role for '{{ domain }}' to receive certificates and do the modification routines"
   include_role:
     name: sys-util-csp-cert
 
-- name: "Copy nginx config to {{ configuration_destination }}"
+- name: "Copy nginx config to '{{ front_proxy_domain_conf_dst }}'"
   template:
-    src: "{{ vhost_template_src }}"
-    dest: "{{ configuration_destination }}"
+    src:  "{{ front_proxy_domain_conf_src }}"
+    dest: "{{ front_proxy_domain_conf_dst }}"
   register: nginx_conf
   notify: restart openresty
 
@@ -39,4 +28,7 @@
     when:
     - site_check.status is defined
     - not site_check.status in [200,301,302]
-  when: not nginx_conf.changed
+  when: not nginx_conf.changed
+
+- name: "Restart Webserver for '{{ front_proxy_domain_conf_dst }}'"
+  meta: flush_handlers

roles/sys-stk-front-proxy/vars/main.yml

@@ -1 +1,2 @@
-configuration_destination:  "{{ [ NGINX.DIRECTORIES.HTTP.SERVERS, domain ~ '.conf'] | path_join }}"
+front_proxy_domain_conf_dst:  "{{ [ NGINX.DIRECTORIES.HTTP.SERVERS, domain ~ '.conf'] | path_join }}"
+front_proxy_domain_conf_src:  "roles/sys-svc-proxy/templates/vhost/{{ vhost_flavour }}.conf.j2"

roles/sys-stk-semi-stateless/README.md

@@ -0,0 +1,13 @@
+# Semi-Stateless Stack (Front + Back) ⚡
+
+## Description
+**sys-stk-semi-stateless** combines the front and back layer into a lightweight, mostly stateless web service stack:
+- Front bootstrap via `sys-stk-front-base` (HTTPS base, optional Cloudflare, handlers)
+- Backend via `sys-stk-back-stateless` (no persistent volumes/DB)
+
+Ideal for services that need TLS/front glue but no database (e.g., TURN/STUN, gateways, simple APIs).
+
+## Responsibilities
+- Prepare the front layer (HTTPS / handlers / optional Cloudflare)
+- Deploy the stateless backend (typically via Docker Compose)
+- Keep domain variables (`domain`) and app-scoped variables (`application_id`) clearly separated

roles/sys-stk-semi-stateless/meta/main.yml

@@ -0,0 +1,24 @@
+galaxy_info:
+  author: "Kevin Veen-Birkenbach"
+  description: "Combined semi-stateless app stack: front bootstrap + stateless backend."
+  license: "Infinito.Nexus NonCommercial License"
+  license_url: "https://s.infinito.nexus/license"
+  company: |
+    Kevin Veen-Birkenbach
+    Consulting & Coaching Solutions
+    https://www.veen.world
+  min_ansible_version: "2.9"
+  platforms:
+    - name: Archlinux
+      versions:
+        - rolling
+  galaxy_tags:
+    - nginx
+    - https
+    - stateless
+    - backend
+    - cloudflare
+    - automation
+  repository: "https://s.infinito.nexus/code"
+  issue_tracker_url: "https://s.infinito.nexus/issues"
+  documentation: "https://docs.infinito.nexus/"

roles/sys-stk-semi-stateless/tasks/main.yml

@@ -0,0 +1,11 @@
+# run_once_sys_stk_semi_stateless: deactivated
+
+- name: "sys-stk-front-base"
+  include_role:
+    name: sys-stk-front-base
+  vars:
+    domain:   "{{ domains | get_domain(application_id) }}"
+
+- name: "For '{{ application_id }}': Load sys-stk-back-stateless"
+  include_role:
+    name: sys-stk-back-stateless

roles/sys-svc-cdn/tasks/01_core.yml

@@ -1,21 +0,0 @@
-- name: "Load CDN for '{{ domain }}'"
-  include_role:
-    name: web-svc-cdn
-    public: false
-  when:
-  - application_id != 'web-svc-cdn'
-  - run_once_web_svc_cdn is not defined
-
-# ------------------------------------------------------------------
-# Only-once creations (shared root and vendor)
-# ------------------------------------------------------------------
-- name: Ensure shared root and vendor exist (run once)
-  ansible.builtin.file:
-    path:  "{{ item }}"
-    state: directory
-    owner: "{{ NGINX.USER }}"
-    group: "{{ NGINX.USER }}"
-    mode:  "0755"
-  loop: "{{ CDN_DIRS_GLOBAL }}"
-
-- include_tasks: utils/run_once.yml

roles/sys-svc-cdn/tasks/main.yml

@@ -1,6 +1,14 @@
 ---
 - block:
-    - include_tasks: 01_core.yml
+    - name: Ensure shared root and vendor exist (run once)
+      ansible.builtin.file:
+        path:  "{{ item }}"
+        state: directory
+        owner: "{{ NGINX.USER }}"
+        group: "{{ NGINX.USER }}"
+        mode:  "0755"
+      loop: "{{ CDN_DIRS_GLOBAL }}"
+    - include_tasks: utils/run_once.yml
   when:
     - run_once_sys_svc_cdn is not defined