Merge pull request #97 from kevinveenbirkenbach/feature/general-optimation

Feature/general optimation

.github/pull_request_template.md

@@ -0,0 +1,94 @@
+## Summary
+
+Briefly describe the purpose of this pull request.
+
+Example:
+
+* Add new role for application X
+* Fix configuration issue in role Y
+* Improve container startup logic
+
+---
+
+## Change Type
+
+Select the semantic version impact of this change:
+
+* [ ] **Major** – Breaking change (incompatible configuration or behaviour)
+* [ ] **Minor** – New feature or new role (backwards compatible)
+* [ ] **Patch** – Bugfix or internal improvement
+
+---
+
+## Impacted Areas
+
+Select affected components.
+
+* [ ] Role implementation
+* [ ] Configuration variables
+* [ ] Documentation
+* [ ] CI/CD
+* [ ] Security
+
+---
+
+## Description of Changes
+
+Explain what was changed and why.
+
+Key points:
+
+* What problem does this solve?
+* Why was this implementation chosen?
+* Are there alternative approaches?
+
+---
+
+## Definition of Done (DoD)
+
+Please ensure all items are completed before requesting review.
+
+### Workflow
+
+* [ ] Contributions follow the [collaboration workflow](https://hub.infinito.nexus/t/working-with-folks-in-infinito-nexus/436).
+
+### Code Quality
+
+* [ ] Code follows repository conventions
+* [ ] Code and comments are written in English
+
+### Functionality
+
+* [ ] Feature works as expected
+
+### Testing
+
+* [ ] Changes tested locally
+* [ ] CI pipeline passes
+* [ ] E2E test introduced for web-* roles
+* [ ] Unit test integrated for *.py files
+* [ ] Integration test integrated for *.py files
+
+### Documentation
+
+* [ ] README.md's and other *.md updated
+* [ ] Link from this request to the https://project.infinito.nexus/ workitem and backwards
+* [ ] Workitem in https://project.infinito.nexus/ updated
+* [ ] Attach a **screenshot of the running application** from the local deployment.([📕local environment setup guide](https://hub.infinito.nexus/t/local-test-environment-deploy/435))
+
+---
+
+## Checklist for Reviewer
+
+Reviewer should verify:
+
+* [ ] Code is understandable
+* [ ] Changes follow project guidelines
+* [ ] Documentation is adequate
+* [ ] No obvious security issues
+
+---
+
+## Additional Notes
+
+Add anything reviewers should know.

cli/deploy/development/compose.py

@@ -39,7 +39,6 @@ class Compose:
         check: bool = True,
         capture: bool = False,
         live: bool = False,
-        keep_lines: int = 400,
         text: bool = True,
         extra_env: dict[str, str] | None = None,
     ) -> subprocess.CompletedProcess:
@@ -49,9 +48,7 @@ class Compose:
             env.update({k: str(v) for k, v in extra_env.items()})
 
         if live:
-            r = run_streaming(
-                cmd, cwd=self.repo_root, env=env, keep_lines=keep_lines, text=text
-            )
+            r = run_streaming(cmd, cwd=self.repo_root, env=env, text=text)
         else:
             r = subprocess.run(
                 cmd,
@@ -158,8 +155,7 @@ class Compose:
                 ["sh", "-lc", "/opt/src/infinito/scripts/docker/entry.sh true"],
                 workdir="/opt/src/infinito",
                 check=False,
-                live=True,  # <--- live output
-                keep_lines=400,  # <--- tail for failure printing
+                live=True,
                 extra_env={
                     "ANSIBLE_FORCE_COLOR": "1",
                     "PY_COLORS": "1",
@@ -168,10 +164,6 @@ class Compose:
             )
 
             if r.returncode != 0:
-                print("===== entry.sh stdout (tail) =====")
-                print((r.stdout or "").rstrip() or "<empty>")
-                print("===== entry.sh stderr (tail) =====")
-                print((r.stderr or "").rstrip() or "<empty>")
                 raise RuntimeError(f"entry.sh init failed (rc={r.returncode})")
 
     def down(self) -> None:
@@ -193,7 +185,6 @@ class Compose:
         workdir: str | None = None,
         capture: bool = False,
         live: bool = False,
-        keep_lines: int = 400,
         tty: bool | None = None,
         extra_env: dict[str, str] | None = None,
     ) -> subprocess.CompletedProcess:
@@ -206,7 +197,7 @@ class Compose:
           - tty=False -> -T (CI safe)
 
         Streaming:
-          - live=True streams stdout/stderr to terminal while keeping a tail buffer.
+          - live=True streams stdout/stderr to terminal.
         """
         if tty is None:
             tty = not self._is_ci()
@@ -229,7 +220,6 @@ class Compose:
             check=check,
             capture=capture,
             live=live,
-            keep_lines=keep_lines,
         )
 
     def _get_infinito_container_id(self) -> str:

cli/deploy/development/deploy.py

@@ -38,13 +38,11 @@ def _run_deploy(
     if passthrough:
         cmd.extend(passthrough)
 
-    # Live stream output for immediate visibility,
-    # while keeping a tail buffer for post-failure printing.
+    # Live stream output for immediate visibility.
     r = compose.exec(
         cmd,
         check=False,
         live=True,
-        keep_lines=400,
         extra_env={
             # Force ANSI colors even when no TTY is allocated (CI default).
             "ANSIBLE_FORCE_COLOR": "1",
@@ -53,13 +51,6 @@ def _run_deploy(
         },
     )
 
-    if r.returncode != 0:
-        print("===== deploy stdout (tail) =====")
-        print((r.stdout or "").rstrip() or "<empty>")
-        print("===== deploy stderr (tail) =====")
-        print((r.stderr or "").rstrip() or "<empty>")
-        print("===============================")
-
     return int(r.returncode)
 
 

cli/deploy/development/proc.py

@@ -3,24 +3,21 @@ from __future__ import annotations
 import sys
 import subprocess
 import threading
-from collections import deque
-from typing import Deque, TextIO
+from typing import TextIO
 
 
 def _drain_stream(
     stream: TextIO,
     *,
     sink: TextIO,
-    buf: Deque[str],
 ) -> None:
     """
-    Read a text stream line-by-line, write to sink, and keep a tail buffer.
+    Read a text stream line-by-line and write to sink.
     """
     try:
         for line in iter(stream.readline, ""):
             sink.write(line)
             sink.flush()
-            buf.append(line.rstrip("\n"))
     finally:
         try:
             stream.close()
@@ -34,13 +31,10 @@ def run_streaming(
     *,
     cwd,
     env: dict[str, str],
-    keep_lines: int = 400,
     text: bool = True,
 ) -> subprocess.CompletedProcess:
     """
-    Run a subprocess, stream stdout/stderr live to the terminal, and return
-    a CompletedProcess whose stdout/stderr contain only the last `keep_lines`
-    lines (tail buffers).
+    Run a subprocess and stream stdout/stderr live to the terminal.
     """
     p = subprocess.Popen(
         cmd,
@@ -52,19 +46,16 @@ def run_streaming(
         bufsize=1,  # line buffered (best effort)
     )
 
-    out_buf: Deque[str] = deque(maxlen=int(keep_lines))
-    err_buf: Deque[str] = deque(maxlen=int(keep_lines))
-
     assert p.stdout is not None
     assert p.stderr is not None
 
     t_out = threading.Thread(
         target=_drain_stream,
-        kwargs={"stream": p.stdout, "sink": sys.stdout, "buf": out_buf},
+        kwargs={"stream": p.stdout, "sink": sys.stdout},
     )
     t_err = threading.Thread(
         target=_drain_stream,
-        kwargs={"stream": p.stderr, "sink": sys.stderr, "buf": err_buf},
+        kwargs={"stream": p.stderr, "sink": sys.stderr},
     )
 
     t_out.daemon = True
@@ -78,7 +69,4 @@ def run_streaming(
     t_out.join()
     t_err.join()
 
-    stdout_tail = ("\n".join(out_buf) + ("\n" if out_buf else "")).rstrip("\n")
-    stderr_tail = ("\n".join(err_buf) + ("\n" if err_buf else "")).rstrip("\n")
-
-    return subprocess.CompletedProcess(cmd, rc, stdout=stdout_tail, stderr=stderr_tail)
+    return subprocess.CompletedProcess(cmd, rc)

filter_plugins/csp_filters.py

@@ -265,8 +265,7 @@ class FilterModule(object):
                     ):
                         # Allow being embedded by the dashboard app domain's site
                         domain = domains.get("web-app-dashboard")[0]
-                        sld_tld = ".".join(domain.split(".")[-2:])  # e.g., example.com
-                        tokens.append(f"{sld_tld}")
+                        tokens.append(f"{domain}")
                     if self.is_feature_enabled(applications, "logout", application_id):
                         tokens.append(get_url(domains, "web-svc-logout", web_protocol))
                         tokens.append(

filter_plugins/ruby.py

@@ -24,7 +24,7 @@ class FilterModule:
 
         Ruby rules handled:
         - "  -> \"
-        - \  -> \\
+        - \\  -> \\\\
         - newlines -> \n
         - carriage return -> \r
         - tab -> \t

lookup_plugins/unit_name.py

@@ -32,7 +32,7 @@ def _render_if_template(templar, value, var_name: str):
         return value
 
     try:
-        rendered = templar.template(value, disable_lookups=False)
+        rendered = templar.template(value)
     except TypeError:
         # Unit tests can inject a minimal templar without Ansible's full signature.
         rendered = templar.template(value)
@@ -51,7 +51,7 @@ def _render_if_template(templar, value, var_name: str):
 def _resolve_version(templar, variables):
     # Try the canonical lookup call first.
     try:
-        version = templar.template("{{ lookup('version') }}", disable_lookups=False)
+        version = templar.template("{{ lookup('version') }}")
     except TypeError:
         # Unit tests can inject a minimal templar without Ansible's full signature.
         version = templar.template("{{ lookup('version') }}")

roles/sys-stk-backend/lookup_plugins/service_should_load.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from ansible.errors import AnsibleError
+from ansible.plugins.lookup import LookupBase
+
+from module_utils.config_utils import get_app_conf
+
+
+def _require_non_empty(value: Any, name: str) -> str:
+    s = str(value).strip()
+    if not s:
+        raise AnsibleError(f"service_should_load: '{name}' must not be empty")
+    return s
+
+
+def _service_enabled_and_shared(
+    applications: Dict[str, Any],
+    application_id: str,
+    service_name: str,
+    default: bool,
+) -> bool:
+    enabled = get_app_conf(
+        applications=applications,
+        application_id=application_id,
+        config_path=f"compose.services.{service_name}.enabled",
+        strict=False,
+        default=default,
+        skip_missing_app=False,
+    )
+    shared = get_app_conf(
+        applications=applications,
+        application_id=application_id,
+        config_path=f"compose.services.{service_name}.shared",
+        strict=False,
+        default=default,
+        skip_missing_app=False,
+    )
+    return bool(enabled) and bool(shared)
+
+
+def _run_once_var_name(service_id: str) -> str:
+    return f"run_once_{service_id.replace('-', '_')}"
+
+
+class LookupModule(LookupBase):
+    """
+    Evaluate whether a backend service should be loaded.
+
+    Usage:
+      {{ query('service_should_load', service_id, application_id=application_id, service_name=service_name) | first }}
+    """
+
+    def run(
+        self,
+        terms: List[Any],
+        variables: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> List[bool]:
+        if not terms:
+            return []
+
+        vars_ = variables or getattr(self._templar, "available_variables", {}) or {}
+        applications = kwargs.get("applications", vars_.get("applications"))
+        if not isinstance(applications, dict):
+            raise AnsibleError(
+                "service_should_load: required variable 'applications' must be a mapping"
+            )
+
+        application_id = _require_non_empty(
+            kwargs.get("application_id", vars_.get("application_id")), "application_id"
+        )
+        service_name = _require_non_empty(
+            kwargs.get("service_name", vars_.get("service_name")), "service_name"
+        )
+        default = bool(kwargs.get("default", False))
+
+        results: List[bool] = []
+        for term in terms:
+            service_id = _require_non_empty(term, "service_id")
+            run_once_defined = _run_once_var_name(service_id) in vars_
+            should_load = (
+                _service_enabled_and_shared(
+                    applications=applications,
+                    application_id=application_id,
+                    service_name=service_name,
+                    default=default,
+                )
+                and (not run_once_defined)
+                and (application_id != service_id)
+            )
+            results.append(bool(should_load))
+
+        return results

roles/sys-stk-backend/tasks/_load_service.yml

@@ -2,13 +2,14 @@
   ansible.builtin.set_fact:
     load_service: >-
       {{
-        (applications | service_load_enabled(application_id, service_name, False))
-        and
-        (
-          ('run_once_' ~ (service_id | replace('-', '_'))) not in vars
-        )
-        and
-        (application_id != service_id)
+        query(
+          'service_should_load',
+          service_id,
+          application_id=application_id,
+          service_name=service_name,
+          applications=applications,
+          default=False
+        ) | first
       }}
 
 - name: "SYS-STK-BACKEND | Load '{{ service_id }}' (Once)"

roles/web-app-matomo/tasks/03b_bootstrap_debug.yml

@@ -96,6 +96,73 @@
       - "--- compose logs {{ MATOMO_BOOTSTRAP_SERVICE }} stderr ---"
       - "{{ matomo_bootstrap_service_logs.stderr | default('') }}"
 
+- name: "DEBUG | matomo container health (inspect json)"
+  ansible.builtin.command:
+    argv:
+      - container
+      - inspect
+      - --format
+      - '{{ "{{" }}json .State.Health{{ "}}" }}'
+      - "{{ MATOMO_NAME }}"
+  register: matomo_container_health
+  changed_when: false
+  failed_when: false
+
+- name: "DEBUG | matomo container health output"
+  ansible.builtin.debug:
+    msg:
+      - "--- matomo .State.Health (json) ---"
+      - "{{ matomo_container_health.stdout | default('') }}"
+      - "{{ matomo_container_health.stderr | default('') }}"
+
+- name: "DEBUG | matomo tmp permissions + write test (www-data)"
+  ansible.builtin.command:
+    argv:
+      - container
+      - exec
+      - --user
+      - root
+      - "{{ MATOMO_NAME }}"
+      - sh
+      - -lc
+      - |
+        set -eu
+        id
+        for p in /var/www/html /var/www/html/tmp /var/www/html/tmp/templates_c /var/www/html/config; do
+          echo "===== PATH: ${p} ====="
+          ls -ld "${p}" || true
+        done
+        echo "===== WRITE TEST AS www-data ====="
+        su -s /bin/sh www-data -c "mkdir -p /var/www/html/tmp/templates_c && touch /var/www/html/tmp/templates_c/.write-test && rm -f /var/www/html/tmp/templates_c/.write-test"
+  register: matomo_fs_permissions_check
+  changed_when: false
+  failed_when: false
+
+- name: "DEBUG | matomo tmp permissions + write test output"
+  ansible.builtin.debug:
+    msg:
+      - "--- filesystem + write-test diagnostics ---"
+      - "{{ matomo_fs_permissions_check.stdout | default('') }}"
+      - "{{ matomo_fs_permissions_check.stderr | default('') }}"
+
+- name: "DEBUG | bootstrap artifact paths from logs"
+  ansible.builtin.debug:
+    msg:
+      - "--- bootstrap installer artifacts (paths from logs) ---"
+      - >-
+        {{
+          (
+            (
+              matomo_bootstrap_service_logs.stdout | default('')
+            ) ~ '\n' ~ (
+              matomo_bootstrap_service_logs.stderr | default('')
+            )
+          )
+          | regex_findall('/tmp/matomo-bootstrap/installer-failure-[^\\s"]+\\.(?:png|html|txt)')
+          | unique
+          | list
+        }}
+
 - name: "DEBUG | show compose file"
   ansible.builtin.command:
     argv:

scripts/tests/deploy/ci/dedicated_distro.sh

@@ -158,8 +158,8 @@ echo ">>> PASS 2: re-init inventory (ASYNC_ENABLED=true)"
   --inventory-dir "${INVENTORY_DIR}" \
   --vars '{"ASYNC_ENABLED": true}'
 
-echo ">>> PASS 2: deploy"
-"${PYTHON}" -m cli.deploy.development deploy "${deploy_args[@]}"
+echo ">>> PASS 2: deploy (skip wrapper cleanup)"
+"${PYTHON}" -m cli.deploy.development deploy "${deploy_args[@]}" -- --skip-cleanup
 
 echo ">>> DISK / DOCKER STATE AFTER DEPLOY (before cleanup, distro=${INFINITO_DISTRO})"
 df -h || true

tasks/stages/01_constructor.yml

@@ -42,10 +42,12 @@
   - name: "Sanity check: all group_names must exist in applications"
     ansible.builtin.assert:
       that:
-        - item in applications
-      fail_msg: "Group '{{ item }}' has no entry in 'applications'"
-      success_msg: "Group '{{ item }}' is defined in 'applications'"
+        - group_name in applications
+      fail_msg: "Group '{{ group_name }}' has no entry in 'applications'"
+      success_msg: "Group '{{ group_name }}' is defined in 'applications'"
     loop: "{{ group_names }}"
+    loop_control:
+      loop_var: group_name
     when: MODE_ASSERT | bool
 
   - name: Merge current play applications
@@ -183,7 +185,7 @@
     - DOCKER_IN_CONTAINER | bool
 
 - name: "Load base roles"
-  ansible.builtin.include_tasks: "./tasks/groups/{{ item }}-roles.yml"
+  ansible.builtin.include_tasks: "./tasks/groups/{{ role_group }}-roles.yml"
   loop:
     - update      # 1. Update system
     - drv         # 2. Load driver roles
@@ -194,7 +196,8 @@
     - svc-ai      # 7. Load AI roles
     - svc-bkp     # 8. Load Backup Roles
   loop_control:
-    label: "{{ item }}-roles.yml"
+    loop_var: role_group
+    label: "{{ role_group }}-roles.yml"
 
 - name: Install inventory-defined system packages
   ansible.builtin.include_role:

tasks/stages/02_server.yml

@@ -1,7 +1,7 @@
 ---
 - name: Setup server base
   include_role:
-    name: "{{ item }}"
+    name: "{{ server_role }}"
   loop:
     - user-administrator
     - user-root
@@ -11,15 +11,18 @@
     - sys-ctl-cln-disc-space
     - sys-ctl-hlth-btrfs
     - sys-ctl-rpr-btrfs-balancer
+  loop_control:
+    loop_var: server_role
 
 - name: "Include server roles"
-  include_tasks: "./tasks/groups/{{ item }}-roles.yml"
+  include_tasks: "./tasks/groups/{{ server_group }}-roles.yml"
   loop:
     - web-svc   # 1. Load web server services
     - web-app   # 2. Load (UI) web applications
     - web-opt   # 4. Load optimation scrips last
   loop_control:
-    label: "{{ item }}-roles.yml"
+    loop_var: server_group
+    label: "{{ server_group }}-roles.yml"
 
 - name: Run E2E Cypress tests
   include_role:

tasks/stages/03_destructor.yml

@@ -1,9 +1,10 @@
 - name: "Load destruction roles"
-  include_tasks: "./tasks/groups/{{ item }}-roles.yml"
+  include_tasks: "./tasks/groups/{{ destruction_group }}-roles.yml"
   loop:
     - svc-opt # Load optimation services
   loop_control:
-    label: "{{ item }}-roles.yml"
+    loop_var: destruction_group
+    label: "{{ destruction_group }}-roles.yml"
 
 - name: "Finalize services"
   include_role:

tests/integration/test_meta_roles_integration.py

@@ -24,9 +24,9 @@ class TestMetaRolesIntegration(unittest.TestCase):
         for path in STAGES_DIR.rglob("*.yml"):
             cls.playbook_contents[path] = path.read_text(encoding="utf-8")
 
-        # Regex for include_tasks line with {{ item }}-roles.yml
+        # Regex for include_tasks line with {{ <loop_var> }}-roles.yml
         cls.include_pattern = re.compile(
-            r'include_tasks:\s*["\']\./tasks/groups/\{\{\s*item\s*\}\}-roles\.yml["\']'
+            r'include_tasks:\s*["\']\./tasks/groups/\{\{\s*[A-Za-z_][A-Za-z0-9_]*\s*\}\}-roles\.yml["\']'
         )
 
     def test_all_role_files_exist(self):

tests/unit/filter_plugins/test_csp_filters/test_toggles_desktop_logout.py

@@ -40,7 +40,9 @@ class TestCspTogglesDashboardLogout(unittest.TestCase):
         self._set_service_enabled(apps, "dashboard", True)
         header = self.filter.build_csp_header(apps, "app1", domains, "https")
         self.assertIn("frame-ancestors", header)
-        self.assertIn("domain-example.com", header)
+        frame_ancestors = self._get_directive_tokens(header, "frame-ancestors")
+        self.assertIn("domain-example.com", frame_ancestors)
+        self.assertNotIn("https://domain-example.com", frame_ancestors)
 
         self._set_service_enabled(apps, "dashboard", False)
         header_no = self.filter.build_csp_header(apps, "app1", domains, "https")

tests/unit/roles/sys-stk-backend/__init__.py

@@ -0,0 +1 @@
+

tests/unit/roles/sys-stk-backend/lookup_plugins/__init__.py

@@ -0,0 +1 @@
+

tests/unit/roles/sys-stk-backend/lookup_plugins/test_service_should_load.py

@@ -0,0 +1,106 @@
+import importlib.util
+import unittest
+from pathlib import Path
+
+from ansible.errors import AnsibleError
+
+
+def _repo_root() -> Path:
+    return Path(__file__).resolve().parents[5]
+
+
+def _load_module(rel_path: str, name: str):
+    path = _repo_root() / rel_path
+    spec = importlib.util.spec_from_file_location(name, str(path))
+    mod = importlib.util.module_from_spec(spec)
+    assert spec and spec.loader
+    spec.loader.exec_module(mod)
+    return mod
+
+
+class TestServiceShouldLoadLookup(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.mod = _load_module(
+            "roles/sys-stk-backend/lookup_plugins/service_should_load.py",
+            "service_should_load",
+        )
+
+    def setUp(self):
+        self.applications = {
+            "web-app-test": {
+                "compose": {
+                    "services": {
+                        "ldap": {"enabled": True, "shared": True},
+                        "ollama": {"enabled": True, "shared": False},
+                    }
+                }
+            }
+        }
+
+    def test_true_when_enabled_shared_not_run_once_and_not_self(self):
+        result = self.mod.LookupModule().run(
+            ["svc-db-openldap"],
+            variables={
+                "applications": self.applications,
+                "application_id": "web-app-test",
+                "service_name": "ldap",
+            },
+        )
+        self.assertEqual(result, [True])
+
+    def test_false_when_run_once_already_defined(self):
+        result = self.mod.LookupModule().run(
+            ["svc-db-openldap"],
+            variables={
+                "applications": self.applications,
+                "application_id": "web-app-test",
+                "service_name": "ldap",
+                "run_once_svc_db_openldap": True,
+            },
+        )
+        self.assertEqual(result, [False])
+
+    def test_false_when_not_shared(self):
+        result = self.mod.LookupModule().run(
+            ["svc-ai-ollama"],
+            variables={
+                "applications": self.applications,
+                "application_id": "web-app-test",
+                "service_name": "ollama",
+            },
+        )
+        self.assertEqual(result, [False])
+
+    def test_false_when_application_equals_service(self):
+        result = self.mod.LookupModule().run(
+            ["web-app-test"],
+            variables={
+                "applications": self.applications,
+                "application_id": "web-app-test",
+                "service_name": "ldap",
+            },
+        )
+        self.assertEqual(result, [False])
+
+    def test_raises_when_service_id_is_empty(self):
+        with self.assertRaises(AnsibleError):
+            self.mod.LookupModule().run(
+                ["   "],
+                variables={
+                    "applications": self.applications,
+                    "application_id": "web-app-test",
+                    "service_name": "ldap",
+                },
+            )
+
+    def test_raises_when_applications_missing(self):
+        with self.assertRaises(AnsibleError):
+            self.mod.LookupModule().run(
+                ["svc-db-openldap"],
+                variables={"application_id": "web-app-test", "service_name": "ldap"},
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()