From f0b323afeee501f8cdcb23d2b64ce00fbe26efcb Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 16 Aug 2025 01:31:49 +0200 Subject: [PATCH] Added auto snippet for webserver injection --- roles/docker-compose/handlers/main.yml | 2 +- .../srv-proxy-6-6-tls-deploy.service.j2 | 4 +- .../filter_plugins/inj_snippets.py | 56 +++++++++ roles/sys-srv-web-inj-compose/tasks/main.yml | 2 + .../templates/location.lua.j2 | 25 ++--- roles/web-app-gitea/Administration.md | 4 +- roles/web-app-listmonk/tasks/main.yml | 2 +- roles/web-app-mastodon/tasks/01_setup.yml | 2 +- .../tasks/02_administrator.yml | 14 ++- roles/web-app-mediawiki/tasks/main.yml | 2 +- roles/web-app-mybb/tasks/main.yml | 2 +- roles/web-app-mybb/vars/main.yml | 2 +- roles/web-app-roulette-wheel/vars/main.yml | 2 +- roles/web-app-taiga/vars/main.yml | 2 +- tasks/utils/update-repository-with-files.yml | 10 +- .../__init__.py | 0 .../filter_plugins/__init__.py | 0 .../filter_plugins/test_inj_enabled.py | 0 .../filter_plugins/test_inj_snippets.py | 106 ++++++++++++++++++ 19 files changed, 200 insertions(+), 37 deletions(-) create mode 100644 roles/sys-srv-web-inj-compose/filter_plugins/inj_snippets.py rename tests/unit/roles/{srv-web-inj-compose => sys-srv-web-inj-compose}/__init__.py (100%) rename tests/unit/roles/{srv-web-inj-compose => sys-srv-web-inj-compose}/filter_plugins/__init__.py (100%) rename tests/unit/roles/{srv-web-inj-compose => sys-srv-web-inj-compose}/filter_plugins/test_inj_enabled.py (100%) create mode 100644 tests/unit/roles/sys-srv-web-inj-compose/filter_plugins/test_inj_snippets.py diff --git a/roles/docker-compose/handlers/main.yml b/roles/docker-compose/handlers/main.yml index 5165c502..06a9dc0e 100644 --- a/roles/docker-compose/handlers/main.yml +++ b/roles/docker-compose/handlers/main.yml @@ -50,5 +50,5 @@ - name: docker compose restart command: cmd: 'docker compose restart' - chdir: "{{docker_compose.directories.instance}}" + chdir: "{{ docker_compose.directories.instance }}" listen: docker compose restart diff --git a/roles/srv-proxy-6-6-tls-deploy/templates/srv-proxy-6-6-tls-deploy.service.j2 b/roles/srv-proxy-6-6-tls-deploy/templates/srv-proxy-6-6-tls-deploy.service.j2 index 07e53750..3d9da477 100644 --- a/roles/srv-proxy-6-6-tls-deploy/templates/srv-proxy-6-6-tls-deploy.service.j2 +++ b/roles/srv-proxy-6-6-tls-deploy/templates/srv-proxy-6-6-tls-deploy.service.j2 @@ -1,7 +1,7 @@ [Unit] -Description=Let's Encrypt deploy to {{docker_compose.directories.instance}} +Description=Let's Encrypt deploy to {{ docker_compose.directories.instance }} OnFailure=sys-alm-compose.infinito@%n.service [Service] Type=oneshot -ExecStart=/usr/bin/bash {{ PATH_ADMINISTRATOR_SCRIPTS }}/srv-proxy-6-6-tls-deploy.sh {{ssl_cert_folder}} {{docker_compose.directories.instance}} +ExecStart=/usr/bin/bash {{ PATH_ADMINISTRATOR_SCRIPTS }}/srv-proxy-6-6-tls-deploy.sh {{ssl_cert_folder}} {{ docker_compose.directories.instance }} diff --git a/roles/sys-srv-web-inj-compose/filter_plugins/inj_snippets.py b/roles/sys-srv-web-inj-compose/filter_plugins/inj_snippets.py new file mode 100644 index 00000000..df3cac27 --- /dev/null +++ b/roles/sys-srv-web-inj-compose/filter_plugins/inj_snippets.py @@ -0,0 +1,56 @@ +# roles/sys-srv-web-inj-compose/filter_plugins/inj_snippets.py +""" +Jinja filter: `inj_features(kind)` filters a list of features to only those +that actually provide the corresponding snippet template file. + +- kind='head' -> roles/sys-srv-web-inj-/templates/head_sub.j2 +- kind='body' -> roles/sys-srv-web-inj-/templates/body_sub.j2 + +If the feature's role directory (roles/sys-srv-web-inj-) does not +exist, this filter raises FileNotFoundError. + +Usage in a template: + {% set head_features = SRV_WEB_INJ_COMP_FEATURES_ALL | inj_features('head') %} + {% set body_features = SRV_WEB_INJ_COMP_FEATURES_ALL | inj_features('body') %} +""" + +import os + +# This file lives at: roles/sys-srv-web-inj-compose/filter_plugins/inj_snippets.py +_THIS_DIR = os.path.dirname(__file__) +_ROLE_DIR = os.path.abspath(os.path.join(_THIS_DIR, "..")) # roles/sys-srv-web-inj-compose +_ROLES_DIR = os.path.abspath(os.path.join(_ROLE_DIR, "..")) # roles + +def _feature_role_dir(feature: str) -> str: + return os.path.join(_ROLES_DIR, f"sys-srv-web-inj-{feature}") + +def _has_snippet(feature: str, kind: str) -> bool: + if kind not in ("head", "body"): + raise ValueError("kind must be 'head' or 'body'") + + role_dir = _feature_role_dir(feature) + if not os.path.isdir(role_dir): + raise FileNotFoundError( + f"[inj_snippets] Expected role directory not found for feature " + f"'{feature}': {role_dir}" + ) + + path = os.path.join(role_dir, "templates", f"{kind}_sub.j2") + return os.path.exists(path) + +def inj_features_filter(features, kind: str = "head"): + if not isinstance(features, (list, tuple)): + return [] + # Validation + filtering in one pass; will raise if a role dir is missing. + valid = [] + for f in features: + name = str(f) + if _has_snippet(name, kind): + valid.append(name) + return valid + +class FilterModule(object): + def filters(self): + return { + "inj_features": inj_features_filter, + } diff --git a/roles/sys-srv-web-inj-compose/tasks/main.yml b/roles/sys-srv-web-inj-compose/tasks/main.yml index f2b0150d..badf4167 100644 --- a/roles/sys-srv-web-inj-compose/tasks/main.yml +++ b/roles/sys-srv-web-inj-compose/tasks/main.yml @@ -39,6 +39,8 @@ - name: Reinitialize 'inj_enabled' for '{{ domain }}', after modification by CDN set_fact: inj_enabled: "{{ applications | inj_enabled(application_id, SRV_WEB_INJ_COMP_FEATURES_ALL) }}" + inj_head_features: "{{ SRV_WEB_INJ_COMP_FEATURES_ALL | inj_features('head') }}" + inj_body_features: "{{ SRV_WEB_INJ_COMP_FEATURES_ALL | inj_features('body') }}" - name: "Activate Corporate CSS for '{{ domain }}'" include_role: diff --git a/roles/sys-srv-web-inj-compose/templates/location.lua.j2 b/roles/sys-srv-web-inj-compose/templates/location.lua.j2 index c924e600..795c27c6 100644 --- a/roles/sys-srv-web-inj-compose/templates/location.lua.j2 +++ b/roles/sys-srv-web-inj-compose/templates/location.lua.j2 @@ -1,15 +1,10 @@ +{# roles/sys-srv-web-inj-compose/templates/location.lua.j2 #} {% macro push_snippets(list_name, features) -%} -{% for f in features -%} -{% if inj_enabled.get(f) -%} +{% set kind = list_name | regex_replace('_snippets$','') %} +{% for f in features if inj_enabled.get(f) -%} {{ list_name }}[#{{ list_name }} + 1] = [=[ - {%- include - 'roles/sys-srv-web-inj-' ~ f ~ - '/templates/' ~ - ('head' if list_name == 'head_snippets' else 'body') ~ - '_sub.j2' - -%} + {%- include 'roles/sys-srv-web-inj-' ~ f ~ '/templates/' ~ kind ~ '_sub.j2' -%} ]=] -{% endif -%} {% endfor -%} {%- endmacro %} @@ -48,7 +43,7 @@ body_filter_by_lua_block { local whole = table.concat(ngx.ctx.buf) ngx.ctx.buf = nil -- clear buffer - -- remove html CSP, due to management via infinito nexus policies + -- remove html CSP, due to management via Infinito.Nexus policies whole = whole:gsub( ']-http%-equiv=["\']Content%-Security%-Policy["\'][^>]->%s*', '' @@ -57,21 +52,21 @@ body_filter_by_lua_block { -- build a list of head-injection snippets local head_snippets = {} - {{ push_snippets('head_snippets', ['css','matomo','desktop','javascript','logout']) }} + {{ push_snippets('head_snippets', inj_head_features) }} -- inject all collected snippets right before local head_payload = table.concat(head_snippets, "\n") .. "" - whole = string.gsub(whole, "", head_payload) + whole = ngx.re.gsub(whole, "", head_payload, "ijo", nil, 1) -- build a list of body-injection snippets local body_snippets = {} - {{ push_snippets('body_snippets', ['matomo','logout','desktop']) }} + {{ push_snippets('body_snippets', inj_body_features) }} -- inject all collected snippets right before local body_payload = table.concat(body_snippets, "\n") .. "" - whole = string.gsub(whole, "", body_payload) + whole = ngx.re.gsub(whole, "", body_payload, "ijo", nil, 1) -- finally send the modified HTML out ngx.arg[1] = whole -} \ No newline at end of file +} diff --git a/roles/web-app-gitea/Administration.md b/roles/web-app-gitea/Administration.md index 261c9503..063ccd6b 100644 --- a/roles/web-app-gitea/Administration.md +++ b/roles/web-app-gitea/Administration.md @@ -2,7 +2,7 @@ ## update ```bash -cd {{docker_compose.directories.instance}} +cd {{ docker_compose.directories.instance }} docker-compose down docker-compose pull docker-compose up -d @@ -17,7 +17,7 @@ Keep in mind to track and to don't interrupt the update process until the migrat ## recreate ```bash -cd {{docker_compose.directories.instance}} && docker-compose -p gitea up -d --force-recreate +cd {{ docker_compose.directories.instance }} && docker-compose -p gitea up -d --force-recreate ``` ## database access diff --git a/roles/web-app-listmonk/tasks/main.yml b/roles/web-app-listmonk/tasks/main.yml index 9daf9bd4..f39d28a6 100644 --- a/roles/web-app-listmonk/tasks/main.yml +++ b/roles/web-app-listmonk/tasks/main.yml @@ -27,7 +27,7 @@ - name: Run Listmonk setup only if DB is empty command: cmd: docker compose run -T --rm application sh -c "yes | ./listmonk --install" - chdir: "{{docker_compose.directories.instance}}" + chdir: "{{ docker_compose.directories.instance }}" when: "'No relations found.' in db_tables.stdout" - name: Build OIDC settings JSON diff --git a/roles/web-app-mastodon/tasks/01_setup.yml b/roles/web-app-mastodon/tasks/01_setup.yml index d105aab4..ec2c0f94 100644 --- a/roles/web-app-mastodon/tasks/01_setup.yml +++ b/roles/web-app-mastodon/tasks/01_setup.yml @@ -1,7 +1,7 @@ - name: "Execute migration for '{{ application_id }}'" command: cmd: "docker-compose run --rm web bundle exec rails db:migrate" - chdir: "{{docker_compose.directories.instance}}" + chdir: "{{ docker_compose.directories.instance }}" - name: "Include administrator routines for '{{ application_id }}'" include_tasks: 02_administrator.yml \ No newline at end of file diff --git a/roles/web-app-mastodon/tasks/02_administrator.yml b/roles/web-app-mastodon/tasks/02_administrator.yml index da621d2c..e7f8739e 100644 --- a/roles/web-app-mastodon/tasks/02_administrator.yml +++ b/roles/web-app-mastodon/tasks/02_administrator.yml @@ -1,7 +1,7 @@ # Routines to create the administrator account # @see https://chatgpt.com/share/67b9b12c-064c-800f-9354-8e42e6459764 -- name: Check health status of {{ item }} container +- name: Check health status of '{{ item }}' container shell: | cid=$(docker compose ps -q {{ item }}) docker inspect \ @@ -19,25 +19,29 @@ - sidekiq loop_control: label: "{{ item }}" + changed_when: false - name: Remove line containing "- administrator" from config/settings.yml to allow creating administrator account command: cmd: "docker compose exec -u root web sed -i '/- administrator/d' config/settings.yml" - chdir: "{{docker_compose.directories.instance}}" + chdir: "{{ docker_compose.directories.instance }}" when: users.administrator.username == "administrator" - name: Create admin account via tootctl - command: + command: cmd: 'docker compose exec -u root web bash -c "RAILS_ENV=production bin/tootctl accounts create {{users.administrator.username}} --email {{ users.administrator.email }} --confirmed --role Owner"' - chdir: "{{docker_compose.directories.instance}}" + chdir: "{{ docker_compose.directories.instance }}" register: tootctl_create changed_when: tootctl_create.rc == 0 failed_when: > tootctl_create.rc != 0 and ("taken" not in tootctl_create.stderr | lower) + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" - name: Approve the administrator account in Mastodon command: cmd: docker compose exec -u root web bash -c "RAILS_ENV=production bin/tootctl accounts modify {{users.administrator.username}} --approve" - chdir: "{{docker_compose.directories.instance}}" \ No newline at end of file + chdir: "{{ docker_compose.directories.instance }}" + async: "{{ ASYNC_TIME if ASYNC_ENABLED | bool else omit }}" + poll: "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}" \ No newline at end of file diff --git a/roles/web-app-mediawiki/tasks/main.yml b/roles/web-app-mediawiki/tasks/main.yml index b53daf34..672720cc 100644 --- a/roles/web-app-mediawiki/tasks/main.yml +++ b/roles/web-app-mediawiki/tasks/main.yml @@ -4,5 +4,5 @@ name: cmp-db-docker-proxy - name: add docker-compose.yml - template: src=docker-compose.yml.j2 dest={{docker_compose.directories.instance}}docker-compose.yml + template: src=docker-compose.yml.j2 dest={{ docker_compose.directories.instance }}docker-compose.yml notify: docker compose up diff --git a/roles/web-app-mybb/tasks/main.yml b/roles/web-app-mybb/tasks/main.yml index fa9b5952..08058176 100644 --- a/roles/web-app-mybb/tasks/main.yml +++ b/roles/web-app-mybb/tasks/main.yml @@ -32,5 +32,5 @@ - name: add docker-compose.yml template: src: "docker-compose.yml.j2" - dest: "{{docker_compose.directories.instance}}docker-compose.yml" + dest: "{{ docker_compose.directories.instance }}docker-compose.yml" notify: docker compose up diff --git a/roles/web-app-mybb/vars/main.yml b/roles/web-app-mybb/vars/main.yml index a0143aae..00cb9f63 100644 --- a/roles/web-app-mybb/vars/main.yml +++ b/roles/web-app-mybb/vars/main.yml @@ -1,6 +1,6 @@ --- application_id: "web-app-mybb" -docker_compose_instance_confd_directory: "{{docker_compose.directories.instance}}conf.d/" +docker_compose_instance_confd_directory: "{{ docker_compose.directories.instance }}conf.d/" docker_compose_instance_confd_defaultconf_file: "{{docker_compose_instance_confd_directory}}default.conf" target_mount_conf_d_directory: "{{ NGINX.DIRECTORIES.HTTP.SERVERS }}" source_domain: "mybb.{{ PRIMARY_DOMAIN }}" diff --git a/roles/web-app-roulette-wheel/vars/main.yml b/roles/web-app-roulette-wheel/vars/main.yml index 81647d28..6f2c6d98 100644 --- a/roles/web-app-roulette-wheel/vars/main.yml +++ b/roles/web-app-roulette-wheel/vars/main.yml @@ -1,2 +1,2 @@ application_id: "web-app-roulette-wheel" -app_path: "{{docker_compose.directories.instance}}/app/" \ No newline at end of file +app_path: "{{ docker_compose.directories.instance }}/app/" \ No newline at end of file diff --git a/roles/web-app-taiga/vars/main.yml b/roles/web-app-taiga/vars/main.yml index 5c0d054f..07c83e3d 100644 --- a/roles/web-app-taiga/vars/main.yml +++ b/roles/web-app-taiga/vars/main.yml @@ -2,7 +2,7 @@ application_id: "web-app-taiga" database_type: "postgres" docker_repository_address: "https://github.com/taigaio/taiga-docker" email_backend: "smtp" ## use an SMTP server or display the emails in the console (either "smtp" or "console") -docker_compose_init: "{{docker_compose.directories.instance}}docker-compose-inits.yml.j2" +docker_compose_init: "{{ docker_compose.directories.instance }}docker-compose-inits.yml.j2" taiga_image_backend: >- {{ 'robrotheram/taiga-back-openid' if applications | get_app_conf(application_id, 'features.oidc', True) and applications | get_app_conf(application_id, 'oidc.flavor', True) == 'robrotheram' else 'taigaio/taiga-back' }} diff --git a/tasks/utils/update-repository-with-files.yml b/tasks/utils/update-repository-with-files.yml index 7b4c5ca9..afd8b1f4 100644 --- a/tasks/utils/update-repository-with-files.yml +++ b/tasks/utils/update-repository-with-files.yml @@ -9,9 +9,9 @@ - name: "backup detached files" command: > - mv "{{docker_compose.directories.instance}}{{ item }}" "/tmp/{{ application_id }}-{{ item }}.backup" + mv "{{ docker_compose.directories.instance }}{{ item }}" "/tmp/{{ application_id }}-{{ item }}.backup" args: - removes: "{{docker_compose.directories.instance}}{{ item }}" + removes: "{{ docker_compose.directories.instance }}{{ item }}" become: true loop: "{{ merged_detached_files | default(detached_files) }}" @@ -19,12 +19,12 @@ ansible.builtin.shell: git checkout . become: true args: - chdir: "{{docker_compose.directories.instance}}" + chdir: "{{ docker_compose.directories.instance }}" ignore_errors: true - name: "restore detached files" command: > - mv "/tmp/{{ application_id }}-{{ item }}.backup" "{{docker_compose.directories.instance}}{{ item }}" + mv "/tmp/{{ application_id }}-{{ item }}.backup" "{{ docker_compose.directories.instance }}{{ item }}" args: removes: "/tmp/{{ application_id }}-{{ item }}.backup" become: true @@ -33,6 +33,6 @@ - name: "copy {{ detached_files }} templates to server" template: src: "{{ item }}.j2" - dest: "{{docker_compose.directories.instance}}{{ item }}" + dest: "{{ docker_compose.directories.instance }}{{ item }}" loop: "{{ detached_files }}" notify: docker compose up diff --git a/tests/unit/roles/srv-web-inj-compose/__init__.py b/tests/unit/roles/sys-srv-web-inj-compose/__init__.py similarity index 100% rename from tests/unit/roles/srv-web-inj-compose/__init__.py rename to tests/unit/roles/sys-srv-web-inj-compose/__init__.py diff --git a/tests/unit/roles/srv-web-inj-compose/filter_plugins/__init__.py b/tests/unit/roles/sys-srv-web-inj-compose/filter_plugins/__init__.py similarity index 100% rename from tests/unit/roles/srv-web-inj-compose/filter_plugins/__init__.py rename to tests/unit/roles/sys-srv-web-inj-compose/filter_plugins/__init__.py diff --git a/tests/unit/roles/srv-web-inj-compose/filter_plugins/test_inj_enabled.py b/tests/unit/roles/sys-srv-web-inj-compose/filter_plugins/test_inj_enabled.py similarity index 100% rename from tests/unit/roles/srv-web-inj-compose/filter_plugins/test_inj_enabled.py rename to tests/unit/roles/sys-srv-web-inj-compose/filter_plugins/test_inj_enabled.py diff --git a/tests/unit/roles/sys-srv-web-inj-compose/filter_plugins/test_inj_snippets.py b/tests/unit/roles/sys-srv-web-inj-compose/filter_plugins/test_inj_snippets.py new file mode 100644 index 00000000..ec3f8660 --- /dev/null +++ b/tests/unit/roles/sys-srv-web-inj-compose/filter_plugins/test_inj_snippets.py @@ -0,0 +1,106 @@ +# tests/unit/roles/sys-srv-web-inj-compose/filter_plugins/test_inj_snippets.py +""" +Unit tests for roles/sys-srv-web-inj-compose/filter_plugins/inj_snippets.py + +- Uses tempfile.TemporaryDirectory for an isolated roles/ tree. +- Loads inj_snippets.py by absolute path (no sys.path issues). +- Monkey-patches inj_snippets._ROLES_DIR to the temp roles/ path. +- Calls the filter function via the loaded module to avoid method-binding. +""" + +import os +import sys +import unittest +import tempfile +import importlib.util + + +class TestInjSnippets(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Find repo root by locating inj_snippets.py upwards from this file + cls.test_dir = os.path.dirname(__file__) + root = cls.test_dir + inj_rel = os.path.join( + "roles", "sys-srv-web-inj-compose", "filter_plugins", "inj_snippets.py" + ) + + while True: + candidate = os.path.join(root, inj_rel) + if os.path.isfile(candidate): + cls.repo_root = root + cls.inj_snippets_path = candidate + break + parent = os.path.dirname(root) + if parent == root: + raise RuntimeError(f"Could not locate {inj_rel} above {cls.test_dir}") + root = parent + + # Create isolated temporary roles tree + cls.tmp = tempfile.TemporaryDirectory(prefix="inj-snippets-test-") + cls.roles_dir = os.path.join(cls.tmp.name, "roles") + os.makedirs(cls.roles_dir, exist_ok=True) + + # Dynamically load inj_snippets by file path + spec = importlib.util.spec_from_file_location("inj_snippets", cls.inj_snippets_path) + if spec is None or spec.loader is None: + raise RuntimeError("Failed to create import spec for inj_snippets.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + # Point the module to our temp roles/ directory + module._ROLES_DIR = cls.roles_dir + + # Keep the loaded module for calls + cls.mod = module + + # Mock feature names + cls.feature_head_only = "zz_headonly" + cls.feature_body_only = "zz_bodyonly" + cls.feature_both = "zz_both" + cls.feature_missing = "zz_missing" + + # Create mock roles and snippet files + cls._mkrole(cls.feature_head_only, head=True, body=False) + cls._mkrole(cls.feature_body_only, head=False, body=True) + cls._mkrole(cls.feature_both, head=True, body=True) + + @classmethod + def _mkrole(cls, feature, head=False, body=False): + role_dir = os.path.join(cls.roles_dir, f"sys-srv-web-inj-{feature}") + tmpl_dir = os.path.join(role_dir, "templates") + os.makedirs(tmpl_dir, exist_ok=True) + if head: + with open(os.path.join(tmpl_dir, "head_sub.j2"), "w", encoding="utf-8") as f: + f.write("\n") + if body: + with open(os.path.join(tmpl_dir, "body_sub.j2"), "w", encoding="utf-8") as f: + f.write("\n") + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def test_head_features_filter(self): + features = [self.feature_head_only, self.feature_both, self.feature_body_only] + result = self.mod.inj_features_filter(features, kind="head") + self.assertEqual(result, [self.feature_head_only, self.feature_both]) + + def test_body_features_filter(self): + features = [self.feature_head_only, self.feature_both, self.feature_body_only] + result = self.mod.inj_features_filter(features, kind="body") + self.assertEqual(result, [self.feature_both, self.feature_body_only]) + + def test_raises_when_role_dir_missing(self): + with self.assertRaises(FileNotFoundError): + self.mod.inj_features_filter([self.feature_missing], kind="head") + with self.assertRaises(FileNotFoundError): + self.mod.inj_features_filter([self.feature_missing], kind="body") + + def test_non_list_input_returns_empty(self): + self.assertEqual(self.mod.inj_features_filter("not-a-list", kind="head"), []) + self.assertEqual(self.mod.inj_features_filter(None, kind="body"), []) + + +if __name__ == "__main__": + unittest.main()