From a18e8880443704d1d6cff061b2e59ec84fb196e7 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sun, 13 Jul 2025 12:57:46 +0200 Subject: [PATCH] Implemented new matomo setup --- filter_plugins/csp_filters.py | 2 +- filter_plugins/docker_service_enabled.py | 25 +++++ group_vars/all/09_ports.yml | 2 +- group_vars/all/10_networks.yml | 2 +- roles/Todo.md | 2 + roles/docker-compose/templates/base.yml.j2 | 4 +- .../docker-compose/templates/networks.yml.j2 | 2 +- .../templates/depends_on/dbms_base.yml.j2 | 2 +- .../templates/depends_on/dmbs_excl.yml.j2 | 2 +- .../templates/networks.yml.j2 | 2 +- .../templates/matomo-tracking.conf.j2 | 2 +- .../templates/matomo-tracking.js.j2 | 2 +- roles/srv-web-7-7-inj-matomo/vars/main.yml | 4 +- roles/svc-db-mariadb/config/main.yml | 5 +- roles/svc-db-mariadb/tasks/main.yml | 4 +- roles/svc-db-postgres/config/main.yml | 3 +- roles/svc-db-postgres/tasks/main.yml | 2 +- roles/web-app-matomo/config/main.yml | 12 ++- .../templates/docker-compose.yml.j2 | 2 +- roles/web-app-matomo/vars/main.yml | 8 +- tasks/utils/debug/docker-compose.yml | 4 +- tests/unit/filter_plugins/test_csp_filters.py | 2 +- .../test_docker_service_enabled.py | 100 ++++++++++++++++++ .../unit/filter_plugins/test_text_filters.py | 4 +- 24 files changed, 168 insertions(+), 31 deletions(-) create mode 100644 filter_plugins/docker_service_enabled.py create mode 100644 roles/Todo.md create mode 100644 tests/unit/filter_plugins/test_docker_service_enabled.py diff --git a/filter_plugins/csp_filters.py b/filter_plugins/csp_filters.py index b260c74b..ae6dc214 100644 --- a/filter_plugins/csp_filters.py +++ b/filter_plugins/csp_filters.py @@ -112,7 +112,7 @@ class FilterModule(object): self.is_feature_enabled(applications, matomo_feature_name, application_id) and directive in ['script-src-elem', 'connect-src'] ): - matomo_domain = domains.get('matomo')[0] + matomo_domain = domains.get('web-app-matomo')[0] if matomo_domain: tokens.append(f"{web_protocol}://{matomo_domain}") diff --git a/filter_plugins/docker_service_enabled.py b/filter_plugins/docker_service_enabled.py new file mode 100644 index 00000000..460727c7 --- /dev/null +++ b/filter_plugins/docker_service_enabled.py @@ -0,0 +1,25 @@ +class FilterModule(object): + ''' Custom filter to safely check if a docker service is enabled for an application_id ''' + + def filters(self): + return { + 'is_docker_service_enabled': self.is_docker_service_enabled + } + + @staticmethod + def is_docker_service_enabled(applications, application_id, service_name): + """ + Returns True if applications[application_id].docker.services[service_name].enabled is truthy, + otherwise returns False (even if intermediate keys are missing). + """ + try: + return bool( + applications + and application_id in applications + and applications[application_id].get('docker', {}) + .get('services', {}) + .get(service_name, {}) + .get('enabled', False) + ) + except Exception: + return False diff --git a/group_vars/all/09_ports.yml b/group_vars/all/09_ports.yml index a3918c94..9d28261e 100644 --- a/group_vars/all/09_ports.yml +++ b/group_vars/all/09_ports.yml @@ -35,7 +35,7 @@ ports: attendize: 8015 pgadmin: 8016 baserow: 8017 - matomo: 8018 + web-app-matomo: 8018 listmonk: 8019 discourse: 8020 matrix_synapse: 8021 diff --git a/group_vars/all/10_networks.yml b/group_vars/all/10_networks.yml index 53b13749..5d2d2782 100644 --- a/group_vars/all/10_networks.yml +++ b/group_vars/all/10_networks.yml @@ -36,7 +36,7 @@ defaults_networks: subnet: 192.168.101.192/28 # Free: # subnet: 192.168.101.208/28 - matomo: + web-app-matomo: subnet: 192.168.101.224/28 mastodon: subnet: 192.168.101.240/28 diff --git a/roles/Todo.md b/roles/Todo.md new file mode 100644 index 00000000..abb338b6 --- /dev/null +++ b/roles/Todo.md @@ -0,0 +1,2 @@ +# Todos +- Use at all applications the ansible role name as application_id \ No newline at end of file diff --git a/roles/docker-compose/templates/base.yml.j2 b/roles/docker-compose/templates/base.yml.j2 index 3dd8242f..fa3baece 100644 --- a/roles/docker-compose/templates/base.yml.j2 +++ b/roles/docker-compose/templates/base.yml.j2 @@ -1,11 +1,11 @@ {# Base template for all docker-compose.yml.j2 #} services: {# Load Database #} -{% if applications[application_id].docker.services.database.enabled | default(false) | bool %} +{% if applications | is_docker_service_enabled(application_id, 'database') %} {% include 'roles/cmp-rdbms/templates/services/main.yml.j2' %} {% endif %} {# Load Redis #} -{% if applications[application_id].docker.services.redis.enabled | default(false) | bool %} +{% if applications | is_docker_service_enabled(application_id, 'redis') %} {% include 'roles/svc-db-redis/templates/service.yml.j2' %} {% endif %} {# Load OAuth2 Proxy #} diff --git a/roles/docker-compose/templates/networks.yml.j2 b/roles/docker-compose/templates/networks.yml.j2 index 7e97bdaf..333c705c 100644 --- a/roles/docker-compose/templates/networks.yml.j2 +++ b/roles/docker-compose/templates/networks.yml.j2 @@ -4,7 +4,7 @@ networks: {{ applications[ 'svc-db-' ~ database_type ].network }}: external: true {% endif %} -{% if applications[application_id].get('features', {}).get('ldap', false) and applications['svc-db-openldap'].network.docker | bool %} +{% if applications | is_feature_enabled('ldap',application_id) and applications['svc-db-openldap'].network.docker | bool %} svc-db-openldap: external: true {% endif %} diff --git a/roles/docker-container/templates/depends_on/dbms_base.yml.j2 b/roles/docker-container/templates/depends_on/dbms_base.yml.j2 index edadb54a..0d62feef 100644 --- a/roles/docker-container/templates/depends_on/dbms_base.yml.j2 +++ b/roles/docker-container/templates/depends_on/dbms_base.yml.j2 @@ -2,7 +2,7 @@ {{ database_host }}: condition: service_healthy {% endif %} -{% if applications[application_id].docker.services.redis.enabled | default(false) | bool %} +{% if applications | is_docker_service_enabled(application_id, 'redis') %} redis: condition: service_healthy {% endif %} \ No newline at end of file diff --git a/roles/docker-container/templates/depends_on/dmbs_excl.yml.j2 b/roles/docker-container/templates/depends_on/dmbs_excl.yml.j2 index 8071b507..456a1aaa 100644 --- a/roles/docker-container/templates/depends_on/dmbs_excl.yml.j2 +++ b/roles/docker-container/templates/depends_on/dmbs_excl.yml.j2 @@ -1,6 +1,6 @@ {# This template needs to be included in docker-compose.yml containers, which depend on a database, redis and optional additional volumes #} {% if - (applications[application_id].docker.services.redis.enabled | default(false)| bool) or + applications | is_docker_service_enabled(application_id, 'redis') or not applications | is_feature_enabled('central_database',application_id) %} depends_on: diff --git a/roles/docker-container/templates/networks.yml.j2 b/roles/docker-container/templates/networks.yml.j2 index 759e7f0a..bb949990 100644 --- a/roles/docker-container/templates/networks.yml.j2 +++ b/roles/docker-container/templates/networks.yml.j2 @@ -3,7 +3,7 @@ {% if applications | is_feature_enabled('central_database',application_id) | bool and database_type is defined %} {{ applications[ 'svc-db-' ~ database_type ].network }}: {% endif %} -{% if applications[application_id].get('features', {}).get('ldap', false) | bool and applications['svc-db-openldap'].network.docker|bool %} +{% if applications | is_feature_enabled('ldap',application_id) | bool and applications['svc-db-openldap'].network.docker|bool %} svc-db-openldap: {% endif %} default: diff --git a/roles/srv-web-7-7-inj-matomo/templates/matomo-tracking.conf.j2 b/roles/srv-web-7-7-inj-matomo/templates/matomo-tracking.conf.j2 index 53f7dc67..76eb6553 100644 --- a/roles/srv-web-7-7-inj-matomo/templates/matomo-tracking.conf.j2 +++ b/roles/srv-web-7-7-inj-matomo/templates/matomo-tracking.conf.j2 @@ -1,2 +1,2 @@ # sub filters to integrate matomo tracking code in nginx websites -sub_filter '' ''; \ No newline at end of file +sub_filter '' ''; \ No newline at end of file diff --git a/roles/srv-web-7-7-inj-matomo/templates/matomo-tracking.js.j2 b/roles/srv-web-7-7-inj-matomo/templates/matomo-tracking.js.j2 index f95df441..bc71b13c 100644 --- a/roles/srv-web-7-7-inj-matomo/templates/matomo-tracking.js.j2 +++ b/roles/srv-web-7-7-inj-matomo/templates/matomo-tracking.js.j2 @@ -7,7 +7,7 @@ _paq.push(["trackPageView"]); _paq.push(["trackAllContentImpressions"]); _paq.push(["enableLinkTracking"]); (function() { - var u="//{{ domains | get_domain('matomo') }}/"; + var u="//{{ domains | get_domain('web-app-matomo') }}/"; _paq.push(["setTrackerUrl", u+"matomo.php"]); _paq.push(["setSiteId", "{{matomo_site_id}}"]); var d=document, g=d.createElement("script"), s=d.getElementsByTagName("script")[0]; diff --git a/roles/srv-web-7-7-inj-matomo/vars/main.yml b/roles/srv-web-7-7-inj-matomo/vars/main.yml index fbbd97fc..c1d33e9c 100644 --- a/roles/srv-web-7-7-inj-matomo/vars/main.yml +++ b/roles/srv-web-7-7-inj-matomo/vars/main.yml @@ -1,4 +1,4 @@ base_domain: "{{ domain | regex_replace('^(?:.*\\.)?(.+\\..+)$', '\\1') }}" -matomo_index_php_url: "{{ domains | get_url('matomo', web_protocol) }}/index.php" -matomo_auth_token: "{{ applications.matomo.credentials.auth_token }}" +matomo_index_php_url: "{{ domains | get_url('web-app-matomo', web_protocol) }}/index.php" +matomo_auth_token: "{{ applications['web-app-matomo'].credentials.auth_token }}" matomo_verification_url: "{{ matomo_index_php_url }}?module=API&method=SitesManager.getSitesIdFromSiteUrl&url=https://{{ base_domain }}&format=json&token_auth={{ matomo_auth_token }}" \ No newline at end of file diff --git a/roles/svc-db-mariadb/config/main.yml b/roles/svc-db-mariadb/config/main.yml index 93a4fbd5..1ad11e4a 100644 --- a/roles/svc-db-mariadb/config/main.yml +++ b/roles/svc-db-mariadb/config/main.yml @@ -1,4 +1,5 @@ version: "latest" hostname: "svc-db-mariadb" -network: "svc-db-mariadb" -port: 5432 \ No newline at end of file +network: "<< defaults_applications[svc-db-mariadb].hostname >>" +port: 5432 +volume: "<< defaults_applications[svc-db-mariadb].hostname >>_data" \ No newline at end of file diff --git a/roles/svc-db-mariadb/tasks/main.yml b/roles/svc-db-mariadb/tasks/main.yml index 0a35227d..4743055b 100644 --- a/roles/svc-db-mariadb/tasks/main.yml +++ b/roles/svc-db-mariadb/tasks/main.yml @@ -17,7 +17,7 @@ networks: - name: "{{ applications['svc-db-mariadb'].network }}" volumes: - - mariadb_database:/var/lib/mysql + - "{{ applications['svc-db-mariadb'].volume }}:/var/lib/mysql" published_ports: - "127.0.0.1:{{database_port}}:3306" # can be that this will be removed if all applications use sockets command: "--transaction-isolation=READ-COMMITTED --binlog-format=ROW" #for nextcloud @@ -36,7 +36,7 @@ state: present when: run_once_docker_mariadb is not defined -- name: Wait until the MariaDB container is healthy +- name: "Wait until the MariaDB container (hostname {{ applications['svc-db-mariadb'].hostname }}) is healthy" community.docker.docker_container_info: name: "{{ applications['svc-db-mariadb'].hostname }}" register: db_info diff --git a/roles/svc-db-postgres/config/main.yml b/roles/svc-db-postgres/config/main.yml index 19ade214..ccccab66 100644 --- a/roles/svc-db-postgres/config/main.yml +++ b/roles/svc-db-postgres/config/main.yml @@ -1,6 +1,7 @@ hostname: "svc-db-postgres" -network: "svc-db-postgres" +network: "<< defaults_applications[svc-db-postgres].hostname >>" port: 5432 +volume: "<< defaults_applications[svc-db-postgres].hostname >>" docker: images: # Postgis is necessary for mobilizon diff --git a/roles/svc-db-postgres/tasks/main.yml b/roles/svc-db-postgres/tasks/main.yml index f2059efb..324327ed 100644 --- a/roles/svc-db-postgres/tasks/main.yml +++ b/roles/svc-db-postgres/tasks/main.yml @@ -19,7 +19,7 @@ published_ports: - "127.0.0.1:{{ applications[application_id].port }}:5432" volumes: - - postgres_database:/var/lib/postgresql/data + - "{{ applications['svc-db-postgres'].volume }}:/var/lib/postgresql/data" restart_policy: "{{ docker_restart_policy }}" healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] diff --git a/roles/web-app-matomo/config/main.yml b/roles/web-app-matomo/config/main.yml index 7bdfdc15..b7a03802 100644 --- a/roles/web-app-matomo/config/main.yml +++ b/roles/web-app-matomo/config/main.yml @@ -1,7 +1,11 @@ images: matomo: "matomo:latest" features: - matomo: true + # If you want to use Matomo on the Matomo page, you + # have to set it here manual to true. + # It's deactivated, because the proxy setup for Matomo + # itself wouldn't be possible + matomo: false css: false portfolio_iframe: true central_database: true @@ -24,9 +28,13 @@ csp: domains: aliases: - "analytics.{{ primary_domain }}" + canonical: + - "matomo.{{ primary_domain }}" excluded_ips: "{{ networks.internet.values() | list }}" docker: services: database: - enabled: true \ No newline at end of file + enabled: true + redis: + enabled: false \ No newline at end of file diff --git a/roles/web-app-matomo/templates/docker-compose.yml.j2 b/roles/web-app-matomo/templates/docker-compose.yml.j2 index 43379339..10749996 100644 --- a/roles/web-app-matomo/templates/docker-compose.yml.j2 +++ b/roles/web-app-matomo/templates/docker-compose.yml.j2 @@ -2,7 +2,7 @@ application: {% set container_port = 80 %} {% include 'roles/docker-container/templates/base.yml.j2' %} - image: "{{ applications[application_id].images[application_id] }}" + image: "{{ applications[application_id].images['matomo'] }}" ports: - "127.0.0.1:{{ports.localhost.http[application_id]}}:{{ container_port }}" volumes: diff --git a/roles/web-app-matomo/vars/main.yml b/roles/web-app-matomo/vars/main.yml index 2518430e..4b30df38 100644 --- a/roles/web-app-matomo/vars/main.yml +++ b/roles/web-app-matomo/vars/main.yml @@ -1,9 +1,9 @@ --- -application_id: "matomo" +application_id: "web-app-matomo" database_type: "mariadb" -matomo_excluded_ips: "{{ applications.matomo.excluded_ips }}" -matomo_index_php_url: "{{ domains | get_url('matomo', web_protocol) }}/index.php" -matomo_auth_token: "{{ applications.matomo.credentials.auth_token }}" +matomo_excluded_ips: "{{ applications[application_id].excluded_ips }}" +matomo_index_php_url: "{{ domains | get_url(application_id, web_protocol) }}/index.php" +matomo_auth_token: "{{ applications[application_id].credentials.auth_token }}" # I don't know if this is still necessary diff --git a/tasks/utils/debug/docker-compose.yml b/tasks/utils/debug/docker-compose.yml index 9a9da175..f73d30bd 100644 --- a/tasks/utils/debug/docker-compose.yml +++ b/tasks/utils/debug/docker-compose.yml @@ -30,12 +30,12 @@ - name: Check if ports.localhost.oauth2_proxy[application_id] exists set_fact: missing_keys: "{{ missing_keys + ['ports.localhost.oauth2_proxy.{}'.format(application_id)] }}" - when: ports.localhost.oauth2_proxy.get(application_id, None) is not defined + when: ports.localhost.oauth2_proxy get(application_id, None) is not defined - name: Check if ports.localhost.http[application_id] exists set_fact: missing_keys: "{{ missing_keys + ['ports.localhost.http.{}'.format(application_id)] }}" - when: ports.localhost.http.get(application_id, None) is not defined + when: ports.localhost.http get(application_id, None) is not defined - name: Check if networks.local[application_id].subnet exists (optional) set_fact: diff --git a/tests/unit/filter_plugins/test_csp_filters.py b/tests/unit/filter_plugins/test_csp_filters.py index cd5bfc72..da34d6d8 100644 --- a/tests/unit/filter_plugins/test_csp_filters.py +++ b/tests/unit/filter_plugins/test_csp_filters.py @@ -49,7 +49,7 @@ class TestCspFilters(unittest.TestCase): 'app2': {} } self.domains = { - 'matomo': ['matomo.example.org'] + 'web-app-matomo': ['matomo.example.org'] } def test_get_csp_whitelist_list(self): diff --git a/tests/unit/filter_plugins/test_docker_service_enabled.py b/tests/unit/filter_plugins/test_docker_service_enabled.py new file mode 100644 index 00000000..d7c0e554 --- /dev/null +++ b/tests/unit/filter_plugins/test_docker_service_enabled.py @@ -0,0 +1,100 @@ +import unittest + +# Import the filter module directly (adjust path as needed) +try: + from filter_plugins.docker_service_enabled import FilterModule +except ModuleNotFoundError: + import sys + import os + sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../filter_plugins'))) + from docker_service_enabled import FilterModule + +class TestIsDockerServiceEnabledFilter(unittest.TestCase): + def setUp(self): + self.filter = FilterModule().filters()['is_docker_service_enabled'] + + def test_enabled_true(self): + applications = { + 'app1': { + 'docker': { + 'services': { + 'redis': {'enabled': True}, + 'database': {'enabled': True}, + } + } + } + } + self.assertTrue(self.filter(applications, 'app1', 'redis')) + self.assertTrue(self.filter(applications, 'app1', 'database')) + + def test_enabled_false(self): + applications = { + 'app1': { + 'docker': { + 'services': { + 'redis': {'enabled': False}, + 'database': {'enabled': False}, + } + } + } + } + self.assertFalse(self.filter(applications, 'app1', 'redis')) + self.assertFalse(self.filter(applications, 'app1', 'database')) + + def test_missing_enabled_key(self): + applications = { + 'app1': { + 'docker': { + 'services': { + 'redis': {}, + 'database': {}, + } + } + } + } + self.assertFalse(self.filter(applications, 'app1', 'redis')) + self.assertFalse(self.filter(applications, 'app1', 'database')) + + def test_missing_service_key(self): + applications = { + 'app1': { + 'docker': { + 'services': { + # no 'redis' or 'database' + } + } + } + } + self.assertFalse(self.filter(applications, 'app1', 'redis')) + self.assertFalse(self.filter(applications, 'app1', 'database')) + + def test_missing_services_key(self): + applications = { + 'app1': { + 'docker': { + # no 'services' + } + } + } + self.assertFalse(self.filter(applications, 'app1', 'redis')) + + def test_missing_docker_key(self): + applications = { + 'app1': { + # no 'docker' + } + } + self.assertFalse(self.filter(applications, 'app1', 'database')) + + def test_missing_app_id(self): + applications = { + 'other_app': {} + } + self.assertFalse(self.filter(applications, 'app1', 'redis')) + + def test_applications_is_none(self): + applications = None + self.assertFalse(self.filter(applications, 'app1', 'database')) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit/filter_plugins/test_text_filters.py b/tests/unit/filter_plugins/test_text_filters.py index 85f07a58..a5bacde6 100644 --- a/tests/unit/filter_plugins/test_text_filters.py +++ b/tests/unit/filter_plugins/test_text_filters.py @@ -47,8 +47,8 @@ Line three""" self.assertEqual(to_one_liner(s), expected) def test_preserve_templating_expressions(self): - s = 'var tracker = "//{{ domains | get_domain(\'matomo\') }}/matomo.js"; // loader' - expected = 'var tracker = "//{{ domains | get_domain(\'matomo\') }}/matomo.js";' + s = 'var tracker = "//{{ domains | get_domain(\'web-app-matomo\') }}/matomo.js"; // loader' + expected = 'var tracker = "//{{ domains | get_domain(\'web-app-matomo\') }}/matomo.js";' self.assertEqual(to_one_liner(s), expected) def test_mixed_string_and_comment(self):