From 60c84d57ba4d5c9c98288f034238b7cfcb449d74 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 11 Feb 2025 04:49:21 +0100 Subject: [PATCH] Optimized networking and matomo --- playbook.servers.yml | 49 ++++++++++++------- .../templates/docker-compose.yml.j2 | 3 +- roles/docker-matomo/tasks/main.yml | 15 ++++++ roles/docker-matomo/vars/main.yml | 9 ++-- .../templates/docker-compose.yml.j2 | 1 + roles/docker-nextcloud/handlers/main.yml | 1 + roles/docker-nextcloud/tasks/main.yml | 6 +-- .../templates/docker-compose.yml.j2 | 6 ++- .../templates/container.yml.j2 | 3 +- .../templates/docker-compose.yml.j2 | 3 +- roles/docker/README.md | 10 ++++ roles/nginx-global-matomo/README.md | 14 ------ roles/nginx-global-matomo/tasks/main.yml | 4 +- .../templates/matomo-tracking.js.j2 | 2 +- roles/nginx-global-matomo/vars/main.yml | 1 - 15 files changed, 77 insertions(+), 50 deletions(-) diff --git a/playbook.servers.yml b/playbook.servers.yml index a6b01912..b5a00108 100644 --- a/playbook.servers.yml +++ b/playbook.servers.yml @@ -13,7 +13,36 @@ - health-btrfs - system-btrfs-auto-balancer -# Docker Roles +######################################################################### +### Docker Roles ### +######################################################################### + +# Priority: 1 +# Almost all other roles depend on the Matomo tracking +- hosts: all + tasks: + - name: "setup matomo hosts if matomo hosts set or global_matomo_tracking_enabled" + include_role: + name: docker-matomo + when: "'matomo' in group_names or (global_matomo_tracking_enabled | bool)" + become: true + +# Priority: 2 +# Much other roles rely on a working ldap setup +- name: setup ldap + hosts: ldap + become: true + roles: + - role: docker-ldap + +# Priority: 3 +# Much other roles use OICD via Keycloak +- name: setup keycloak + hosts: keycloak + become: true + roles: + - role: docker-keycloak + - name: setup nextcloud hosts hosts: nextcloud_server become: true @@ -122,12 +151,6 @@ roles: - role: docker-baserow -- name: setup matomo hosts - hosts: matomo - become: true - roles: - - role: docker-matomo - - name: setup listmonk hosts: listmonk become: true @@ -197,18 +220,6 @@ roles: - role: docker-bluesky -- name: setup keycloak - hosts: keycloak - become: true - roles: - - role: docker-keycloak - -- name: setup ldap - hosts: ldap - become: true - roles: - - role: docker-ldap - - name: setup PHPMyAdmin hosts: phpmyadmin become: true diff --git a/roles/docker-funkwhale/templates/docker-compose.yml.j2 b/roles/docker-funkwhale/templates/docker-compose.yml.j2 index 8b529e5d..cc8ed238 100644 --- a/roles/docker-funkwhale/templates/docker-compose.yml.j2 +++ b/roles/docker-funkwhale/templates/docker-compose.yml.j2 @@ -1,5 +1,5 @@ services: - +# @todo Test which containers can be removed crom cental_database networks {% include 'roles/docker-central-database/templates/services/' + database_type + '.yml.j2' %} {% include 'templates/docker/services/redis.yml.j2' %} @@ -57,7 +57,6 @@ services: ports: # override those variables in your .env file if needed - "127.0.0.1:{{http_port}}:80" -{% include 'templates/docker/container/networks.yml.j2' %} typesense: {% include 'roles/docker-compose/templates/services/base.yml.j2' %} diff --git a/roles/docker-matomo/tasks/main.yml b/roles/docker-matomo/tasks/main.yml index 8be5742d..7fcc7db0 100644 --- a/roles/docker-matomo/tasks/main.yml +++ b/roles/docker-matomo/tasks/main.yml @@ -1,4 +1,19 @@ --- +- name: check if matomo is up + uri: + url: "https://{{ domains.matomo }}/" + method: GET + return_content: yes + status_code: 200 + validate_certs: yes + register: site_check + ignore_errors: yes + +- name: implement matomo tracking for matomo if matomo is up and tracking enabled + set_fact: + global_matomo_tracking_enabled: true + when: site_check is defined and site_check.status == 200 and global_matomo_tracking_enabled | bool + - name: "include docker-central-database" include_role: name: docker-central-database diff --git a/roles/docker-matomo/vars/main.yml b/roles/docker-matomo/vars/main.yml index f3611954..7d4c521f 100644 --- a/roles/docker-matomo/vars/main.yml +++ b/roles/docker-matomo/vars/main.yml @@ -1,4 +1,7 @@ --- -application_id: "matomo" -database_type: "mariadb" -database_password: "{{matomo_database_password}}" \ No newline at end of file +application_id: "matomo" +database_type: "mariadb" +database_password: "{{matomo_database_password}}" + +# Disable matomo tracking for matomo, because otherwise recursiv loading technics would be neccessary +# global_matomo_tracking_enabled: false \ No newline at end of file diff --git a/roles/docker-moodle/templates/docker-compose.yml.j2 b/roles/docker-moodle/templates/docker-compose.yml.j2 index 8c6ed175..19d2491a 100644 --- a/roles/docker-moodle/templates/docker-compose.yml.j2 +++ b/roles/docker-moodle/templates/docker-compose.yml.j2 @@ -2,6 +2,7 @@ services: {% include 'roles/docker-central-database/templates/services/' + database_type + '.yml.j2' %} moodle: + container_name: moodle image: docker.io/bitnami/moodle:{{applications.moodle.version}} ports: - 127.0.0.1:{{http_port}}:8080 diff --git a/roles/docker-nextcloud/handlers/main.yml b/roles/docker-nextcloud/handlers/main.yml index 04e09b2b..ead9f208 100644 --- a/roles/docker-nextcloud/handlers/main.yml +++ b/roles/docker-nextcloud/handlers/main.yml @@ -3,3 +3,4 @@ command: cmd: "docker exec {{nextcloud_nginx_container_name}} nginx -s reload" listen: restart docker nginx service + ignore_errors: true # Ignoring if container is restarting diff --git a/roles/docker-nextcloud/tasks/main.yml b/roles/docker-nextcloud/tasks/main.yml index de41dd90..a8d10544 100644 --- a/roles/docker-nextcloud/tasks/main.yml +++ b/roles/docker-nextcloud/tasks/main.yml @@ -25,6 +25,6 @@ include_tasks: oidc.yml when: oidc.enabled | bool -- name: Include LDAP specific tasks - include_tasks: ldap.yml - when: ldap.enabled | bool \ No newline at end of file +#- name: Include LDAP specific tasks +# include_tasks: ldap.yml +# when: ldap.enabled | bool \ No newline at end of file diff --git a/roles/docker-nextcloud/templates/docker-compose.yml.j2 b/roles/docker-nextcloud/templates/docker-compose.yml.j2 index cc8cbf56..703a689a 100644 --- a/roles/docker-nextcloud/templates/docker-compose.yml.j2 +++ b/roles/docker-nextcloud/templates/docker-compose.yml.j2 @@ -17,6 +17,7 @@ services: {% include 'roles/docker-compose/templates/services/base.yml.j2' %} {% include 'templates/docker/container/depends-on-database-redis.yml.j2' %} {% include 'templates/docker/container/networks.yml.j2' %} + ipv4_address: 192.168.102.66 web: image: nginx:alpine @@ -35,7 +36,9 @@ services: interval: 1m timeout: 10s retries: 3 -{% include 'templates/docker/container/networks.yml.j2' %} + networks: + default: + ipv4_address: 192.168.102.67 cron: container_name: nextcloud-cron @@ -53,6 +56,7 @@ services: retries: 3 {% include 'templates/docker/container/depends-on-database-redis.yml.j2' %} {% include 'templates/docker/container/networks.yml.j2' %} + ipv4_address: 192.168.102.68 {% include 'templates/docker/compose/volumes.yml.j2' %} data: diff --git a/roles/docker-oauth2-proxy/templates/container.yml.j2 b/roles/docker-oauth2-proxy/templates/container.yml.j2 index 3db6b243..3a043e10 100644 --- a/roles/docker-oauth2-proxy/templates/container.yml.j2 +++ b/roles/docker-oauth2-proxy/templates/container.yml.j2 @@ -6,5 +6,4 @@ ports: - {{ports.localhost.oauth2_proxy_ports[application_id]}}:4180/tcp volumes: - - "{{docker_compose.directories.volumes}}{{applications.oauth2_proxy.configuration_file}}:/oauth2-proxy.cfg" -{% include 'templates/docker/container/networks.yml.j2' %} \ No newline at end of file + - "{{docker_compose.directories.volumes}}{{applications.oauth2_proxy.configuration_file}}:/oauth2-proxy.cfg" \ No newline at end of file diff --git a/roles/docker-openproject/templates/docker-compose.yml.j2 b/roles/docker-openproject/templates/docker-compose.yml.j2 index 1eddd656..d00363c6 100644 --- a/roles/docker-openproject/templates/docker-compose.yml.j2 +++ b/roles/docker-openproject/templates/docker-compose.yml.j2 @@ -1,3 +1,4 @@ +# @todo Test which containers can be removed crom cental_database networks x-op-app: &app logging: driver: journald @@ -15,7 +16,6 @@ services: image: memcached container_name: openproject-memcached {% include 'roles/docker-compose/templates/services/base.yml.j2' %} -{% include 'templates/docker/container/networks.yml.j2' %} proxy: {% include 'roles/docker-compose/templates/services/base.yml.j2' %} @@ -28,7 +28,6 @@ services: APP_HOST: web depends_on: - web -{% include 'templates/docker/container/networks.yml.j2' %} volumes: - "data:/var/openproject/assets" - "{{dummy_volume}}:/var/openproject/pgdata" # This mount is unnecessary and just done to prevent anonymous volumes diff --git a/roles/docker/README.md b/roles/docker/README.md index 68c8a580..41f4306e 100644 --- a/roles/docker/README.md +++ b/roles/docker/README.md @@ -12,6 +12,16 @@ docker volume rm $(docker volume ls -q -f "dangling=true") ``` +### network issues +```bash +docker stop $(docker ps -a -q) +docker rm $(docker ps -a -q) +docker network prune -f +sudo iptables -t nat -F DOCKER +sudo iptables -t nat -F DOCKER-USER +``` + + ## performance - https://forums.docker.com/t/mysql-slow-performance-in-docker/37179/21 diff --git a/roles/nginx-global-matomo/README.md b/roles/nginx-global-matomo/README.md index 61ac9a0c..773a6598 100644 --- a/roles/nginx-global-matomo/README.md +++ b/roles/nginx-global-matomo/README.md @@ -11,23 +11,9 @@ This Ansible role automates the integration of Matomo tracking code into Nginx-s - Nginx installed on the target server. - Matomo analytics platform set up and accessible. -## Role Variables -- `matomo_domain`: The domain of your Matomo installation. -- `domain`: The domain of the website you wish to track. -- `matomo_auth_token`: Matomo auth token - ## Dependencies - None. This role is designed to be included in Nginx server block configurations. -## Example Usage -To enable Matomo tracking on your Nginx website, include the role in your playbook and set the required variables. - -```yaml -- hosts: webserver - roles: - - { role: nginx-global-matomo, matomo_domain: 'matomo.example.com', base_domain: 'example.com', matomo_site_id: '1' } -``` - ## Customization You can customize the tracking script and the noscript image tracker by editing the `matomo-tracking.js.j2` and `matomo.subfilter.conf.j2` templates. diff --git a/roles/nginx-global-matomo/tasks/main.yml b/roles/nginx-global-matomo/tasks/main.yml index 885ef335..c987b3ac 100644 --- a/roles/nginx-global-matomo/tasks/main.yml +++ b/roles/nginx-global-matomo/tasks/main.yml @@ -1,6 +1,6 @@ - name: Check if site already exists in Matomo uri: - url: "https://{{matomo_domain}}/index.php?module=API&method=SitesManager.getSitesIdFromSiteUrl&url=https://{{base_domain}}&format=json&token_auth={{matomo_auth_token}}" + url: "https://{{domains.matomo}}/index.php?module=API&method=SitesManager.getSitesIdFromSiteUrl&url=https://{{base_domain}}&format=json&token_auth={{matomo_auth_token}}" method: GET return_content: yes status_code: 200 @@ -14,7 +14,7 @@ - name: Add site to Matomo and get ID if not exists uri: - url: "https://{{ matomo_domain }}/index.php" + url: "https://{{ domains.matomo }}/index.php" method: POST body: "module=API&method=SitesManager.addSite&siteName={{ base_domain }}&urls=https://{{ base_domain }}&token_auth={{ matomo_auth_token }}&format=json" body_format: form-urlencoded diff --git a/roles/nginx-global-matomo/templates/matomo-tracking.js.j2 b/roles/nginx-global-matomo/templates/matomo-tracking.js.j2 index 2407ab7b..4d118c87 100644 --- a/roles/nginx-global-matomo/templates/matomo-tracking.js.j2 +++ b/roles/nginx-global-matomo/templates/matomo-tracking.js.j2 @@ -7,7 +7,7 @@ _paq.push(["trackPageView"]); _paq.push(["trackAllContentImpressions"]); _paq.push(["enableLinkTracking"]); (function() { - var u="//{{matomo_domain}}/"; + var u="//{{domains.matomo}}/"; _paq.push(["setTrackerUrl", u+"matomo.php"]); _paq.push(["setSiteId", "{{matomo_site_id}}"]); var d=document, g=d.createElement("script"), s=d.getElementsByTagName("script")[0]; diff --git a/roles/nginx-global-matomo/vars/main.yml b/roles/nginx-global-matomo/vars/main.yml index 311412d0..8cdd2ad0 100644 --- a/roles/nginx-global-matomo/vars/main.yml +++ b/roles/nginx-global-matomo/vars/main.yml @@ -1,2 +1 @@ -matomo_domain: "matomo.{{primary_domain}}" base_domain: "{{ domain | regex_replace('^(?:.*\\.)?(.+\\..+)$', '\\1') }}"