diff --git a/filter_plugins/to_primary_domain.py b/filter_plugins/to_primary_domain.py new file mode 100644 index 00000000..75583076 --- /dev/null +++ b/filter_plugins/to_primary_domain.py @@ -0,0 +1,30 @@ +from ansible.errors import AnsibleFilterError + +try: + import tld + from tld.exceptions import TldDomainNotFound, TldBadUrl +except ImportError: + raise AnsibleFilterError("The 'tld' Python package is required for the to_primary_domain filter. Install with 'pip install tld'.") + +class FilterModule(object): + ''' Custom filter to extract the primary/zone domain from a full domain name ''' + + def filters(self): + return { + 'to_primary_domain': self.to_primary_domain, + } + + def to_primary_domain(self, domain): + """ + Converts a full domain or subdomain into its primary/zone domain. + E.g. 'foo.bar.example.co.uk' -> 'example.co.uk' + """ + if not isinstance(domain, str): + raise AnsibleFilterError("Input to to_primary_domain must be a string") + try: + res = tld.get_fld(domain, fix_protocol=True) + if not res: + raise AnsibleFilterError(f"Could not extract primary domain from: {domain}") + return res + except (TldDomainNotFound, TldBadUrl) as exc: + raise AnsibleFilterError(str(exc)) diff --git a/group_vars/all/00_general.yml b/group_vars/all/00_general.yml index 192b1632..b3c41ebd 100644 --- a/group_vars/all/00_general.yml +++ b/group_vars/all/00_general.yml @@ -45,7 +45,7 @@ dns_provider: cloudflare # The DNS Prov certbot_acme_challenge_method: "cloudflare" certbot_credentials_dir: /etc/certbot certbot_credentials_file: "{{ certbot_credentials_dir }}/{{ certbot_acme_challenge_method }}.ini" -certbot_dns_api_token: "" # Define in inventory file +certbot_dns_api_token: "" # Define in inventory file: More information here: group_vars/all/docs/CLOUDFLARE_API_TOKEN.md certbot_dns_propagation_wait_seconds: 40 # How long should the script wait for DNS propagation before continuing certbot_flavor: san # Possible options: san (recommended, with a dns flavor like cloudflare, or hetzner), wildcard(doesn't function with www redirect), deicated diff --git a/group_vars/all/09_ports.yml b/group_vars/all/09_ports.yml index edf41cf7..eabbe0b3 100644 --- a/group_vars/all/09_ports.yml +++ b/group_vars/all/09_ports.yml @@ -84,3 +84,5 @@ ports: turn: web-app-bigbluebutton: 5349 # Not sure if it's right placed here or if it should be moved to localhost section web-app-nextcloud: 5350 # Not used yet + federation: + web-app-matrix_synapse: 8448 diff --git a/group_vars/all/docs/CLOUDFLARE_API_TOKEN.md b/group_vars/all/docs/CLOUDFLARE_API_TOKEN.md new file mode 100644 index 00000000..73484bb0 --- /dev/null +++ b/group_vars/all/docs/CLOUDFLARE_API_TOKEN.md @@ -0,0 +1,61 @@ +# Cloudflare API Token for Ansible (`certbot_dns_api_token`) + +This document explains how to generate and use a Cloudflare API Token for DNS automation and certificate operations in Ansible (e.g., with Certbot). + +## Purpose + +The `certbot_dns_api_token` variable must contain a valid Cloudflare API Token. +This token is used for all DNS operations and ACME (SSL/TLS certificate) challenges that require access to your Cloudflare-managed domains. + +**Never commit your API token to a public repository. Always keep it secure!** + +--- + +## How to Create a Cloudflare API Token + +### 1. Log In to Cloudflare + +- Go to: [https://dash.cloudflare.com/](https://dash.cloudflare.com/) and log in. + +### 2. Open the API Tokens Page + +- Click your profile icon (top right) → **My Profile** +- In the sidebar, choose **API Tokens** + Or use this direct link: [https://dash.cloudflare.com/profile/api-tokens](https://dash.cloudflare.com/profile/api-tokens) + +### 3. Click **Create Token** + +### 4. Select **Custom Token** + +- Give your token a descriptive name (e.g., `Ansible Certbot Automation`). + +### 5. Set Permissions + +Add the following permissions: + +| Category | Permission | Access | +| -------- | ------------ | -------- | +| Zone | Zone | Read | +| Zone | DNS | Edit | +| Zone | Cache Purge | Purge | + +- These permissions are required for DNS record management, CAA/SPF/DKIM handling, cache purging, and certificate provisioning. + +### 6. Zone Resources + +- **Zone Resources:** Set to `Include → All zones` + (Or restrict to specific zones as needed for your environment.) + +### 7. Create and Save the Token + +- Click **Continue to summary** and then **Create Token**. +- Copy the API Token. **It will only be shown once!** + +--- + +## Using the Token in Ansible + +Set the token in your Ansible inventory or secrets file: + +```yaml +certbot_dns_api_token: "cf_your_generated_token_here" diff --git a/requirements.txt b/requirements.txt index a235039f..92471f0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ colorscheme-generator @ https://github.com/kevinveenbirkenbach/colorscheme-generator/archive/refs/tags/v0.3.0.zip numpy bcrypt -ruamel.yaml \ No newline at end of file +ruamel.yaml +tld +passlib \ No newline at end of file diff --git a/requirements.yml b/requirements.yml index ebd4a5d6..79d66710 100644 --- a/requirements.yml +++ b/requirements.yml @@ -2,6 +2,7 @@ collections: - name: kewlfft.aur - name: community.general pacman: +# Propably it makes sense to move the following to the requirements.txt to just install it in the python venv - ansible - python-passlib - python-pytest diff --git a/roles/srv-proxy-6-6-domain/tasks/cleanup.yml b/roles/srv-proxy-6-6-domain/tasks/cleanup.yml new file mode 100644 index 00000000..a930c9ca --- /dev/null +++ b/roles/srv-proxy-6-6-domain/tasks/cleanup.yml @@ -0,0 +1,33 @@ +- name: "Lookup Cloudflare Zone ID for {{ domain }}" + vars: + cf_api_url: "https://api.cloudflare.com/client/v4/zones" + ansible.builtin.uri: + url: "{{ cf_api_url }}?name={{ domain | to_primary_domain }}" + method: GET + headers: + Authorization: "Bearer {{ certbot_dns_api_token }}" + Content-Type: "application/json" + return_content: yes + register: cf_zone_lookup + when: dns_provider == "cloudflare" + +- name: "Set fact cf_zone_id" + set_fact: + cf_zone_id: "{{ cf_zone_lookup.json.result[0].id }}" + when: + - dns_provider == "cloudflare" + - cf_zone_lookup.json.result | length > 0 + +- name: "Purge everything from Cloudflare cache for domain {{ domain }}" + ansible.builtin.uri: + url: "https://api.cloudflare.com/client/v4/zones/{{ cf_zone_id }}/purge_cache" + method: POST + headers: + Authorization: "Bearer {{ certbot_dns_api_token }}" + Content-Type: "application/json" + body: + purge_everything: true + body_format: json + return_content: yes + register: cf_purge + when: dns_provider == "cloudflare" diff --git a/roles/srv-proxy-6-6-domain/tasks/main.yml b/roles/srv-proxy-6-6-domain/tasks/main.yml index dd61e0ba..010dcabc 100644 --- a/roles/srv-proxy-6-6-domain/tasks/main.yml +++ b/roles/srv-proxy-6-6-domain/tasks/main.yml @@ -1,6 +1,9 @@ # run_once_srv_proxy_6_6_domain: deactivated +- name: Cleanup Domain + include_tasks: cleanup.yml + when: mode_cleanup | bool -- name: "include role for {{domain}} to receive certificates and do the modification routines" +- name: "include role for {{ domain }} to receive certificates and do the modification routines" include_role: name: srv-web-7-6-composer diff --git a/roles/srv-proxy-7-4-core/templates/location/README.md b/roles/srv-proxy-7-4-core/templates/location/README.md new file mode 100644 index 00000000..a1f7fb67 --- /dev/null +++ b/roles/srv-proxy-7-4-core/templates/location/README.md @@ -0,0 +1,58 @@ +# Nginx Location Templates + +This directory contains Jinja2 templates for different Nginx `location` blocks, each designed to proxy and optimize different types of web traffic. These templates are used by the `srv-proxy-7-4-core` role to modularize and standardize reverse proxy configuration across a wide variety of applications. + +--- + +## Overview of Files + +### `html.conf.j2` +- **Purpose:** + Handles "normal" web traffic such as HTML pages, API endpoints, and general HTTP(S) requests. +- **Features:** + - Proxies requests to the backend service. + - Optionally integrates with OAuth2 proxy for authentication. + - Sets all necessary proxy headers. + - Applies a Content Security Policy header. + - Activates buffering for advanced features such as Lua-based string replacements. + - Supports WebSocket upgrades for hybrid APIs. + +--- + +### `ws.conf.j2` +- **Purpose:** + Handles WebSocket connections, enabling real-time features such as live updates or chats. +- **Features:** + - Sets all headers required for WebSocket upgrades. + - Disables proxy buffering (required for WebSockets). + - Uses `tcp_nodelay` for low latency. + - Proxies traffic to the backend WebSocket server. + +--- + +### `media.conf.j2` +- **Purpose:** + Proxies and caches static media files (images, icons, etc.). +- **Features:** + - Matches image file extensions (jpg, png, gif, webp, ico, svg, etc.). + - Enables browser-side and proxy-side caching for efficient delivery. + - Adds cache control headers and exposes the upstream cache status. + +--- + +## Usage + +These templates are intended for inclusion in larger Nginx configuration files via Jinja2. +They modularize your configuration by separating HTML, WebSocket, and media proxying, allowing for clear, reusable, and maintainable reverse proxy logic. + +- Use `html.conf.j2` for standard application HTTP/S endpoints. +- Use `ws.conf.j2` for dedicated WebSocket endpoints. +- Use `media.conf.j2` for efficient handling of static media content. + +--- + +## Best Practices + +- Only enable WebSocket proxying (`ws.conf.j2`) for routes that actually require it, to avoid breaking buffering for standard HTTP. +- Activate media proxying (`media.conf.j2`) if your application benefits from image caching at the proxy layer. +- Keep templates modular for maintainability and scalability as your application grows. \ No newline at end of file diff --git a/roles/srv-proxy-7-4-core/templates/location/Todo.md b/roles/srv-proxy-7-4-core/templates/location/Todo.md new file mode 100644 index 00000000..5ccc0e38 --- /dev/null +++ b/roles/srv-proxy-7-4-core/templates/location/Todo.md @@ -0,0 +1,2 @@ +# TODOS +- ATM it seems like the media proxy isn't used. Propably it could make sense to activate it. -> Research it. \ No newline at end of file diff --git a/roles/srv-proxy-7-4-core/templates/location/proxy_basic.conf.j2 b/roles/srv-proxy-7-4-core/templates/location/html.conf.j2 similarity index 81% rename from roles/srv-proxy-7-4-core/templates/location/proxy_basic.conf.j2 rename to roles/srv-proxy-7-4-core/templates/location/html.conf.j2 index ad4b2b74..65fe950b 100644 --- a/roles/srv-proxy-7-4-core/templates/location/proxy_basic.conf.j2 +++ b/roles/srv-proxy-7-4-core/templates/location/html.conf.j2 @@ -21,13 +21,16 @@ location {{location | default("/")}} proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; - # deactivate buffering - proxy_buffering off; - proxy_request_buffering off; + # Activate buffering + # Needs to be enabled, so that lua can do str replaces + proxy_buffering on; + proxy_request_buffering on; # timeouts proxy_connect_timeout 1s; proxy_send_timeout 900s; proxy_read_timeout 900s; send_timeout 900s; + + {% include 'roles/srv-web-7-7-inj-compose/templates/location.lua.j2'%} } \ No newline at end of file diff --git a/roles/srv-proxy-7-4-core/templates/location/proxy_cache.conf.j2 b/roles/srv-proxy-7-4-core/templates/location/media.conf.j2 similarity index 100% rename from roles/srv-proxy-7-4-core/templates/location/proxy_cache.conf.j2 rename to roles/srv-proxy-7-4-core/templates/location/media.conf.j2 diff --git a/roles/srv-proxy-7-4-core/templates/location/ws.conf.j2 b/roles/srv-proxy-7-4-core/templates/location/ws.conf.j2 new file mode 100644 index 00000000..82a39286 --- /dev/null +++ b/roles/srv-proxy-7-4-core/templates/location/ws.conf.j2 @@ -0,0 +1,14 @@ +location {{ location_ws }} { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_pass http://127.0.0.1:{{ ws_port }}; + + # Proxy buffering needs to be disabled for websockets. + proxy_buffering off; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + tcp_nodelay on; +} \ No newline at end of file diff --git a/roles/srv-proxy-7-4-core/templates/vhost/README.md b/roles/srv-proxy-7-4-core/templates/vhost/README.md new file mode 100644 index 00000000..44d779ae --- /dev/null +++ b/roles/srv-proxy-7-4-core/templates/vhost/README.md @@ -0,0 +1,78 @@ +# Nginx vHost Templates: Basic vs. WebSocket (ws_generic) + +This directory provides two Nginx server templates for reverse proxying Dockerized applications behind Nginx: +- `basic.conf.j2` +- `ws_generic.conf.j2` + +--- + +## When to Use Which Template? + +### 1. `basic.conf.j2` +**Use this template for standard HTTP/S applications.** +It is optimized for typical web applications (e.g., static sites, PHP, Node.js, Django, etc.) that do **not** require persistent, bidirectional WebSocket connections. + +- **Features:** + - HTTP/2 support, TLS/SSL integration + - Reverse proxy with buffering enabled (`proxy_buffering on`) + - Allows advanced content filtering (e.g., via Lua body/headers) + - Suitable for most REST APIs, web frontends, and admin panels + +- **Pros:** + - Enables HTML/body manipulation (for injecting snippets, analytics, CSP, etc.) + - Optimized for efficient caching and GZIP compression + - Good default for "normal" web traffic + +- **Cons:** + - **Not** suitable for WebSocket endpoints (buffering can break WS) + - Slightly more latency for streaming data due to buffering + +--- + +### 2. `ws_generic.conf.j2` +**Use this template for applications requiring WebSocket support.** +Designed for services (e.g., chat servers, real-time dashboards) needing fast, persistent connections using the WebSocket protocol. + +- **Features:** + - WebSocket-aware: `proxy_buffering off`, special upgrade headers + - Supports standard HTTP/S traffic alongside WebSockets + - Proper handling of connection upgrades and protocol switching + +- **Pros:** + - Required for all WebSocket endpoints + - Allows instant, low-latency bidirectional traffic + - Prevents data loss or connection drops due to proxy buffering + +- **Cons:** + - Disables body/content filtering and response manipulation + - No buffering means less effective for caching/optimization + - Not suitable for scenarios requiring Lua/JS content injection + +--- + +## Summary Table + +| Use Case | Template | Buffering | WebSocket? | Can Filter Content? | +|--------------------------|---------------------|-----------|------------|--------------------| +| Static/Classic Website | `basic.conf.j2` | On | No | Yes | +| REST API | `basic.conf.j2` | On | No | Yes | +| Real-Time Chat/App | `ws_generic.conf.j2`| Off | Yes | No | +| Dashboard w/Live Data | `ws_generic.conf.j2`| Off | Yes | No | +| Needs HTML Injection | `basic.conf.j2` | On | No | Yes | + +--- + +## Good to Know + +- **Never enable buffering for true WebSocket connections!** + Use `proxy_buffering off;` (as in `ws_generic.conf.j2`) or connections may fail. +- For most classic web applications, use the **basic template**. +- For apps where you want to inject or modify HTML (e.g., analytics scripts), **only the basic template** supports this. + +--- + +## Author & Project + +By [Kevin Veen-Birkenbach](https://www.veen.world) +Part of the [CyMaIS Project](https://s.veen.world/cymais) +Licensed under the [CyMaIS NonCommercial License (CNCL)](https://s.veen.world/cncl) diff --git a/roles/srv-proxy-7-4-core/templates/vhost/basic.conf.j2 b/roles/srv-proxy-7-4-core/templates/vhost/basic.conf.j2 index 5dc84f51..df65ba38 100644 --- a/roles/srv-proxy-7-4-core/templates/vhost/basic.conf.j2 +++ b/roles/srv-proxy-7-4-core/templates/vhost/basic.conf.j2 @@ -6,7 +6,7 @@ server {% include 'roles/web-app-oauth2-proxy/templates/endpoint.conf.j2'%} {% endif %} - {% include 'roles/srv-web-7-7-inj-compose/templates/global.includes.lua.j2'%} + {% include 'roles/srv-web-7-7-inj-compose/templates/server.conf.j2'%} {% if proxy_extra_configuration is defined %} {# Additional Domain Specific Configuration #} @@ -15,9 +15,6 @@ server {% include 'roles/srv-web-7-7-letsencrypt/templates/ssl_header.j2' %} - {% if applications | get_app_conf(application_id, 'features.logout', False) or domain == primary_domain %} - {% include 'roles/web-svc-logout/templates/logout-proxy.conf.j2' %} - {% endif %} {% if applications | get_app_conf(application_id, 'features.oauth2', False) %} {% set acl = applications | get_app_conf(application_id, 'oauth2_proxy.acl', False, {}) %} @@ -25,38 +22,38 @@ server {# 1. Expose everything by default, then protect blacklisted paths #} {% set oauth2_proxy_enabled = false %} {% set location = "/" %} - {% include 'roles/srv-proxy-7-4-core/templates/location/proxy_basic.conf.j2' %} + {% include 'roles/srv-proxy-7-4-core/templates/location/html.conf.j2' %} {% for loc in acl.blacklist %} {% set oauth2_proxy_enabled = true %} {% set location = loc %} - {% include 'roles/srv-proxy-7-4-core/templates/location/proxy_basic.conf.j2' %} + {% include 'roles/srv-proxy-7-4-core/templates/location/html.conf.j2' %} {% endfor %} {% elif acl.whitelist is defined %} {# 2. Protect everything by default, then expose whitelisted paths #} {% set oauth2_proxy_enabled = true %} {% set location = "/" %} - {% include 'roles/srv-proxy-7-4-core/templates/location/proxy_basic.conf.j2' %} + {% include 'roles/srv-proxy-7-4-core/templates/location/html.conf.j2' %} {% for loc in acl.whitelist %} {% set oauth2_proxy_enabled = false %} {% set location = loc %} - {% include 'roles/srv-proxy-7-4-core/templates/location/proxy_basic.conf.j2' %} + {% include 'roles/srv-proxy-7-4-core/templates/location/html.conf.j2' %} {% endfor %} {% else %} {# 3. OAuth2 enabled but no (or empty) ACL — protect all #} {% set oauth2_proxy_enabled = true %} {% set location = "/" %} - {% include 'roles/srv-proxy-7-4-core/templates/location/proxy_basic.conf.j2' %} + {% include 'roles/srv-proxy-7-4-core/templates/location/html.conf.j2' %} {% endif %} {% else %} {# 4. OAuth2 completely disabled — expose all #} {% set oauth2_proxy_enabled = false %} {% set location = "/" %} - {% include 'roles/srv-proxy-7-4-core/templates/location/proxy_basic.conf.j2' %} + {% include 'roles/srv-proxy-7-4-core/templates/location/html.conf.j2' %} {% endif %} } diff --git a/roles/srv-proxy-7-4-core/templates/vhost/ws_generic.conf.j2 b/roles/srv-proxy-7-4-core/templates/vhost/ws_generic.conf.j2 index 2a6c0bda..4dccbded 100644 --- a/roles/srv-proxy-7-4-core/templates/vhost/ws_generic.conf.j2 +++ b/roles/srv-proxy-7-4-core/templates/vhost/ws_generic.conf.j2 @@ -7,7 +7,8 @@ server { server_name {{ domain }}; {% include 'roles/srv-web-7-7-letsencrypt/templates/ssl_header.j2' %} - {% include 'roles/srv-web-7-7-inj-compose/templates/global.includes.lua.j2' %} + + {% include 'roles/srv-web-7-7-inj-compose/templates/server.conf.j2' %} client_max_body_size {{ client_max_body_size | default('100m') }}; keepalive_timeout 70; @@ -24,26 +25,10 @@ server { add_header Strict-Transport-Security "max-age=31536000"; - {% include 'roles/srv-proxy-7-4-core/templates/location/proxy_basic.conf.j2' %} + {% include 'roles/srv-proxy-7-4-core/templates/location/html.conf.j2' %} - {% if applications | get_app_conf(application_id, 'features.logout', False) or domain == primary_domain %} - {% include 'roles/web-svc-logout/templates/logout-proxy.conf.j2' %} - {% endif %} - - {% if ws_path is defined %} - location {{ ws_path }} { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - - proxy_pass http://127.0.0.1:{{ ws_port }}; - proxy_buffering off; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - tcp_nodelay on; - } + {% if location_ws is defined %} + {% include 'roles/srv-proxy-7-4-core/templates/location/ws.conf.j2' %} {% endif %} error_page 500 501 502 503 504 /500.html; diff --git a/roles/srv-web-7-4-core/templates/nginx.conf.j2 b/roles/srv-web-7-4-core/templates/nginx.conf.j2 index dfb9bc0b..5b30e63d 100644 --- a/roles/srv-web-7-4-core/templates/nginx.conf.j2 +++ b/roles/srv-web-7-4-core/templates/nginx.conf.j2 @@ -8,6 +8,9 @@ events http { include mime.types; + + {# default_type application/octet-stream; If html filter does not work, this one needs to be used#} + default_type text/html; {# caching #} diff --git a/roles/srv-web-7-7-dns-records/tasks/main.yml b/roles/srv-web-7-7-dns-records/tasks/main.yml index 04bb38f4..1ae2674f 100644 --- a/roles/srv-web-7-7-dns-records/tasks/main.yml +++ b/roles/srv-web-7-7-dns-records/tasks/main.yml @@ -2,7 +2,7 @@ - name: Create or update Cloudflare A-record for {{ item }} community.general.cloudflare_dns: - api_token: "{{ cloudflare_api_token }}" + api_token: "{{ certbot_dns_api_token }}" zone: "{{ item.split('.')[-2:] | join('.') }}" state: present type: A diff --git a/roles/srv-web-7-7-inj-compose/templates/global.includes.lua.j2 b/roles/srv-web-7-7-inj-compose/templates/location.lua.j2 similarity index 54% rename from roles/srv-web-7-7-inj-compose/templates/global.includes.lua.j2 rename to roles/srv-web-7-7-inj-compose/templates/location.lua.j2 index 55ae147b..8d758782 100644 --- a/roles/srv-web-7-7-inj-compose/templates/global.includes.lua.j2 +++ b/roles/srv-web-7-7-inj-compose/templates/location.lua.j2 @@ -1,12 +1,22 @@ -{% set modifier_css_enabled = applications | get_app_conf(application_id, 'features.css', false) | bool %} -{% if modifier_css_enabled %} -{%- include 'roles/srv-web-7-7-inj-css/templates/location.conf.j2' -%} -{% endif %} lua_need_request_body on; +header_filter_by_lua_block { + local ct = ngx.header.content_type or "" + if ct:lower():find("^text/html") then + ngx.ctx.is_html = true + else + ngx.ctx.is_html = false + end +} + body_filter_by_lua_block { - -- initialize buffer + -- only apply further processing if this is an HTML response + if not ngx.ctx.is_html then + return + end + + -- initialize or reuse the buffer ngx.ctx.buf = ngx.ctx.buf or {} local chunk, eof = ngx.arg[1], ngx.arg[2] @@ -15,18 +25,22 @@ body_filter_by_lua_block { end if not eof then + -- drop intermediate chunks; we’ll emit only on eof ngx.arg[1] = nil return end - -- on eof: concatenate and reset buffer + -- on eof: concatenate all buffered chunks local whole = table.concat(ngx.ctx.buf) - ngx.ctx.buf = nil + ngx.ctx.buf = nil -- clear buffer +{# whole = string.gsub(whole, "", "\n") + ngx.arg[1] = whole #} - -- build head-injection snippets + -- build a list of head-injection snippets local head_snippets = {} - {% for head_feature in ['css', 'matomo', 'port-ui-desktop', 'javascript', 'logout'] %} +{# Deactivated 'logout' temporary due to chunk size. Needs an CDN. #} + {% for head_feature in ['css', 'matomo', 'port-ui-desktop', 'javascript' ] %} {% if applications | get_app_conf(application_id, 'features.' ~ head_feature, false) | bool %} head_snippets[#head_snippets + 1] = [=[ {%- include "roles/srv-web-7-7-inj-" ~ head_feature ~ "/templates/head_sub.j2" -%} @@ -34,19 +48,19 @@ body_filter_by_lua_block { {% endif %} {% endfor %} - -- inject into + -- inject all collected snippets right before local head_payload = table.concat(head_snippets, "\n") .. "" whole = string.gsub(whole, "", head_payload) {% if applications | get_app_conf(application_id, 'features.matomo', false) | bool %} - -- build Matomo noscript tracking for body + -- build Matomo noscript snippet for the body local body_matomo = [=[ {%- include 'roles/srv-web-7-7-inj-matomo/templates/body_sub.j2' -%} ]=] - - -- inject before + -- inject it right before whole = string.gsub(whole, "", body_matomo) {% endif %} + -- finally send the modified HTML out ngx.arg[1] = whole -} +} \ No newline at end of file diff --git a/roles/srv-web-7-7-inj-compose/templates/server.conf.j2 b/roles/srv-web-7-7-inj-compose/templates/server.conf.j2 new file mode 100644 index 00000000..1a223ed5 --- /dev/null +++ b/roles/srv-web-7-7-inj-compose/templates/server.conf.j2 @@ -0,0 +1,9 @@ +{% set modifier_css_enabled = applications | get_app_conf(application_id, 'features.css', false) | bool %} +{% if modifier_css_enabled %} +{%- include 'roles/srv-web-7-7-inj-css/templates/location.conf.j2' -%} +{% endif %} + +{% set modifier_logout_enabled = applications | get_app_conf(application_id, 'features.logout', False) or domain == primary_domain %} +{% if modifier_logout_enabled %} +{% include 'roles/web-svc-logout/templates/logout-proxy.conf.j2' %} +{% endif %} \ No newline at end of file diff --git a/roles/srv-web-7-7-inj-logout/templates/logout.js.j2 b/roles/srv-web-7-7-inj-logout/templates/logout.js.j2 index 7e8e613c..2a84e13d 100644 --- a/roles/srv-web-7-7-inj-logout/templates/logout.js.j2 +++ b/roles/srv-web-7-7-inj-logout/templates/logout.js.j2 @@ -85,7 +85,7 @@ // Initial scan scanAndPatch(document.querySelectorAll('*')); - +{# // MutationObserver for dynamic content const observer = new MutationObserver(mutations => { mutations.forEach(mutation => { @@ -97,4 +97,5 @@ }); observer.observe(document.body, { childList: true, subtree: true }); +#} })(); diff --git a/roles/svc-prx-openresty/docs/CACHE.md b/roles/svc-prx-openresty/docs/CACHE.md new file mode 100644 index 00000000..b504eabf --- /dev/null +++ b/roles/svc-prx-openresty/docs/CACHE.md @@ -0,0 +1,63 @@ +# Caching in OpenResty and Cloudflare + +## Overview + +When deploying OpenResty as a reverse proxy, content may be cached at multiple layers: + +- **Local Proxy Cache:** If you configure Nginx/OpenResty with a cache zone (using directives like `proxy_cache`), responses can be stored locally on disk and served to future clients. +- **Browser Cache:** Browsers cache responses based on HTTP headers like `Cache-Control` or `Expires`. +- **CDN Cache (Cloudflare):** If your domain is proxied through Cloudflare, Cloudflare may cache your content at their edge servers and serve it from there, often without requests reaching your origin server. + +## Troubleshooting Cache Issues + +Caching can cause problems, especially when you update your web content or configuration but still receive outdated responses. Typical symptoms include: + +- Changes to HTML/CSS/JS are not visible immediately. +- Old redirects or headers persist even after config changes. +- Assets do not update after a deployment. + +If this happens, always consider all caching layers: +1. **Browser:** Clear the browser cache or use a private window. +2. **OpenResty:** If using a proxy cache, purge or clear the cache directory, or temporarily disable the cache zone. +3. **Cloudflare:** Purge the CDN cache as described below. + +## Purging Cloudflare Cache + +Cloudflare aggressively caches static content by default. Even after you deploy new files or update your proxy configuration, users may continue to see cached versions until the cache expires. + +### Manual Purge via Cloudflare Dashboard + +1. Log into your Cloudflare dashboard at [https://dash.cloudflare.com](https://dash.cloudflare.com). +2. Select your domain. +3. Go to **Caching** → **Configuration**. +4. Click **Purge Cache**. + - Choose **Purge Everything** to delete all cached files (recommended after a deployment). + - Or use **Custom Purge** to specify individual URLs. + +### Purge with Cloudflare API + +You can also purge the cache programmatically with Cloudflare’s API: + +```bash +curl -X POST "https://api.cloudflare.com/client/v4/zones//purge_cache" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + --data '{"purge_everything":true}' +```` + +* Replace `` with your Cloudflare Zone ID. +* Replace `` with a valid API token with cache purge permissions. + +To find your Zone ID, go to the overview page for your domain in the Cloudflare dashboard. +**Note:** It can take a few seconds for the cache to be purged globally. + +## Recommendations + +* Always purge the Cloudflare cache after significant changes to your website or OpenResty/Nginx configuration. +* If you use custom cache rules in OpenResty, consider providing cache-busting mechanisms (e.g., versioned URLs). +* Test changes in a private/incognito window to rule out browser cache. + +## Further Reading + +* [Cloudflare Purge Cache Documentation](https://developers.cloudflare.com/cache/how-to/purge-cache/) +* [Nginx/OpenResty Proxy Cache Guide](https://openresty.org/en/using-ngx_lua.html#caching) diff --git a/roles/web-app-collabora/templates/nginx.conf.j2 b/roles/web-app-collabora/templates/nginx.conf.j2 index 5c38e3ba..9fea319f 100644 --- a/roles/web-app-collabora/templates/nginx.conf.j2 +++ b/roles/web-app-collabora/templates/nginx.conf.j2 @@ -3,13 +3,13 @@ server { {% include 'roles/srv-web-7-7-letsencrypt/templates/ssl_header.j2' %} - {% include 'roles/srv-web-7-7-inj-compose/templates/global.includes.lua.j2'%} + {% include 'roles/srv-web-7-7-inj-compose/templates/server.conf.j2'%} {% include 'roles/srv-proxy-7-4-core/templates/headers/content_security_policy.conf.j2' %} - {% include 'roles/srv-proxy-7-4-core/templates/location/proxy_basic.conf.j2' %} + {% include 'roles/srv-proxy-7-4-core/templates/location/html.conf.j2' %} {% set location = '^~ /cool/' %} - {% include 'roles/srv-proxy-7-4-core/templates/location/proxy_basic.conf.j2' %} + {% include 'roles/srv-proxy-7-4-core/templates/location/html.conf.j2' %} } \ No newline at end of file diff --git a/roles/web-app-espocrm/vars/main.yml b/roles/web-app-espocrm/vars/main.yml index 6e4b47e6..965d00e4 100644 --- a/roles/web-app-espocrm/vars/main.yml +++ b/roles/web-app-espocrm/vars/main.yml @@ -1,6 +1,6 @@ application_id: "web-app-espocrm" database_type: "mariadb" -ws_path: "/ws" +location_ws: "/ws" ws_port: "{{ ports.localhost.websocket[application_id] }}" client_max_body_size: "100m" vhost_flavour: "ws_generic" diff --git a/roles/web-app-mastodon/tasks/main.yml b/roles/web-app-mastodon/tasks/main.yml index 1ba7c779..dbdbe331 100644 --- a/roles/web-app-mastodon/tasks/main.yml +++ b/roles/web-app-mastodon/tasks/main.yml @@ -11,7 +11,7 @@ loop_var: domain vars: http_port: "{{ ports.localhost.http[application_id] }}" - ws_path: "/api/v1/streaming" + location_ws: "/api/v1/streaming" ws_port: "{{ ports.localhost.websocket[application_id] }}" client_max_body_size: "80m" vhost_flavour: "ws_generic" diff --git a/roles/web-app-matrix/templates/nginx.conf.j2 b/roles/web-app-matrix/templates/nginx.conf.j2 index 76537ebb..01cc60c1 100644 --- a/roles/web-app-matrix/templates/nginx.conf.j2 +++ b/roles/web-app-matrix/templates/nginx.conf.j2 @@ -3,19 +3,16 @@ server { {# Could be that this is related to the set_fact use #} {% set domain = domains[application_id].synapse %} {% set http_port = ports.localhost.http['web-app-matrix_synapse'] %} + {% set federation_port = ports.public.federation['web-app-matrix_synapse'] %} server_name {{domains[application_id].synapse}}; {% include 'roles/srv-web-7-7-letsencrypt/templates/ssl_header.j2' %} # For the federation port - listen 8448 ssl default_server; - listen [::]:8448 ssl default_server; + listen {{ federation_port }} ssl default_server; + listen [::]:{{ federation_port }} ssl default_server; - {% include 'roles/srv-web-7-7-inj-compose/templates/global.includes.lua.j2'%} - {% include 'roles/srv-proxy-7-4-core/templates/location/proxy_basic.conf.j2' %} - - {% if applications | get_app_conf(application_id, 'features.logout', False) %} - {% include 'roles/web-svc-logout/templates/logout-proxy.conf.j2' %} - {% endif %} + {% include 'roles/srv-web-7-7-inj-compose/templates/server.conf.j2'%} + {% include 'roles/srv-proxy-7-4-core/templates/location/html.conf.j2' %} } \ No newline at end of file diff --git a/roles/web-app-nextcloud/config/main.yml b/roles/web-app-nextcloud/config/main.yml index ca6aa10a..4f9b0c24 100644 --- a/roles/web-app-nextcloud/config/main.yml +++ b/roles/web-app-nextcloud/config/main.yml @@ -69,6 +69,9 @@ performance: memory_limit: "{{ ((ansible_memtotal_mb | int) / 30)|int }}M" # Dynamic set memory limit upload_limit: "5G" # Set upload limit to 5GB for big media files opcache_memory_consumption: "{{ ((ansible_memtotal_mb | int) / 30)|int }}M" # Dynamic set memory consumption + +plugins_enabled: true # Implemented for speeding up testing and debugging process. For productive environments keep it true and steer the apps via the plugins config + plugins: # List for Nextcloud Plugin Routine # Decides if plugins should be activated or deactivated diff --git a/roles/web-app-nextcloud/tasks/main.yml b/roles/web-app-nextcloud/tasks/main.yml index 3661ddc8..8313134c 100644 --- a/roles/web-app-nextcloud/tasks/main.yml +++ b/roles/web-app-nextcloud/tasks/main.yml @@ -49,6 +49,7 @@ vars: plugin_key: "{{ plugin_item.key }}" plugin_value: "{{ plugin_item.value }}" + when: nextcloud_plugins_enabled - name: Load system configuration include_tasks: 03_system.yml diff --git a/roles/web-app-nextcloud/templates/env.j2 b/roles/web-app-nextcloud/templates/env.j2 index b3a0b783..519fae77 100644 --- a/roles/web-app-nextcloud/templates/env.j2 +++ b/roles/web-app-nextcloud/templates/env.j2 @@ -29,7 +29,7 @@ NEXTCLOUD_ADMIN_PASSWORD= "{{applications | get_app_conf(application_id, ' # Security -NEXTCLOUD_TRUSTED_DOMAINS= "{{ nextcloud_domains }}" +NEXTCLOUD_TRUSTED_DOMAINS= "{{ domains[application_id] | select | join(',') }}" # Whitelist local docker gateway in Nextcloud to prevent brute-force throtteling TRUSTED_PROXIES= "{{ networks.internet.values() | select | join(',') }}" OVERWRITECLIURL= "{{ domains | get_url(application_id, web_protocol) }}" diff --git a/roles/web-app-nextcloud/templates/nginx/host.conf.j2 b/roles/web-app-nextcloud/templates/nginx/host.conf.j2 index 4168791f..29ca7207 100644 --- a/roles/web-app-nextcloud/templates/nginx/host.conf.j2 +++ b/roles/web-app-nextcloud/templates/nginx/host.conf.j2 @@ -6,7 +6,8 @@ server {% include 'roles/srv-web-7-7-letsencrypt/templates/ssl_header.j2' %} - {% include 'roles/srv-web-7-7-inj-compose/templates/global.includes.lua.j2'%} + {% include 'roles/srv-web-7-7-inj-compose/templates/server.conf.j2'%} + # Remove X-Powered-By, which is an information leak fastcgi_hide_header X-Powered-By; @@ -18,11 +19,7 @@ server client_body_buffer_size 400M; fastcgi_buffers 64 4K; - {% include 'roles/srv-proxy-7-4-core/templates/location/proxy_basic.conf.j2' %} - - {% if applications | get_app_conf(application_id, 'features.logout', False) %} - {% include 'roles/web-svc-logout/templates/logout-proxy.conf.j2' %} - {% endif %} + {% include 'roles/srv-proxy-7-4-core/templates/location/html.conf.j2' %} location ^~ /.well-known { rewrite ^/\.well-known/host-meta\.json /public.php?service=host-meta-json last; diff --git a/roles/web-app-nextcloud/vars/main.yml b/roles/web-app-nextcloud/vars/main.yml index 5cc66e5b..1ebfef67 100644 --- a/roles/web-app-nextcloud/vars/main.yml +++ b/roles/web-app-nextcloud/vars/main.yml @@ -6,6 +6,7 @@ container_port: 80 # Database database_password: "{{ applications | get_app_conf(application_id, 'credentials.database_password', True)}}" database_type: "mariadb" # Database flavor +nextcloud_plugins_enabled: "{{ applications | get_app_conf(application_id, 'plugins_enabled', True) }}" # Networking domain: "{{ domains | get_domain(application_id) }}" # Public domain at which Nextcloud will be accessable @@ -23,15 +24,13 @@ nextcloud_control_node_plugin_tasks_directory: "{{role_path}}/tasks/plugins/" nextcloud_host_config_additives_directory: "{{ docker_compose.directories.volumes }}cymais/" # This folder is the path to which the additive configurations will be copied nextcloud_host_include_instructions_file: "{{ docker_compose.directories.volumes }}includes.php" # Path to the instruction file on the host. Responsible for loading the additional configurations -nextcloud_domains: "{{ domains | get_domain(application_id) }}" # This is wrong and should be optimized @todo implement support for multiple domains - # Docker nextcloud_volume: "{{ applications | get_app_conf(application_id, 'docker.volumes.data', True) }}" nextcloud_version: "{{ applications | get_app_conf(application_id, 'docker.services.nextcloud.version', True) }}" nextcloud_image: "{{ applications | get_app_conf(application_id, 'docker.services.nextcloud.image', True) }}" -nextcloud_container: "{{ applications | get_app_conf(application_id, 'docker.services.nextcloud.name', True) }}" +nextcloud_container: "{{ applications | get_app_conf(application_id, 'docker.services.nextcloud.name', True) }}" nextcloud_proxy_name: "{{ applications | get_app_conf(application_id, 'docker.services.proxy.name', True) }}" nextcloud_proxy_image: "{{ applications | get_app_conf(application_id, 'docker.services.proxy.image', True) }}" diff --git a/roles/web-app-peertube/templates/peertube.conf.j2 b/roles/web-app-peertube/templates/peertube.conf.j2 index 3abe1971..fac278e1 100644 --- a/roles/web-app-peertube/templates/peertube.conf.j2 +++ b/roles/web-app-peertube/templates/peertube.conf.j2 @@ -3,7 +3,7 @@ server { {% include 'roles/srv-web-7-7-letsencrypt/templates/ssl_header.j2' %} - {% include 'roles/srv-web-7-7-inj-compose/templates/global.includes.lua.j2'%} + {% include 'roles/srv-web-7-7-inj-compose/templates/server.conf.j2'%} {% include 'roles/srv-proxy-7-4-core/templates/headers/content_security_policy.conf.j2' %} @@ -11,35 +11,18 @@ server { # Application ## - location @api { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - - client_max_body_size 100k; # default is 1M - - proxy_connect_timeout 10m; - proxy_send_timeout 10m; - proxy_read_timeout 10m; - send_timeout 10m; - - #adapt - proxy_pass http://127.0.0.1:{{ports.localhost.http[application_id]}}; - } - - {% if applications | get_app_conf(application_id, 'features.logout', False) %} - {% include 'roles/web-svc-logout/templates/logout-proxy.conf.j2' %} - {% endif %} + {% set location = "@html" %} + {% include 'roles/srv-proxy-7-4-core/templates/location/html.conf.j2' %} location / { - try_files /dev/null @api; + try_files /dev/null {{ location }}; } location = /api/v1/videos/upload-resumable { client_max_body_size 0; proxy_request_buffering off; - try_files /dev/null @api; + try_files /dev/null {{ location }}; } location ~ ^/api/v1/videos/(upload|([^/]+/studio/edit))$ { @@ -47,33 +30,25 @@ server { client_max_body_size 12G; # default is 1M add_header X-File-Maximum-Size 8G always; # inform backend of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size) - try_files /dev/null @api; + try_files /dev/null {{ location }}; } location ~ ^/api/v1/(videos|video-playlists|video-channels|users/me) { client_max_body_size 6M; # default is 1M add_header X-File-Maximum-Size 4M always; # inform backend of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size) - try_files /dev/null @api; + try_files /dev/null {{ location }}; } ## # Websocket ## - location @api_websocket { - proxy_http_version 1.1; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - - proxy_pass http://127.0.0.1:{{ports.localhost.http[application_id]}}; - } + {% set location_ws = "@websocket" %} + {% include 'roles/srv-proxy-7-4-core/templates/location/ws.conf.j2' %} location /socket.io { - try_files /dev/null @api_websocket; + try_files /dev/null {{ location_ws }}; } location /tracker/socket { @@ -81,6 +56,6 @@ server { # Don't close the websocket before then proxy_read_timeout 15m; # default is 60s - try_files /dev/null @api_websocket; + try_files /dev/null {{ location_ws }}; } } \ No newline at end of file diff --git a/roles/web-app-syncope/templates/proxy.conf b/roles/web-app-syncope/templates/proxy.conf index 522ab428..d12cd18b 100644 --- a/roles/web-app-syncope/templates/proxy.conf +++ b/roles/web-app-syncope/templates/proxy.conf @@ -6,7 +6,7 @@ server {% include 'roles/web-app-oauth2-proxy/templates/endpoint.conf.j2'%} {% endif %} - {% include 'roles/srv-web-7-7-inj-compose/templates/global.includes.lua.j2'%} + {% include 'roles/srv-web-7-7-inj-compose/templates/server.conf.j2'%} {% if proxy_extra_configuration is defined %} {# Additional Domain Specific Configuration #} @@ -17,6 +17,6 @@ server {% for path in syncope_paths.values() %} {% set location = web_protocol ~ '://' ~ domains | get_domain(application_id) ~ '/' ~ path ~ '/' %} - {% include 'roles/srv-proxy-7-4-core/templates/location/proxy_basic.conf.j2'%} + {% include 'roles/srv-proxy-7-4-core/templates/location/html.conf.j2'%} {% endfor %} } \ No newline at end of file diff --git a/roles/web-svc-file/templates/nginx.conf.j2 b/roles/web-svc-file/templates/nginx.conf.j2 index 1dd6c11d..11df2a2a 100644 --- a/roles/web-svc-file/templates/nginx.conf.j2 +++ b/roles/web-svc-file/templates/nginx.conf.j2 @@ -4,7 +4,7 @@ server {% include 'roles/srv-web-7-7-letsencrypt/templates/ssl_header.j2' %} - {% include 'roles/srv-web-7-7-inj-compose/templates/global.includes.lua.j2'%} + {% include 'roles/srv-web-7-7-inj-compose/templates/server.conf.j2'%} {% include 'roles/srv-proxy-7-4-core/templates/headers/content_security_policy.conf.j2' %} charset utf-8; @@ -15,6 +15,7 @@ server autoindex on; {# Enable directory listing #} autoindex_exact_size off; {# Display sizes in a human-readable format #} autoindex_localtime on; {# Show local time #} + {% include 'roles/srv-web-7-7-inj-compose/templates/location.lua.j2' %} } location /.well-known/ { diff --git a/roles/web-svc-html/templates/nginx.conf.j2 b/roles/web-svc-html/templates/nginx.conf.j2 index 311e8485..bbc93394 100644 --- a/roles/web-svc-html/templates/nginx.conf.j2 +++ b/roles/web-svc-html/templates/nginx.conf.j2 @@ -4,7 +4,7 @@ server {% include 'roles/srv-web-7-7-letsencrypt/templates/ssl_header.j2' %} - {% include 'roles/srv-web-7-7-inj-compose/templates/global.includes.lua.j2'%} + {% include 'roles/srv-web-7-7-inj-compose/templates/server.conf.j2'%} {% include 'roles/srv-proxy-7-4-core/templates/headers/content_security_policy.conf.j2' %} charset utf-8; @@ -13,6 +13,7 @@ server { root {{nginx.directories.data.html}}; index index.html index.htm; + {% include 'roles/srv-web-7-7-inj-compose/templates/location.lua.j2' %} } location /.well-known/ { diff --git a/tests/unit/filter_plugins/test_to_primary_domain.py b/tests/unit/filter_plugins/test_to_primary_domain.py new file mode 100644 index 00000000..0554b9cb --- /dev/null +++ b/tests/unit/filter_plugins/test_to_primary_domain.py @@ -0,0 +1,35 @@ +import unittest +from ansible.errors import AnsibleFilterError + +# Import the filter plugin +from filter_plugins.to_primary_domain import FilterModule + +class TestToPrimaryDomain(unittest.TestCase): + def setUp(self): + self.filtermod = FilterModule() + + def test_valid_domains(self): + cases = [ + ("example.com", "example.com"), + ("www.example.com", "example.com"), + ("foo.bar.example.com", "example.com"), + ("mail.test.example.co.uk", "example.co.uk"), + ("test.foo.bar.jp", "bar.jp"), + ] + for input_domain, expected in cases: + with self.subTest(domain=input_domain): + self.assertEqual(self.filtermod.to_primary_domain(input_domain), expected) + + def test_invalid_domains(self): + invalid_cases = [ + "localhost", # not a real domain + 1234, # not a string + "", # empty string + ] + for input_domain in invalid_cases: + with self.subTest(domain=input_domain): + with self.assertRaises(AnsibleFilterError): + self.filtermod.to_primary_domain(input_domain) + +if __name__ == "__main__": + unittest.main()