Solved proxy cache bugs

group_vars/all/05_nginx.yml

@@ -14,5 +14,8 @@ nginx:
       html:           "/var/www/public_html/"           # Path where the static homepage files are stored
       files:          "/var/www/public_files/"          # Path where the web accessable files are stored
       global:         "/var/www/global/"                # Directory containing files which will be globaly accessable
+    cache:
+      general:        "/tmp/cache_nginx_general/"       # Directory which nginx uses to cache general data
+      image:          "/tmp/cache_nginx_image/"         # Directory which nginx uses to cache images
   user:               "http"                            # Default nginx user in ArchLinux
   iframe:             true                              # Allows applications to be loaded in iframe

roles/cleanup-certs/meta/main.yml

@@ -24,5 +24,4 @@ galaxy_info:
   documentation: "https://github.com/kevinveenbirkenbach/certreap#readme"
 
 dependencies:
-  - systemd-timer
   - systemd-notifier

roles/cleanup-certs/tasks/main.yml

@@ -2,7 +2,7 @@
   include_role:
     name: pkgmgr-install
   vars:
-    package_name: cleanup-certs 
+    package_name: certreap
   when: run_once_cleanup_certs is not defined
 
 - name: configure cleanup-certs.cymais.service

roles/cleanup-certs/templates/cleanup-certs.service.j2

@@ -4,4 +4,4 @@ OnFailure=systemd-notifier.cymais@%n.service
 
 [Service]
 Type=oneshot
-ExecStartPre=/bin/sh -c '/usr/bin/python certreap --force'
+ExecStart=/bin/sh -c 'certreap --force'

roles/cleanup-domains/README.md

@@ -1,4 +1,4 @@
-# nginx-domains-cleanup
+# cleanup-domains
 
 ## Description
 

roles/docker-keycloak/templates/import/realm.json.j2

@@ -2666,6 +2666,17 @@
           "autheticatorFlow": false,
           "userSetupAllowed": false
         },
+{%- if applications | is_feature_enabled('recaptcha', application_id) %}
+        {
+          "authenticatorConfig": "Google reCaptcha",
+          "authenticator": "registration-recaptcha-action",
+          "authenticatorFlow": false,
+          "requirement": "REQUIRED",
+          "priority": 60,
+          "autheticatorFlow": false,
+          "userSetupAllowed": false
+        },
+{%- else %}
         {
           "authenticator": "registration-recaptcha-action",
           "authenticatorFlow": false,
@@ -2674,6 +2685,7 @@
           "autheticatorFlow": false,
           "userSetupAllowed": false
         },
+{%- endif %}
         {
           "authenticator": "registration-terms-and-conditions",
           "authenticatorFlow": false,
@@ -2746,6 +2758,19 @@
     }
   ],
   "authenticatorConfig": [
+{%- if applications | is_feature_enabled('recaptcha',application_id) %}
+    {
+      "id": "c6dcf381-7e39-4f7f-8d1f-631faec31b56",
+      "alias": "Google reCaptcha",
+      "config": {
+        "action": "register",
+        "useRecaptchaNet": "false",
+        "recaptcha.v3": "true",
+        "secret.key": "{{ applications[application_id].credentials.recaptcha.secret_key }}",
+        "site.key": "{{ applications[application_id].credentials.recaptcha.website_key }}"
+      }
+    },
+{%- endif %}
     {
       "id": "3e40f95e-d9a7-405d-b393-398bfc54c2e8",
       "alias": "create unique user config",

roles/docker-mariadb/tasks/main.yml

@@ -36,10 +36,11 @@
     state: present
   when: run_once_docker_mariadb is not defined
 
-- name: Wait for MariaDB inside the container to respond
-  shell: docker exec central-mariadb mysqladmin ping -h localhost --silent
-  register: mysql_ping
-  until: mysql_ping.rc == 0
+- name: Wait until the MariaDB container is healthy
+  community.docker.docker_container_info:
+    name: central-mariadb
+  register: db_info
+  until: db_info.containers[0].State.Health.Status == "healthy"
   retries: 30
   delay: 5
   when:

roles/docker-nextcloud/tasks/plugin.yml

@@ -16,46 +16,59 @@
   when: not (plugin_value.enabled | bool)
 
 - name: install {{ plugin_key }} nextcloud plugin
-  command: "{{nextcloud_docker_exec_occ}} app:install {{ plugin_key }}"
+  command: "{{ nextcloud_docker_exec_occ }} app:install {{ plugin_key }}"
   register: install_result
-  failed_when: install_result.rc != 0 and ("already installed" not in install_result.stdout)
-  changed_when: install_result.rc == 0 and ("already installed" not in install_result.stdout)
+  failed_when: >
+    install_result.rc != 0
+    and
+    ("already installed" not in install_result.stdout)
+    and
+    ("not compatible with this version of the server" not in install_result.stdout)
+  changed_when: >
+    install_result.rc == 0
+    and
+    ("already installed" not in install_result.stdout)
   when: plugin_value.enabled | bool
 
-- name: enable {{plugin_key}} nextcloud plugin
-  command: "{{nextcloud_docker_exec_occ}} app:enable {{plugin_key}}"
-  register: enable_result
-  changed_when: enable_result.rc == 0 and ("already enabled" not in enable_result.stdout)
-  when: plugin_value.enabled | bool
+- block:
+  - name: enable {{plugin_key}} nextcloud plugin
+    command: "{{nextcloud_docker_exec_occ}} app:enable {{plugin_key}}"
+    register: enable_result
+    changed_when: enable_result.rc == 0 and ("already enabled" not in enable_result.stdout)
 
-- name: Check if {{nextcloud_control_node_plugin_vars_directory}}{{ plugin_key }}.yml exists
-  stat:
-    path: "{{nextcloud_control_node_plugin_vars_directory}}{{ plugin_key }}.yml"
-  delegate_to: localhost
-  become: false
-  register: plugin_vars_file
+  - name: Check if {{nextcloud_control_node_plugin_vars_directory}}{{ plugin_key }}.yml exists
+    stat:
+      path: "{{nextcloud_control_node_plugin_vars_directory}}{{ plugin_key }}.yml"
+    delegate_to: localhost
+    become: false
+    register: plugin_vars_file
 
+  - name: "Load {{ plugin_key }} configuration variables"
+    include_vars:
+      file: "{{nextcloud_control_node_plugin_vars_directory}}{{ plugin_key }}.yml"
+    when: plugin_vars_file.stat.exists
 
-- name: "Load {{ plugin_key }} configuration variables"
-  include_vars:
-    file: "{{nextcloud_control_node_plugin_vars_directory}}{{ plugin_key }}.yml"
-  when: plugin_vars_file.stat.exists
-    
-- name: "Set {{ item.configkey }} for {{ item.appid }}"
-  loop: "{{ plugin_configuration }}"
-  command: >
-    {{ nextcloud_docker_exec_occ }} config:app:set {{ item.appid }} {{ item.configkey }} --value '{{ item.configvalue | to_json if item.configvalue is mapping else item.configvalue }}'
-  register: config_set_result
-  changed_when: (config_set_result.stdout is defined) and ("Config value were not updated" not in config_set_result.stdout)
-  when: plugin_vars_file.stat.exists
+  - name: "Set {{ item.configkey }} for {{ item.appid }}"
+    loop: "{{ plugin_configuration }}"
+    command: >
+      {{ nextcloud_docker_exec_occ }} config:app:set {{ item.appid }} {{ item.configkey }} --value '{{ item.configvalue | to_json if item.configvalue is mapping else item.configvalue }}'
+    register: config_set_result
+    changed_when: (config_set_result.stdout is defined) and ("Config value were not updated" not in config_set_result.stdout)
+    when: plugin_vars_file.stat.exists
 
-- name: Check if {{nextcloud_control_node_plugin_tasks_directory}}{{ plugin_key }}.yml exists
-  stat:
-    path: "{{nextcloud_control_node_plugin_tasks_directory}}{{ plugin_key }}.yml"
-  delegate_to: localhost
-  become: false
-  register: plugin_tasks_file
+  - name: Check if {{nextcloud_control_node_plugin_tasks_directory}}{{ plugin_key }}.yml exists
+    stat:
+      path: "{{nextcloud_control_node_plugin_tasks_directory}}{{ plugin_key }}.yml"
+    delegate_to: localhost
+    become: false
+    register: plugin_tasks_file
 
-- name: "include {{nextcloud_control_node_plugin_tasks_directory}}{{ plugin_key }}.yml"
-  include_tasks: "{{nextcloud_control_node_plugin_tasks_directory}}{{ plugin_key }}.yml"
-  when: plugin_tasks_file.stat.exists
+  - name: "include {{nextcloud_control_node_plugin_tasks_directory}}{{ plugin_key }}.yml"
+    include_tasks: "{{nextcloud_control_node_plugin_tasks_directory}}{{ plugin_key }}.yml"
+    when: plugin_tasks_file.stat.exists
+  when:
+    - plugin_value.enabled | bool
+    - install_result is defined
+    - >
+      install_result.rc == 0
+      or "already installed" in install_result.stdout

roles/docker-nextcloud/vars/plugins/bbb.yml

@@ -4,4 +4,4 @@ plugin_configuration:
     configvalue:  "{{ applications.bigbluebutton.credentials.shared_secret }}"
   - appid:        "bbb"
     configkey:    "api.url"
-    configvalue:  "{{ applications.bigbluebutton.urls.api }}"
+    configvalue:  "{{ applications.bigbluebutton.urls.api }}"

roles/nginx-docker-reverse-proxy/templates/headers/content_security_policy.conf.j2

@@ -3,6 +3,13 @@
 {# default-src: Fallback for all other directives if not explicitly defined #}
 {%- set csp_parts = csp_parts + ["default-src 'self';"] %}
 
+{# connect-src: Controls where fetch(), XHR, WebSocket etc. can connect to #}
+{%- set connect_src = "connect-src 'self' https://ka-f.fontawesome.com" %}
+{%- if applications | is_feature_enabled('matomo', application_id) | bool %}
+  {%- set connect_src = connect_src + " " + web_protocol + "://" + domains.matomo %}
+{%- endif %}
+{%- set csp_parts = csp_parts + [connect_src + ";"] %}
+
 {# frame-ancestors: Restricts which origins can embed this site in a frame or iframe #}
 {%- set frame_ancestors = "frame-ancestors 'self'" %}
 {%- if applications | is_feature_enabled('iframe', application_id) | bool %}
@@ -13,21 +20,22 @@
 {# frame-src: Controls which URLs can be embedded as iframes #}
 {%- set frame_src = "frame-src 'self'" %}
 {%- if applications | is_feature_enabled('recaptcha', application_id) | bool %}
-  {%- set frame_src = frame_src + " https://www.google.com https://www.recaptcha.net" %}
+  {%- set frame_src = frame_src + " https://www.google.com" %}
 {%- endif %}
 {%- set csp_parts = csp_parts + [frame_src + ";"] %}
 
-{# img-src: Allow images from own domain and files deliverer. Also from Matomo if enabled. #}
-{%- set img_src = "img-src 'self' " + web_protocol + "://" + domains.file_server %}
-{%- if applications | is_feature_enabled('matomo', application_id) | bool %}
-  {%- set img_src = img_src + " " + web_protocol + "://" + domains.matomo %}
-{%- endif %}
+{# img-src: Allow images. Prevent tracking by caching on server and client side. #}
+{%- set img_src = "img-src * data: blob:"%}
 {%- set csp_parts = csp_parts + [img_src + ";"] %}
 
 {# script-src: Allow JavaScript from self, FontAwesome, jsDelivr, and Matomo if enabled #}
-{%- set script_src = "script-src 'self' 'unsafe-inline'" %}
+{# unsafe eval is set for sphinx #}
+{%- set script_src = "script-src 'self' 'unsafe-eval' 'unsafe-inline'" %}
 {%- if applications | is_feature_enabled('matomo', application_id) | bool %}
-  {%- set script_src = script_src + " " + domains.matomo %}
+  {%- set script_src = script_src + " " + web_protocol + "://" + domains.matomo %}
+{%- endif %}
+{%- if applications | is_feature_enabled('recaptcha', application_id) | bool %}
+  {%- set script_src = script_src + " https://www.google.com" %}
 {%- endif %}
 {%- set script_src = script_src + " https://kit.fontawesome.com https://cdn.jsdelivr.net" %}
 {%- set csp_parts = csp_parts + [script_src + ";"] %}
@@ -36,4 +44,10 @@
 {%- set style_src = "style-src 'self' 'unsafe-inline' https://kit.fontawesome.com https://cdn.jsdelivr.net" %}
 {%- set csp_parts = csp_parts + [style_src + ";"] %}
 
+{# font-src: Allow font-src from self, FontAwesome, jsDelivr and inline styles #}
+{%- set font_src = "font-src 'self' https://kit.fontawesome.com https://cdn.jsdelivr.net" %}
+{%- set csp_parts = csp_parts + [font_src + ";"] %}
+
 add_header Content-Security-Policy "{{ csp_parts | join(' ') }}" always;
+# Oppress header send by proxied application
+proxy_hide_header Content-Security-Policy;

roles/nginx-docker-reverse-proxy/templates/location/proxy_basic.conf.j2

@@ -31,3 +31,6 @@ location {{location | default("/")}}
   proxy_read_timeout          900s;
   send_timeout                900s;
 }
+
+# Load caching
+{% include 'roles/nginx-docker-reverse-proxy/templates/location/proxy_cache.conf.j2' %}

roles/nginx-docker-reverse-proxy/templates/location/proxy_cache.conf.j2

@@ -0,0 +1,12 @@
+location ~* \.(jpg|jpeg|png|gif|webp|ico|svg)$ {
+    # Cache in browser
+    expires 30d;
+    add_header Cache-Control "public, max-age=2592000, immutable";
+
+    # Cache on reverse proxy side
+    proxy_pass http://127.0.0.1:{{http_port}};
+    proxy_cache imgcache;
+    proxy_cache_valid 200 302 60m;
+    proxy_cache_valid 404 1m;
+    add_header X-Proxy-Cache $upstream_cache_status;
+}

roles/nginx-https/meta/main.yml

@@ -1,4 +1,4 @@
 dependencies:
 - nginx
-- nginx-domains-cleanup
+- cleanup-domains
 - letsencrypt

roles/nginx/tasks/cache_directories.yml

@@ -0,0 +1,30 @@
+- name: Cleanup all NGINX cache directories
+  become: true
+  ansible.builtin.file:
+    path: "{{ item.value }}"
+    state: absent
+  when:
+    - mode_cleanup | bool
+    - run_once_nginx_reverse_proxy is not defined
+  loop: "{{ nginx.directories.cache | dict2items }}"
+  loop_control:
+    label: "{{ item.key }}"
+
+- name: Ensure all NGINX cache directories exist
+  become: true
+  ansible.builtin.file:
+    path: "{{ item.value }}"
+    state: directory
+    owner: "{{ nginx.user }}"
+    group: "{{ nginx.user }}"
+    mode: '0700'
+      
+  when: run_once_nginx_reverse_proxy is not defined
+  loop: "{{ nginx.directories.cache | dict2items }}"
+  loop_control:
+    label: "{{ item.key }}"
+
+- name: run the nginx_reverse_proxy tasks once
+  set_fact:
+    run_once_nginx_reverse_proxy: true
+  when: run_once_nginx_reverse_proxy is not defined

roles/nginx/tasks/main.yml

@@ -42,6 +42,9 @@
     {{ nginx.directories.data.values() | list }}
   when: run_once_nginx is not defined
 
+- name: "Include tasks to create cache directories"
+  include_tasks: cache_directories.yml
+
 - name: create nginx config file
   template: 
     src:  nginx.conf.j2 

roles/nginx/templates/nginx.conf.j2

@@ -12,7 +12,14 @@ http
   default_type text/html;
 
   {# caching #}
-  proxy_cache_path /tmp/cache levels=1:2 keys_zone=cache:20m max_size=20g inactive=14d use_temp_path=off;
+  proxy_cache_path {{ nginx.directories.cache.general }} levels=1:2 keys_zone=cache:20m max_size=20g inactive=14d use_temp_path=off;
+  proxy_cache_path {{ nginx.directories.cache.image }} levels=1:2 keys_zone=imgcache:10m inactive=60m use_temp_path=off;
+
+  # --------------------------------------------------------------------------------
+  # Tweak the hash table used to store your server_name entries:
+  server_names_hash_bucket_size 64;   # size of each bucket for server_name lookups (in bytes)
+  server_names_hash_max_size    512;  # maximum total buckets for the server_name hash table
+  # --------------------------------------------------------------------------------
 
   {# logging and debugging #}
 {% if enable_debug | bool %}

roles/update/tasks/main.yml

@@ -21,9 +21,9 @@
 
 - name: "Check if yay is installed"
   command: which yay
-  ignore_errors: yes
   register: yay_installed
   changed_when: false
+  failed_when: false
 
 - name: "Update with yay"
   include_role:
@@ -32,9 +32,9 @@
 
 - name: "Check if pip is installed"
   command: which pip
-  ignore_errors: yes
   register: pip_installed
   changed_when: false
+  failed_when: false
 
 - name: "Update with pip"
   include_role:
@@ -43,9 +43,9 @@
 - name: "Check if pkgmgr command is available"
   command: "which pkgmgr"
   register: pkgmgr_available
-  ignore_errors: yes
+  failed_when: false
 
 - name: "Update all repositories using pkgmgr"
   include_role:
     name: update-pkgmgr
-  when: pkgmgr_available.rc == 0
+  when: pkgmgr_available.rc == 0

templates/vars/features.yml.j2

@@ -1,13 +1,14 @@
 {% macro render_features(options) %}
     features:
     {%- set feature_map = {
-        'matomo':   'Enables Matomo tracking',
-        'css':      'Enables custom CSS styling',
-        'iframe':   'Allows embedding via iframe on landing page',
-        'ldap':     'Enables LDAP integration and networking',
-        'oidc':     'Enables OpenID Connect (OIDC) authentication',
-        'oauth2':   'Enables OAuth2 proxy integration',
-        'database': 'Enables use of central database'
+        'matomo':       'Enables Matomo tracking',
+        'css':          'Enables custom CSS styling',
+        'iframe':       'Allows embedding via iframe on landing page',
+        'ldap':         'Enables LDAP integration and networking',
+        'oidc':         'Enables OpenID Connect (OIDC) authentication',
+        'oauth2':       'Enables OAuth2 proxy integration',
+        'database':     'Enables use of central database',
+        'recaptcha':    'Enables recaptcha functionality'
     } %}
     {%- for key, comment in feature_map.items() %}
         {%- if key in options %}