From e5e394d470e1972089d3b35b547039f30df0ee88 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 29 Apr 2025 02:20:10 +0200 Subject: [PATCH] Optimized cloudflare implementation --- library/__init__.py | 0 library/cert_check_exists.py | 92 +++++++++++++++++++ ...ind_cert_folder.py => cert_folder_find.py} | 47 ++-------- module_utils/cert_utils.py | 39 ++++++++ roles/dns-records-cloudflare/tasks/main.yml | 2 +- roles/docker-bigbluebutton/tasks/main.yml | 8 ++ roles/docker-bigbluebutton/templates/env.j2 | 14 +-- roles/nginx-domains-cleanup/tasks/main.yml | 80 ++++++++-------- .../tasks/remove_deprecated_nginx_configs.yml | 20 ++++ .../tasks/flavors/dedicated.yml | 7 ++ roles/nginx-https-get-cert/tasks/main.yml | 32 +++---- roles/nginx-redirect-www/tasks/main.yml | 20 ++-- tasks/destructor.yml | 5 +- 13 files changed, 249 insertions(+), 117 deletions(-) create mode 100644 library/__init__.py create mode 100644 library/cert_check_exists.py rename library/{find_cert_folder.py => cert_folder_find.py} (74%) create mode 100644 module_utils/cert_utils.py create mode 100644 roles/nginx-domains-cleanup/tasks/remove_deprecated_nginx_configs.yml diff --git a/library/__init__.py b/library/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/library/cert_check_exists.py b/library/cert_check_exists.py new file mode 100644 index 00000000..c57fdbd2 --- /dev/null +++ b/library/cert_check_exists.py @@ -0,0 +1,92 @@ +#!/usr/bin/python + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +DOCUMENTATION = r''' +--- +module: cert_check_exists +short_description: Check if a SSL certificate exists for a domain +description: + - Checks if any certificate covers the given domain. +options: + domain: + description: + - Domain name to check for in the certificates. + required: true + type: str + cert_base_path: + description: + - Path where certificates are stored. + required: false + type: str + default: /etc/letsencrypt/live + debug: + description: + - Enable verbose debug output. + required: false + type: bool + default: false +author: + - Kevin Veen-Birkenbach +''' + +EXAMPLES = r''' +- name: Check if cert exists + cert_check_exists: + domain: "matomo.cymais.cloud" + cert_base_path: "/etc/letsencrypt/live" + register: result +''' + +RETURN = r''' +exists: + description: True if a certificate covering the domain exists, false otherwise. + type: bool + returned: always +''' + +import os +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.cert_utils import CertUtils + +def cert_exists(domain, cert_files, debug=False): + for cert_path in cert_files: + cert_text = CertUtils.run_openssl(cert_path) + if not cert_text: + continue + sans = CertUtils.extract_sans(cert_text) + if debug: + print(f"Checking {cert_path}: {sans}") + for entry in sans: + if entry == domain or (entry.startswith('*.') and domain.endswith('.' + entry[2:])): + return True + return False + +def cert_check_exists(module): + domain = module.params['domain'] + cert_base_path = module.params['cert_base_path'] + debug = module.params['debug'] + + cert_files = CertUtils.list_cert_files(cert_base_path) + + exists = cert_exists(domain, cert_files, debug) + + module.exit_json(exists=exists) + +def main(): + module_args = dict( + domain=dict(type='str', required=True), + cert_base_path=dict(type='str', required=False, default='/etc/letsencrypt/live'), + debug=dict(type='bool', required=False, default=False), + ) + + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=True + ) + + cert_check_exists(module) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/library/find_cert_folder.py b/library/cert_folder_find.py similarity index 74% rename from library/find_cert_folder.py rename to library/cert_folder_find.py index 87c24bae..6e306c39 100644 --- a/library/find_cert_folder.py +++ b/library/cert_folder_find.py @@ -5,7 +5,7 @@ __metaclass__ = type DOCUMENTATION = r''' --- -module: find_cert_folder +module: cert_folder_find short_description: Find SSL certificate folder covering a given domain description: - Searches through certificates to find a folder that covers the given domain. @@ -38,7 +38,7 @@ author: EXAMPLES = r''' - name: Find cert folder for matomo.cymais.cloud - find_cert_folder: + cert_folder_find: domain: "matomo.cymais.cloud" certbot_flavor: "san" cert_base_path: "/etc/letsencrypt/live" @@ -53,42 +53,18 @@ folder: ''' import os -import subprocess from ansible.module_utils.basic import AnsibleModule - -def run_openssl(cert_path): - try: - output = subprocess.check_output( - ['openssl', 'x509', '-in', cert_path, '-noout', '-text'], - universal_newlines=True - ) - return output - except subprocess.CalledProcessError: - return "" - -def extract_sans(cert_text): - dns_entries = [] - in_san = False - for line in cert_text.splitlines(): - line = line.strip() - if 'X509v3 Subject Alternative Name:' in line: - in_san = True - continue - if in_san: - if not line: - break - dns_entries += [e.strip().replace('DNS:', '') for e in line.split(',') if e.strip()] - return dns_entries +from ansible.module_utils.cert_utils import CertUtils # IMPORT def find_matching_folders(domain, cert_files, flavor, debug): exact_matches = [] wildcard_matches = [] for cert_path in cert_files: - cert_text = run_openssl(cert_path) + cert_text = CertUtils.run_openssl(cert_path) if not cert_text: continue - sans = extract_sans(cert_text) + sans = CertUtils.extract_sans(cert_text) if debug: print(f"Checking {cert_path}: {sans}") for entry in sans: @@ -106,16 +82,13 @@ def find_matching_folders(domain, cert_files, flavor, debug): else: return [] -def find_cert_folder(module): +def cert_folder_find(module): domain = module.params['domain'] certbot_flavor = module.params['certbot_flavor'] cert_base_path = module.params['cert_base_path'] debug = module.params['debug'] - cert_files = [] - for root, dirs, files in os.walk(cert_base_path): - if 'cert.pem' in files: - cert_files.append(os.path.join(root, 'cert.pem')) + cert_files = CertUtils.list_cert_files(cert_base_path) if debug: print(f"Found {len(cert_files)} cert.pem files under {cert_base_path}") @@ -126,7 +99,7 @@ def find_cert_folder(module): if debug: print("Fallback: searching SAN matches without SAN structure parsing") for cert_path in cert_files: - cert_text = run_openssl(cert_path) + cert_text = CertUtils.run_openssl(cert_path) if f"DNS:{domain}" in cert_text: preferred.append(os.path.dirname(cert_path)) @@ -151,7 +124,7 @@ def main(): supports_check_mode=True ) - find_cert_folder(module) + cert_folder_find(module) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/module_utils/cert_utils.py b/module_utils/cert_utils.py new file mode 100644 index 00000000..a0569e39 --- /dev/null +++ b/module_utils/cert_utils.py @@ -0,0 +1,39 @@ +#!/usr/bin/python + +import os +import subprocess + +class CertUtils: + @staticmethod + def run_openssl(cert_path): + try: + output = subprocess.check_output( + ['openssl', 'x509', '-in', cert_path, '-noout', '-text'], + universal_newlines=True + ) + return output + except subprocess.CalledProcessError: + return "" + + @staticmethod + def extract_sans(cert_text): + dns_entries = [] + in_san = False + for line in cert_text.splitlines(): + line = line.strip() + if 'X509v3 Subject Alternative Name:' in line: + in_san = True + continue + if in_san: + if not line: + break + dns_entries += [e.strip().replace('DNS:', '') for e in line.split(',') if e.strip()] + return dns_entries + + @staticmethod + def list_cert_files(cert_base_path): + cert_files = [] + for root, dirs, files in os.walk(cert_base_path): + if 'cert.pem' in files: + cert_files.append(os.path.join(root, 'cert.pem')) + return cert_files diff --git a/roles/dns-records-cloudflare/tasks/main.yml b/roles/dns-records-cloudflare/tasks/main.yml index 89300c7c..464b989f 100644 --- a/roles/dns-records-cloudflare/tasks/main.yml +++ b/roles/dns-records-cloudflare/tasks/main.yml @@ -12,7 +12,7 @@ name: "{{ item }}" content: "{{ cloudflare_target_ip }}" ttl: 1 - proxied: "{{ cloudflare_target_ip }}" + proxied: "{{ cloudflare_proxied | int }}" loop: "{{ cloudflare_domains }}" loop_control: label: "{{ item }}" diff --git a/roles/docker-bigbluebutton/tasks/main.yml b/roles/docker-bigbluebutton/tasks/main.yml index acba9f14..fa954c4d 100644 --- a/roles/docker-bigbluebutton/tasks/main.yml +++ b/roles/docker-bigbluebutton/tasks/main.yml @@ -4,6 +4,14 @@ include_role: name: docker-compose +- name: "Seed BigBlueButton Database for Backup" + include_tasks: "{{ playbook_dir }}/roles/backup-docker-to-local/tasks/seed-database-to-backup.yml" + vars: + database_instance: "{{ application_id }}" + database_password: "{{ applications[application_id].credentials.postgresql_secret }}" + database_username: "postgres" + database_name: "" # Multiple databases + - name: "include role nginx-domain-setup for {{application_id}}" include_role: name: nginx-domain-setup diff --git a/roles/docker-bigbluebutton/templates/env.j2 b/roles/docker-bigbluebutton/templates/env.j2 index 72c90b34..df6c0cff 100644 --- a/roles/docker-bigbluebutton/templates/env.j2 +++ b/roles/docker-bigbluebutton/templates/env.j2 @@ -1,7 +1,7 @@ ENABLE_COTURN=true COTURN_TLS_CERT_PATH={{ certbot_cert_path }}/{{ ssl_cert_folder }}/fullchain.pem COTURN_TLS_KEY_PATH={{ certbot_cert_path }}/{{ ssl_cert_folder }}/privkey.pem -ENABLE_GREENLIGHT={{applications.bigbluebutton.enable_greenlight}} +ENABLE_GREENLIGHT={{applications[application_id].enable_greenlight}} # Enable Webhooks # used by some integrations @@ -27,11 +27,11 @@ RECORDING_MAX_AGE_DAYS=365 # SECRETS # ==================================== # important! change these to any random values -SHARED_SECRET={{applications.bigbluebutton.credentials.shared_secret}} -ETHERPAD_API_KEY={{applications.bigbluebutton.credentials.etherpad_api_key}} -RAILS_SECRET={{applications.bigbluebutton.credentials.rails_secret}} -POSTGRESQL_SECRET={{applications.bigbluebutton.credentials.postgresql_secret}} -FSESL_PASSWORD={{applications.bigbluebutton.credentials.fsesl_password}} +SHARED_SECRET={{applications[application_id].credentials.shared_secret}} +ETHERPAD_API_KEY={{applications[application_id].credentials.etherpad_api_key}} +RAILS_SECRET={{applications[application_id].credentials.rails_secret}} +POSTGRESQL_SECRET={{applications[application_id].credentials.postgresql_secret}} +FSESL_PASSWORD={{applications[application_id].credentials.fsesl_password}} # ==================================== # CONNECTION @@ -51,7 +51,7 @@ STUN_PORT={{ ports.public.stun[application_id] }} # TURN SERVER # uncomment and adjust following two lines to add an external TURN server TURN_SERVER=turns:{{domains[application_id]}}:{{ ports.public.turn[application_id] }}?transport=tcp -TURN_SECRET={{applications.bigbluebutton.credentials.turn_secret}} +TURN_SECRET={{applications[application_id].credentials.turn_secret}} # Allowed SIP IPs # due to high traffic caused by bots, by default the SIP port is blocked. diff --git a/roles/nginx-domains-cleanup/tasks/main.yml b/roles/nginx-domains-cleanup/tasks/main.yml index 63112b33..ceeadb8b 100644 --- a/roles/nginx-domains-cleanup/tasks/main.yml +++ b/roles/nginx-domains-cleanup/tasks/main.yml @@ -1,52 +1,50 @@ --- -- name: "Remove Nginx configuration for deprecated domains" - ansible.builtin.command: - cmd: >- - rm -fv /etc/nginx/conf.d/http/servers/*.{{ item }}.conf; - rm -fv /etc/nginx/conf.d/http/servers/{{ item }}.conf +- name: Include task to remove deprecated nginx configs + include_tasks: remove_deprecated_nginx_configs.yml loop: "{{ deprecated_domains }}" loop_control: label: "{{ item }}" - notify: restart nginx + vars: + domain: "{{ item }}" when: - mode_cleanup | bool - run_once_nginx_domains_cleanup is not defined -# The revoking just works for the base domain -- name: "Revoke Certbot certificate for {{ item }}" - ansible.builtin.command: - cmd: "certbot revoke -n --cert-name {{ item }}" - become: true - loop: "{{ deprecated_domains }}" - loop_control: - label: "{{ item }}" - when: - - mode_cleanup | bool - - run_once_nginx_domains_cleanup is not defined - register: certbot_revoke_result - failed_when: > - certbot_revoke_result.rc != 0 and - 'No certificate found with name' not in certbot_revoke_result.stderr - changed_when: > - certbot_revoke_result.rc == 0 - -# The deleting just works for the base domain -- name: "Delete Certbot certificate for {{ item }}" - ansible.builtin.command: - cmd: "certbot delete -n --cert-name {{ item }}" - become: true - loop: "{{ deprecated_domains }}" - loop_control: - label: "{{ item }}" - when: - - mode_cleanup | bool - - run_once_nginx_domains_cleanup is not defined - register: certbot_delete_result - failed_when: > - certbot_delete_result.rc != 0 and - 'No certificate found with name' not in certbot_delete_result.stderr - changed_when: > - certbot_delete_result.rc == 0 +## The revoking just works for the base domain +#- name: "Revoke Certbot certificate for {{ item }}" +# ansible.builtin.command: +# cmd: "certbot revoke -n --cert-name {{ item }} --non-interactive" +# become: true +# loop: "{{ deprecated_domains }}" +# loop_control: +# label: "{{ item }}" +# when: +# - mode_cleanup | bool +# - run_once_nginx_domains_cleanup is not defined +# register: certbot_revoke_result +# failed_when: > +# certbot_revoke_result.rc != 0 and +# 'No certificate found with name' not in certbot_revoke_result.stderr +# changed_when: > +# certbot_revoke_result.rc == 0 +# +## The deleting just works for the base domain +#- name: "Delete Certbot certificate for {{ item }}" +# ansible.builtin.command: +# cmd: "certbot delete -n --cert-name {{ item }} --non-interactive" +# become: true +# loop: "{{ deprecated_domains }}" +# loop_control: +# label: "{{ item }}" +# when: +# - mode_cleanup | bool +# - run_once_nginx_domains_cleanup is not defined +# register: certbot_delete_result +# failed_when: > +# certbot_delete_result.rc != 0 and +# 'No certificate found with name' not in certbot_delete_result.stderr +# changed_when: > +# certbot_delete_result.rc == 0 - name: run the nginx_domains_cleanup role once set_fact: diff --git a/roles/nginx-domains-cleanup/tasks/remove_deprecated_nginx_configs.yml b/roles/nginx-domains-cleanup/tasks/remove_deprecated_nginx_configs.yml new file mode 100644 index 00000000..dd0f289b --- /dev/null +++ b/roles/nginx-domains-cleanup/tasks/remove_deprecated_nginx_configs.yml @@ -0,0 +1,20 @@ +--- +- name: Find matching nginx configs for {{ domain }} + ansible.builtin.find: + paths: /etc/nginx/conf.d/http/servers + patterns: "*.{{ domain }}.conf" + register: find_result + +- name: Remove wildcard nginx configs for {{ domain }} + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + loop: "{{ find_result.files | default([]) }}" + when: item is defined + notify: restart nginx + +- name: Remove exact nginx config for {{ domain }} + ansible.builtin.file: + path: "/etc/nginx/conf.d/http/servers/{{ domain }}.conf" + state: absent + notify: restart nginx \ No newline at end of file diff --git a/roles/nginx-https-get-cert/tasks/flavors/dedicated.yml b/roles/nginx-https-get-cert/tasks/flavors/dedicated.yml index af4f9ee4..08962944 100644 --- a/roles/nginx-https-get-cert/tasks/flavors/dedicated.yml +++ b/roles/nginx-https-get-cert/tasks/flavors/dedicated.yml @@ -1,3 +1,9 @@ +- name: "Check if certificate already exists for {{ domain }}" + cert_check_exists: + domain: "{{ domain }}" + cert_base_path: "{{ certbot_cert_path }}" + register: cert_check + - name: "receive certificate for {{ domain }}" command: >- certbot certonly @@ -21,3 +27,4 @@ {{ '--test-cert' if mode_test | bool else '' }} register: certbot_result changed_when: "'Certificate not yet due for renewal' not in certbot_result.stdout" + when: not cert_check.exists \ No newline at end of file diff --git a/roles/nginx-https-get-cert/tasks/main.yml b/roles/nginx-https-get-cert/tasks/main.yml index 4eb58322..0ca0eb6b 100644 --- a/roles/nginx-https-get-cert/tasks/main.yml +++ b/roles/nginx-https-get-cert/tasks/main.yml @@ -1,24 +1,24 @@ - name: "Include flavor" include_tasks: "{{ role_path }}/tasks/flavors/{{ certbot_flavor }}.yml" -- name: "Cleanup dedicated cert for {{ domain }}" - command: >- - certbot delete --cert-name {{ domain }} --non-interactive - when: - - mode_cleanup | bool - # Cleanup mode is enabled - - certbot_flavor != 'dedicated' - # Wildcard certificate is enabled - - domain.split('.') | length == (primary_domain.split('.') | length + 1) and domain.endswith(primary_domain) - # AND: The domain is a direct first-level subdomain of the primary domain - - domain != primary_domain - # The domain is not the primary domain - register: certbot_result - failed_when: certbot_result.rc != 0 and ("No certificate found with name" not in certbot_result.stderr) - changed_when: certbot_result.rc == 0 and ("No certificate found with name" not in certbot_result.stderr) +#- name: "Cleanup dedicated cert for {{ domain }}" +# command: >- +# certbot delete --cert-name {{ domain }} --non-interactive +# when: +# - mode_cleanup | bool +# # Cleanup mode is enabled +# - certbot_flavor != 'dedicated' +# # Wildcard certificate is enabled +# - domain.split('.') | length == (primary_domain.split('.') | length + 1) and domain.endswith(primary_domain) +# # AND: The domain is a direct first-level subdomain of the primary domain +# - domain != primary_domain +# # The domain is not the primary domain +# register: certbot_result +# failed_when: certbot_result.rc != 0 and ("No certificate found with name" not in certbot_result.stderr) +# changed_when: certbot_result.rc == 0 and ("No certificate found with name" not in certbot_result.stderr) - name: Find SSL cert folder for domain - find_cert_folder: + cert_folder_find: domain: "{{ domain }}" certbot_flavor: "{{ certbot_flavor }}" cert_base_path: "{{ certbot_cert_path }}" diff --git a/roles/nginx-redirect-www/tasks/main.yml b/roles/nginx-redirect-www/tasks/main.yml index 9b8f1ec6..bde7ef31 100644 --- a/roles/nginx-redirect-www/tasks/main.yml +++ b/roles/nginx-redirect-www/tasks/main.yml @@ -8,21 +8,17 @@ set_fact: www_domains: "{{ all_domains | select('match', '^www\\.') | list }}" -- name: Build redirect mappings for www domains - set_fact: - domain_mappings: >- - {{ www_domains - | map('regex_replace', '^www\\.(.+)$', '{ source: \"www.\\1\", target: \"\\1\" }') - | map('from_yaml') - | list - }} - - name: Include nginx-redirect-domain role for www-to-bare redirects include_role: name: nginx-redirect-domain vars: - domain_mappings: "{{ domain_mappings }}" - when: certbot_flavor == 'dedicated' + domain_mappings: "{{ www_domains + | map('regex_replace', + '^www\\.(.+)$', + '{ source: \"www.\\1\", target: \"\\1\" }') + | map('from_yaml') + | list + }}" - name: Include DNS role to set redirects include_role: @@ -31,5 +27,5 @@ cloudflare_api_token: "{{ certbot_dns_api_token }}" cloudflare_domains: "{{ www_domains }}" cloudflare_target_ip: "{{ networks.internet.ip4 }}" - cloudflare_proxied_false: false + cloudflare_proxied: false when: dns_provider == 'cloudflare' \ No newline at end of file diff --git a/tasks/destructor.yml b/tasks/destructor.yml index 020252a9..f1d7f931 100644 --- a/tasks/destructor.yml +++ b/tasks/destructor.yml @@ -8,9 +8,8 @@ name: cleanup-docker-anonymous-volumes when: mode_cleanup | bool -- name: "Show User Configuration (Important when mailu tokens are created automatic)" +- name: Show all facts debug: - msg: - users: "{{users}}" + var: ansible_facts when: enable_debug | bool