From 5948d7aa93c71c591e680cc6e8a63268bdfa95f5 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 20 May 2025 07:00:29 +0200 Subject: [PATCH] General code optimations and peertube optimation --- Makefile | 2 +- cli/generate-role-includes.py | 79 ---------- cli/generate_playbook.py | 140 ++++++++++++++++++ filter_plugins/redirect_filters.py | 37 +++++ roles/backup-data-to-usb/meta/main.yml | 4 +- roles/client-spotify/meta/main.yml | 2 +- roles/docker-akaunting/vars/configuration.yml | 2 +- roles/docker-keycloak/meta/main.yml | 5 + roles/docker-ldap/meta/main.yml | 3 + roles/docker-mastodon/meta/main.yml | 3 +- roles/docker-matomo/meta/main.yml | 6 +- roles/docker-peertube/tasks/enable-oidc.yml | 36 +++-- .../nginx-docker-reverse-proxy/meta/main.yml | 4 +- roles/nginx-modifier-css/meta/main.yml | 2 +- roles/persona-gamer/meta/main.yml | 6 +- roles/systemd-notifier/meta/main.yml | 4 +- tasks/constructor.yml | 27 ++-- ...t_no_circular_before_after_dependencies.py | 2 +- tests/unit/test_redirect_filters.py | 57 +++++++ 19 files changed, 306 insertions(+), 115 deletions(-) delete mode 100644 cli/generate-role-includes.py create mode 100644 cli/generate_playbook.py create mode 100644 filter_plugins/redirect_filters.py create mode 100644 tests/unit/test_redirect_filters.py diff --git a/Makefile b/Makefile index d2ec31d3..66320398 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ROLES_DIR := ./roles APPLICATIONS_OUT := ./group_vars/all/03_applications.yml APPLICATIONS_SCRIPT := ./cli/generate-applications-defaults.py INCLUDES_OUT := ./tasks/include-docker-roles.yml -INCLUDES_SCRIPT := ./cli/generate-role-includes.py +INCLUDES_SCRIPT := ./cli/generate_playbook.py .PHONY: build install test diff --git a/cli/generate-role-includes.py b/cli/generate-role-includes.py deleted file mode 100644 index 3ae02dec..00000000 --- a/cli/generate-role-includes.py +++ /dev/null @@ -1,79 +0,0 @@ -import os -import argparse -import yaml - -def find_roles(roles_dir, prefix=None): - """ - Yield absolute paths of role directories under roles_dir. - Only include roles whose directory name starts with prefix (if given) and contain vars/main.yml. - """ - for entry in os.listdir(roles_dir): - if prefix and not entry.startswith(prefix): - continue - path = os.path.join(roles_dir, entry) - vars_file = os.path.join(path, 'vars', 'main.yml') - if os.path.isdir(path) and os.path.isfile(vars_file): - yield path, vars_file - - -def load_application_id(vars_file): - """ - Load the vars/main.yml and return the value of application_id key. - Returns None if not found. - """ - with open(vars_file, 'r') as f: - data = yaml.safe_load(f) or {} - return data.get('application_id') - - -def generate_playbook_entries(roles_dir, prefix=None): - entries = [] - for role_path, vars_file in find_roles(roles_dir, prefix): - app_id = load_application_id(vars_file) - if not app_id: - continue - # Derive role name from directory name - role_name = os.path.basename(role_path) - # entry text - entry = ( - f"- name: setup {app_id}\n" - f" when: (\"{app_id}\" in group_names)\n" - f" include_role:\n" - f" name: {role_name}\n" - ) - entries.append(entry) - return entries - - -def main(): - parser = argparse.ArgumentParser( - description='Generate an Ansible playbook include file from Docker roles and application_ids.' - ) - parser.add_argument( - 'roles_dir', - help='Path to directory containing role folders' - ) - parser.add_argument( - '-p', '--prefix', - help='Only include roles whose names start with this prefix (e.g. docker-, client-)', - default=None - ) - parser.add_argument( - '-o', '--output', - help='Output file path (default: stdout)', - default=None - ) - args = parser.parse_args() - - entries = generate_playbook_entries(args.roles_dir, args.prefix) - output = ''.join(entries) - - if args.output: - with open(args.output, 'w') as f: - f.write(output) - print(f"Playbook entries written to {args.output}") - else: - print(output) - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/cli/generate_playbook.py b/cli/generate_playbook.py new file mode 100644 index 00000000..77d2072f --- /dev/null +++ b/cli/generate_playbook.py @@ -0,0 +1,140 @@ +import os +import argparse +import yaml + + +def find_roles(roles_dir, prefix=None): + """ + Yield absolute paths of role directories under roles_dir. + Only include roles whose directory name starts with prefix (if given) and contain meta/main.yml. + """ + for entry in os.listdir(roles_dir): + if prefix and not entry.startswith(prefix): + continue + path = os.path.join(roles_dir, entry) + meta_file = os.path.join(path, 'meta', 'main.yml') + if os.path.isdir(path) and os.path.isfile(meta_file): + yield path, meta_file + + +def load_role_order(meta_file): + """ + Load the meta/main.yml and return the role_run_order field. + Returns a dict with 'before' and 'after' keys. Defaults to empty lists if not found. + """ + with open(meta_file, 'r') as f: + data = yaml.safe_load(f) or {} + run_order = data.get('role_run_order', {}) + before = run_order.get('before', []) + after = run_order.get('after', []) + + # If "all" is in before or after, treat it as a special value + if "all" in before: + before.remove("all") + before.insert(0, "all") # Treat "all" as the first item + if "all" in after: + after.remove("all") + after.append("all") # Treat "all" as the last item + + return { + 'before': before, + 'after': after + } + + +def sort_roles_by_order(roles_dir, prefix=None): + roles = [] + + # Collect roles and their before/after dependencies + for role_path, meta_file in find_roles(roles_dir, prefix): + run_order = load_role_order(meta_file) + role_name = os.path.basename(role_path) + roles.append({ + 'role_name': role_name, + 'before': run_order['before'], + 'after': run_order['after'], + 'path': role_path + }) + + # Now sort the roles based on before/after relationships + sorted_roles = [] + unresolved_roles = roles[:] + + # First, place roles with "before: all" at the start + roles_with_before_all = [role for role in unresolved_roles if "all" in role['before']] + sorted_roles.extend(roles_with_before_all) + unresolved_roles = [role for role in unresolved_roles if "all" not in role['before']] + + while unresolved_roles: + # Find roles with no dependencies in 'before' + ready_roles = [role for role in unresolved_roles if not any(dep in [r['role_name'] for r in unresolved_roles] for dep in role['before'])] + + if not ready_roles: + raise ValueError("Circular dependency detected in 'before'/'after' fields") + + for role in ready_roles: + sorted_roles.append(role) + unresolved_roles.remove(role) + + # Remove from the 'before' lists of remaining roles + for r in unresolved_roles: + r['before'] = [dep for dep in r['before'] if dep != role['role_name']] + + # Finally, place roles with "after: all" at the end + roles_with_after_all = [role for role in unresolved_roles if "all" in role['after']] + sorted_roles.extend(roles_with_after_all) + unresolved_roles = [role for role in unresolved_roles if "all" not in role['after']] + + return sorted_roles + + +def generate_playbook_entries(roles_dir, prefix=None): + entries = [] + sorted_roles = sort_roles_by_order(roles_dir, prefix) + + for role in sorted_roles: + # entry text + entry = ( + f"- name: setup {role['role_name']}\n" + f" when: (\"{role['role_name']}\" in group_names)\n" + f" include_role:\n" + f" name: {role['role_name']}\n" + ) + entries.append(entry) + + return entries + + +def main(): + parser = argparse.ArgumentParser( + description='Generate an Ansible playbook include file from Docker roles and application_ids, sorted by role_run_order.' + ) + parser.add_argument( + 'roles_dir', + help='Path to directory containing role folders' + ) + parser.add_argument( + '-p', '--prefix', + help='Only include roles whose names start with this prefix (e.g. docker-, client-)', + default=None + ) + parser.add_argument( + '-o', '--output', + help='Output file path (default: stdout)', + default=None + ) + args = parser.parse_args() + + entries = generate_playbook_entries(args.roles_dir, args.prefix) + output = ''.join(entries) + + if args.output: + with open(args.output, 'w') as f: + f.write(output) + print(f"Playbook entries written to {args.output}") + else: + print(output) + + +if __name__ == '__main__': + main() diff --git a/filter_plugins/redirect_filters.py b/filter_plugins/redirect_filters.py new file mode 100644 index 00000000..1ac4f587 --- /dev/null +++ b/filter_plugins/redirect_filters.py @@ -0,0 +1,37 @@ +# roles//filter_plugins/redirect_filters.py +from ansible.errors import AnsibleFilterError + +class FilterModule(object): + """ + Custom filters for redirect domain mappings + """ + + def filters(self): + return { + "add_redirect_if_group": self.add_redirect_if_group, + } + + @staticmethod + def add_redirect_if_group(redirect_list, group, source, target, group_names): + """ + Append {"source": source, "target": target} to *redirect_list* + **only** if *group* is contained in *group_names*. + + Usage in Jinja: + {{ redirect_list + | add_redirect_if_group('lam', + 'ldap.' ~ primary_domain, + domains | get_domain('lam'), + group_names) }} + """ + try: + # Make a copy so we don’t mutate the original list in place + redirects = list(redirect_list) + + if group in group_names: + redirects.append({"source": source, "target": target}) + + return redirects + + except Exception as exc: + raise AnsibleFilterError(f"add_redirect_if_group failed: {exc}") diff --git a/roles/backup-data-to-usb/meta/main.yml b/roles/backup-data-to-usb/meta/main.yml index 979d0d3c..bd8773c9 100644 --- a/roles/backup-data-to-usb/meta/main.yml +++ b/roles/backup-data-to-usb/meta/main.yml @@ -23,5 +23,5 @@ galaxy_info: issue_tracker_url: https://s.veen.world/cymaisissues documentation: https://s.veen.world/cymais dependencies: - - role: cleanup-backups-service - - role: system-maintenance-lock + - cleanup-backups-service + - system-maintenance-lock diff --git a/roles/client-spotify/meta/main.yml b/roles/client-spotify/meta/main.yml index b59799da..b0c6765f 100644 --- a/roles/client-spotify/meta/main.yml +++ b/roles/client-spotify/meta/main.yml @@ -24,4 +24,4 @@ galaxy_info: issue_tracker_url: https://s.veen.world/cymaisissues documentation: https://s.veen.world/cymais dependencies: - - role: system-aur-helper + - system-aur-helper diff --git a/roles/docker-akaunting/vars/configuration.yml b/roles/docker-akaunting/vars/configuration.yml index 70375bcb..1252db45 100644 --- a/roles/docker-akaunting/vars/configuration.yml +++ b/roles/docker-akaunting/vars/configuration.yml @@ -5,7 +5,7 @@ setup_admin_email: "{{users.administrator.email}}" features: matomo: true css: true - portfolio_iframe: false + portfolio_iframe: false central_database: true credentials: # database_password: Needs to be defined in inventory file diff --git a/roles/docker-keycloak/meta/main.yml b/roles/docker-keycloak/meta/main.yml index 1dd890da..86587eb0 100644 --- a/roles/docker-keycloak/meta/main.yml +++ b/roles/docker-keycloak/meta/main.yml @@ -20,3 +20,8 @@ galaxy_info: logo: class: "fa-solid fa-lock" dependencies: [] +role_run_order: + before: + - all + after: + - docker-ldap diff --git a/roles/docker-ldap/meta/main.yml b/roles/docker-ldap/meta/main.yml index 499f5711..227cc4bc 100644 --- a/roles/docker-ldap/meta/main.yml +++ b/roles/docker-ldap/meta/main.yml @@ -21,3 +21,6 @@ galaxy_info: logo: class: "fa-solid fa-users" dependencies: [] +role_run_order: + before: + - all diff --git a/roles/docker-mastodon/meta/main.yml b/roles/docker-mastodon/meta/main.yml index bbe5e299..07f1322c 100644 --- a/roles/docker-mastodon/meta/main.yml +++ b/roles/docker-mastodon/meta/main.yml @@ -21,4 +21,5 @@ galaxy_info: documentation: "https://s.veen.world/cymais" logo: class: "fa-solid fa-bullhorn" -dependencies: [] +role_run_order: + after: docker-keycloak \ No newline at end of file diff --git a/roles/docker-matomo/meta/main.yml b/roles/docker-matomo/meta/main.yml index 52906b5b..6f53a3ce 100644 --- a/roles/docker-matomo/meta/main.yml +++ b/roles/docker-matomo/meta/main.yml @@ -18,4 +18,8 @@ galaxy_info: documentation: "https://s.veen.world/cymais" logo: class: "fa-solid fa-chart-line" -dependencies: [] +role_run_order: + before: + - all + after: + - docker-keycloak \ No newline at end of file diff --git a/roles/docker-peertube/tasks/enable-oidc.yml b/roles/docker-peertube/tasks/enable-oidc.yml index 9aa51843..a09f0df5 100644 --- a/roles/docker-peertube/tasks/enable-oidc.yml +++ b/roles/docker-peertube/tasks/enable-oidc.yml @@ -3,14 +3,28 @@ docker exec {{ container_name }} \ npm run plugin:install -- --npm-name {{oidc_plugin}} -- name: Update Peertube config for OpenID Connect - ansible.builtin.lineinfile: - path: /opt/peertube/config/production.yaml - regexp: '^{{ item.key }}:' - line: "{{ item.key }}: {{ item.value }}" - loop: - - { key: "oidc.client_id", value: "{{ oidc_client_id }}" } - - { key: "oidc.client_secret", value: "{{ oidc_client_secret }}" } - - { key: "oidc.discover_url", value: "{{ oidc_discover_url }}" } - - { key: "oidc.scope", value: "openid email profile" } - become: yes +- name: "Update the settings column of the auth-openid-connect plugin" + community.postgresql.postgresql_query: + db: "{{ database_name }}" + login_user: "{{ database_username }}" + login_password: "{{ database_password }}" + login_host: "127.0.0.1" + login_port: "{{ database_port }}" + query: | + UPDATE plugins + SET settings = '{ + "scope": "openid email profile", + "client-id": "{{ oidc.client.id }}", + "discover-url": "{{ oidc.client.discovery_document }}", + "client-secret": "{{ oidc.client.secret }}", + "mail-property": "email", + "auth-display-name": "{{ oidc.button_text }}", + "username-property": "{{ oidc.attributes.username }}", + "signature-algorithm": "RS256", + "display-name-property": "{{ oidc.attributes.username }}" + }', + enabled = TRUE + WHERE name = 'auth-openid-connect'; + when: applications | is_feature_enabled('oidc', application_id) + become: true + become_user: "{{ container_name }}" \ No newline at end of file diff --git a/roles/nginx-docker-reverse-proxy/meta/main.yml b/roles/nginx-docker-reverse-proxy/meta/main.yml index 7d13e95c..6c9d4150 100644 --- a/roles/nginx-docker-reverse-proxy/meta/main.yml +++ b/roles/nginx-docker-reverse-proxy/meta/main.yml @@ -24,5 +24,5 @@ galaxy_info: issue_tracker_url: https://s.veen.world/cymaisissues documentation: https://s.veen.world/cymais dependencies: - - role: docker - - role: nginx-https \ No newline at end of file + - docker + - nginx-https \ No newline at end of file diff --git a/roles/nginx-modifier-css/meta/main.yml b/roles/nginx-modifier-css/meta/main.yml index 56795ea7..6081d545 100644 --- a/roles/nginx-modifier-css/meta/main.yml +++ b/roles/nginx-modifier-css/meta/main.yml @@ -26,4 +26,4 @@ galaxy_info: issue_tracker_url: https://s.veen.world/cymaisissues documentation: https://s.veen.world/cymais dependencies: - - role: nginx \ No newline at end of file + - nginx \ No newline at end of file diff --git a/roles/persona-gamer/meta/main.yml b/roles/persona-gamer/meta/main.yml index 734fd85a..9da65c9c 100644 --- a/roles/persona-gamer/meta/main.yml +++ b/roles/persona-gamer/meta/main.yml @@ -26,6 +26,6 @@ galaxy_info: issue_tracker_url: https://s.veen.world/cymaisissues documentation: https://s.veen.world/cymais dependencies: - - role: persona-gamer-retro - - role: persona-gamer-default - - role: persona-gamer-core \ No newline at end of file + - persona-gamer-retro + - persona-gamer-default + - persona-gamer-core \ No newline at end of file diff --git a/roles/systemd-notifier/meta/main.yml b/roles/systemd-notifier/meta/main.yml index 8f6804d2..bc7acab3 100644 --- a/roles/systemd-notifier/meta/main.yml +++ b/roles/systemd-notifier/meta/main.yml @@ -23,5 +23,5 @@ galaxy_info: issue_tracker_url: "https://s.veen.world/cymaisissues" documentation: "https://s.veen.world/cymais" dependencies: - - role: systemd-notifier-telegram - - role: systemd-notifier-email + - systemd-notifier-telegram + - systemd-notifier-email diff --git a/tasks/constructor.yml b/tasks/constructor.yml index 900675ad..29a2ba01 100644 --- a/tasks/constructor.yml +++ b/tasks/constructor.yml @@ -28,6 +28,24 @@ canonical_domains_map(primary_domain) | combine(domains | default({}, true), recursive=True) }} + + - name: Merge domain definitions for all domains + set_fact: + domains: >- + {{ + defaults_applications | + canonical_domains_map(primary_domain) | + combine(domains | default({}, true), recursive=True) + }} + + - name: Merge redirect_domain_mappings + set_fact: + # The following mapping is necessary to define the exceptions for domains which are created, but which aren't used + redirect_domain_mappings: "{{ + [] | + add_redirect_if_group('assets-server', domains | get_domain('assets-server'), domains | get_domain('file-server'), group_names) | + merge_mapping(redirect_domain_mappings, 'source') + }}" - name: Set current play redirect domain mappings set_fact: @@ -53,15 +71,6 @@ ) }} - - name: Merge domain definitions for all domains - set_fact: - domains: >- - {{ - defaults_applications | - canonical_domains_map(primary_domain) | - combine(domains | default({}, true), recursive=True) - }} - - name: Merge networks definitions set_fact: networks: "{{ defaults_networks | combine(networks | default({}, true), recursive=True) }}" diff --git a/tests/integration/test_no_circular_before_after_dependencies.py b/tests/integration/test_no_circular_before_after_dependencies.py index a14f1532..caebd63d 100644 --- a/tests/integration/test_no_circular_before_after_dependencies.py +++ b/tests/integration/test_no_circular_before_after_dependencies.py @@ -13,7 +13,7 @@ def get_meta_info(role_path): if not os.path.isfile(meta_file): return [], [] meta_data = load_yaml_file(meta_file) - run_order = meta_data.get('applications_run_order', {}) + run_order = meta_data.get('role_run_order', {}) before = run_order.get('before', []) after = run_order.get('after', []) return before, after diff --git a/tests/unit/test_redirect_filters.py b/tests/unit/test_redirect_filters.py new file mode 100644 index 00000000..aa87fbd1 --- /dev/null +++ b/tests/unit/test_redirect_filters.py @@ -0,0 +1,57 @@ +import os +import sys +import unittest + +sys.path.insert( + 0, + os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../") + ), +) + +from filter_plugins.redirect_filters import FilterModule + + +class TestAddRedirectIfGroup(unittest.TestCase): + """Unit-tests for the add_redirect_if_group filter.""" + + def setUp(self): + # Obtain the callable once for reuse + self.add_redirect = FilterModule().filters()["add_redirect_if_group"] + + def test_appends_redirect_when_group_present(self): + original = [{"source": "a", "target": "b"}] + result = self.add_redirect( + original, + group="lam", + source="ldap.example.com", + target="lam.example.com", + group_names=["lam", "other"], + ) + + # Original list must stay unchanged + self.assertEqual(len(original), 1) + # Result list must contain the extra entry + self.assertEqual(len(result), 2) + self.assertIn( + {"source": "ldap.example.com", "target": "lam.example.com"}, result + ) + + def test_keeps_list_unchanged_when_group_absent(self): + original = [{"source": "a", "target": "b"}] + result = self.add_redirect( + original, + group="lam", + source="ldap.example.com", + target="lam.example.com", + group_names=["unrelated"], + ) + + # No new entries + self.assertEqual(result, original) + # But ensure a new list object was returned (no in-place mutation) + self.assertIsNot(result, original) + + +if __name__ == "__main__": + unittest.main()