From 76aef5949b1e0d906fbbe68981b510e3ff0098be Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 15 May 2025 21:11:21 +0200 Subject: [PATCH] Optimized Docker Matrix Role in Preparation for use on CyMaIS.Cloud Server --- filter_plugins/__init__.py | 2 + group_vars/all/00_general.yml | 1 + roles/docker-gitea/templates/env.j2 | 2 +- roles/docker-keycloak/vars/configuration.yml | 2 + roles/docker-matrix/Todo.md | 5 ++ .../docker-matrix/filter_plugins/__init__.py | 2 + .../filter_plugins/bridge_filters.py | 13 +++ roles/docker-matrix/tasks/main.yml | 9 ++ .../templates/docker-compose.yml.j2 | 86 ++++++++++--------- .../templates/synapse/homeserver.yaml.j2 | 4 +- roles/docker-matrix/vars/bridges.yml | 30 +++++++ roles/docker-matrix/vars/configuration.yml | 12 ++- roles/docker-matrix/vars/main.yml | 37 +------- roles/docker-openproject/tasks/ldap.yml | 2 +- roles/docker-openproject/tasks/main.yml | 2 +- .../templates/update-docker.py.j2 | 2 +- tests/unit/test_bridge_filters.py | 64 ++++++++++++++ tests/unit/test_configuration_filters.py | 10 +++ tests/unit/test_csp_filters.py | 10 +++ tests/unit/test_redirect_filters.py | 8 +- 20 files changed, 219 insertions(+), 84 deletions(-) create mode 100644 roles/docker-matrix/Todo.md create mode 100644 roles/docker-matrix/filter_plugins/__init__.py create mode 100644 roles/docker-matrix/filter_plugins/bridge_filters.py create mode 100644 roles/docker-matrix/vars/bridges.yml create mode 100644 tests/unit/test_bridge_filters.py diff --git a/filter_plugins/__init__.py b/filter_plugins/__init__.py index e69de29b..0bfb5a62 100644 --- a/filter_plugins/__init__.py +++ b/filter_plugins/__init__.py @@ -0,0 +1,2 @@ +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) \ No newline at end of file diff --git a/group_vars/all/00_general.yml b/group_vars/all/00_general.yml index c59f1339..3df78cf4 100644 --- a/group_vars/all/00_general.yml +++ b/group_vars/all/00_general.yml @@ -1,3 +1,4 @@ +CYMAIS_ENVIRONMENT: "production" HOST_CURRENCY: "EUR" HOST_TIMEZONE: "UTC" diff --git a/roles/docker-gitea/templates/env.j2 b/roles/docker-gitea/templates/env.j2 index b28a7905..478cc093 100644 --- a/roles/docker-gitea/templates/env.j2 +++ b/roles/docker-gitea/templates/env.j2 @@ -12,7 +12,7 @@ SSH_PORT={{ports.public.ssh[application_id]}} SSH_LISTEN_PORT=22 DOMAIN={{domains[application_id]}} SSH_DOMAIN={{domains[application_id]}} -RUN_MODE="{{run_mode}}" +RUN_MODE="{{ 'dev' if (CYMAIS_ENVIRONMENT | lower) == 'development' else 'prod' }}" ROOT_URL="{{ web_protocol }}://{{domains[application_id]}}/" # Mail Configuration diff --git a/roles/docker-keycloak/vars/configuration.yml b/roles/docker-keycloak/vars/configuration.yml index 4d1275bd..ae28a135 100644 --- a/roles/docker-keycloak/vars/configuration.yml +++ b/roles/docker-keycloak/vars/configuration.yml @@ -14,4 +14,6 @@ features: csp: flags: script-src: + unsafe-inline: true + style-src: unsafe-inline: true \ No newline at end of file diff --git a/roles/docker-matrix/Todo.md b/roles/docker-matrix/Todo.md new file mode 100644 index 00000000..f99aceb0 --- /dev/null +++ b/roles/docker-matrix/Todo.md @@ -0,0 +1,5 @@ +# Todo +- Enable Whatsapp by default +- Enable Telegram by default +- Enable Slack by default +- Enable ChatGPT by default \ No newline at end of file diff --git a/roles/docker-matrix/filter_plugins/__init__.py b/roles/docker-matrix/filter_plugins/__init__.py new file mode 100644 index 00000000..0bfb5a62 --- /dev/null +++ b/roles/docker-matrix/filter_plugins/__init__.py @@ -0,0 +1,2 @@ +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) \ No newline at end of file diff --git a/roles/docker-matrix/filter_plugins/bridge_filters.py b/roles/docker-matrix/filter_plugins/bridge_filters.py new file mode 100644 index 00000000..3c738ee0 --- /dev/null +++ b/roles/docker-matrix/filter_plugins/bridge_filters.py @@ -0,0 +1,13 @@ +def filter_enabled_bridges(bridges, plugins): + """ + Return only those bridge definitions whose 'bridge_name' is set to True in plugins. + :param bridges: list of dicts, each with a 'bridge_name' key + :param plugins: dict mapping bridge_name to a boolean + """ + return [b for b in bridges if plugins.get(b['bridge_name'], False)] + +class FilterModule(object): + def filters(self): + return { + 'filter_enabled_bridges': filter_enabled_bridges, + } diff --git a/roles/docker-matrix/tasks/main.yml b/roles/docker-matrix/tasks/main.yml index 729fe1a2..f23b699d 100644 --- a/roles/docker-matrix/tasks/main.yml +++ b/roles/docker-matrix/tasks/main.yml @@ -1,4 +1,13 @@ --- +- name: Load bridges configuration + include_vars: + file: "bridges.yml" + +- name: Filter enabled bridges and register as fact + set_fact: + bridges: "{{ bridges_configuration | filter_enabled_bridges(applications[application_id].plugins) }}" + changed_when: false + - name: "include docker-central-database" include_role: name: docker-central-database diff --git a/roles/docker-matrix/templates/docker-compose.yml.j2 b/roles/docker-matrix/templates/docker-compose.yml.j2 index 92080ba8..05121070 100644 --- a/roles/docker-matrix/templates/docker-compose.yml.j2 +++ b/roles/docker-matrix/templates/docker-compose.yml.j2 @@ -25,7 +25,11 @@ services: interval: 1m timeout: 10s retries: 3 +{% if bridges | bool %} {% include 'templates/docker/container/depends-on-also-database.yml.j2' %} +{% else %} +{% include 'templates/docker/container/depends-on-just-database.yml.j2' %} +{% endif %} {% for item in bridges %} mautrix-{{item.bridge_name}}: condition: service_healthy @@ -61,47 +65,47 @@ services: retries: 3 {% include 'templates/docker/container/networks.yml.j2' %} {% endfor %} -# Deactivated chatgpt. -# @todo needs to be reactivated as soon as bug is found -# matrix-chatgpt-bot: -# restart: {{docker_restart_policy}} -# container_name: matrix-chatgpt -# image: ghcr.io/matrixgpt/matrix-chatgpt-bot:latest -# volumes: -# - chatgpt_data:/storage -# environment: -# OPENAI_API_KEY: '{{applications[application_id].credentials.chatgpt_bridge_openai_api_key}}' -# # Uncomment the next two lines if you are using Azure OpenAI API -# # OPENAI_AZURE: 'false' -# # CHATGPT_REVERSE_PROXY: 'your-completion-endpoint-here' -# CHATGPT_CONTEXT: 'thread' -# CHATGPT_API_MODEL: 'gpt-3.5-turbo' -# # Uncomment and edit the next line if needed -# # CHATGPT_PROMPT_PREFIX: 'Instructions:\nYou are ChatGPT, a large language model trained by OpenAI.' -# # CHATGPT_IGNORE_MEDIA: 'false' -# CHATGPT_REVERSE_PROXY: 'https://api.openai.com/v1/chat/completions' -# # Uncomment and edit the next line if needed -# # CHATGPT_TEMPERATURE: '0.8' -# # Uncomment and edit the next line if needed -# #CHATGPT_MAX_CONTEXT_TOKENS: '4097' -# CHATGPT_MAX_PROMPT_TOKENS: '3000' -# KEYV_BACKEND: 'file' -# KEYV_URL: '' -# KEYV_BOT_ENCRYPTION: 'false' -# KEYV_BOT_STORAGE: 'true' -# MATRIX_HOMESERVER_URL: 'https://{{domains.synapse}}' -# MATRIX_BOT_USERNAME: '@chatgptbot:{{applications.matrix.server_name}}' -# MATRIX_ACCESS_TOKEN: '{{ applications[application_id].credentials.chatgpt_bridge_access_token | default('') }}' -# MATRIX_BOT_PASSWORD: '{{applications[application_id].credentials.chatgpt_bridge_user_password}}' -# MATRIX_DEFAULT_PREFIX: '!chatgpt' -# MATRIX_DEFAULT_PREFIX_REPLY: 'false' -# #MATRIX_BLACKLIST: '' -# MATRIX_WHITELIST: ':{{applications.matrix.server_name}}' -# MATRIX_AUTOJOIN: 'true' -# MATRIX_ENCRYPTION: 'true' -# MATRIX_THREADS: 'true' -# MATRIX_PREFIX_DM: 'false' -# MATRIX_RICH_TEXT: 'true' +{% if applications[application_id] | bool %} + matrix-chatgpt-bot: + restart: {{docker_restart_policy}} + container_name: matrix-chatgpt + image: ghcr.io/matrixgpt/matrix-chatgpt-bot:latest + volumes: + - chatgpt_data:/storage + environment: + OPENAI_API_KEY: '{{applications[application_id].credentials.chatgpt_bridge_openai_api_key}}' + # Uncomment the next two lines if you are using Azure OpenAI API + # OPENAI_AZURE: 'false' + # CHATGPT_REVERSE_PROXY: 'your-completion-endpoint-here' + CHATGPT_CONTEXT: 'thread' + CHATGPT_API_MODEL: 'gpt-3.5-turbo' + # Uncomment and edit the next line if needed + # CHATGPT_PROMPT_PREFIX: 'Instructions:\nYou are ChatGPT, a large language model trained by OpenAI.' + # CHATGPT_IGNORE_MEDIA: 'false' + CHATGPT_REVERSE_PROXY: 'https://api.openai.com/v1/chat/completions' + # Uncomment and edit the next line if needed + # CHATGPT_TEMPERATURE: '0.8' + # Uncomment and edit the next line if needed + #CHATGPT_MAX_CONTEXT_TOKENS: '4097' + CHATGPT_MAX_PROMPT_TOKENS: '3000' + KEYV_BACKEND: 'file' + KEYV_URL: '' + KEYV_BOT_ENCRYPTION: 'false' + KEYV_BOT_STORAGE: 'true' + MATRIX_HOMESERVER_URL: 'https://{{domains.synapse}}' + MATRIX_BOT_USERNAME: '@chatgptbot:{{applications.matrix.server_name}}' + MATRIX_ACCESS_TOKEN: '{{ applications[application_id].credentials.chatgpt_bridge_access_token | default('') }}' + MATRIX_BOT_PASSWORD: '{{applications[application_id].credentials.chatgpt_bridge_user_password}}' + MATRIX_DEFAULT_PREFIX: '!chatgpt' + MATRIX_DEFAULT_PREFIX_REPLY: 'false' + #MATRIX_BLACKLIST: '' + MATRIX_WHITELIST: ':{{applications.matrix.server_name}}' + MATRIX_AUTOJOIN: 'true' + MATRIX_ENCRYPTION: 'true' + MATRIX_THREADS: 'true' + MATRIX_PREFIX_DM: 'false' + MATRIX_RICH_TEXT: 'true' +{% endif %} {% include 'templates/docker/compose/volumes.yml.j2' %} synapse_data: diff --git a/roles/docker-matrix/templates/synapse/homeserver.yaml.j2 b/roles/docker-matrix/templates/synapse/homeserver.yaml.j2 index b77ed1fa..5d334776 100644 --- a/roles/docker-matrix/templates/synapse/homeserver.yaml.j2 +++ b/roles/docker-matrix/templates/synapse/homeserver.yaml.j2 @@ -61,7 +61,9 @@ oidc_providers: backchannel_logout_enabled: true {% endif %} +{% if bridges | bool %} app_service_config_files: {% for item in bridges %} - {{registration_file_folder}}{{item.bridge_name}}.registration.yaml -{% endfor %} \ No newline at end of file +{% endfor %} +{% endif %} \ No newline at end of file diff --git a/roles/docker-matrix/vars/bridges.yml b/roles/docker-matrix/vars/bridges.yml new file mode 100644 index 00000000..1f950cdd --- /dev/null +++ b/roles/docker-matrix/vars/bridges.yml @@ -0,0 +1,30 @@ +bridges_configuration: + - database_password: "{{ applications[application_id].credentials.mautrix_whatsapp_bridge_database_password }}" + database_username: "mautrix_whatsapp_bridge" + database_name: "mautrix_whatsapp_bridge" + bridge_name: "whatsapp" + + - database_password: "{{ applications[application_id].credentials.mautrix_telegram_bridge_database_password }}" + database_username: "mautrix_telegram_bridge" + database_name: "mautrix_telegram_bridge" + bridge_name: "telegram" + + - database_password: "{{ applications[application_id].credentials.mautrix_signal_bridge_database_password }}" + database_username: "mautrix_signal_bridge" + database_name: "mautrix_signal_bridge" + bridge_name: "signal" + + - database_password: "{{ applications[application_id].credentials.mautrix_slack_bridge_database_password }}" + database_username: "mautrix_slack_bridge" + database_name: "mautrix_slack_bridge" + bridge_name: "slack" + + - database_password: "{{ applications[application_id].credentials.mautrix_facebook_bridge_database_password }}" + database_username: "mautrix_facebook_bridge" + database_name: "mautrix_facebook_bridge" + bridge_name: "facebook" + + - database_password: "{{ applications[application_id].credentials.mautrix_instagram_bridge_database_password }}" + database_username: "mautrix_instagram_bridge" + database_name: "mautrix_instagram_bridge" + bridge_name: "instagram" \ No newline at end of file diff --git a/roles/docker-matrix/vars/configuration.yml b/roles/docker-matrix/vars/configuration.yml index 61986228..d0e41419 100644 --- a/roles/docker-matrix/vars/configuration.yml +++ b/roles/docker-matrix/vars/configuration.yml @@ -3,7 +3,6 @@ users: administrator: username: "{{users.administrator.username}}" # Accountname of the matrix admin playbook_tags: "setup-all,start" # For the initial update use: install-all,ensure-matrix-users-created,start -role: "compose" # Role to setup Matrix. Valid values: ansible, compose server_name: "{{primary_domain}}" # Adress for the account names etc. synapse: version: "latest" @@ -30,3 +29,14 @@ csp: script-src: - "{{ domains.synapse }}" - "https://cdn.jsdelivr.net" +plugins: +# You need to enable them in the inventory file + chatgpt: false + facebook: false + immesage: false + instagram: false + signal: false + slack: false + telegram: false + whatsapp: false + diff --git a/roles/docker-matrix/vars/main.yml b/roles/docker-matrix/vars/main.yml index 012963a4..318e5012 100644 --- a/roles/docker-matrix/vars/main.yml +++ b/roles/docker-matrix/vars/main.yml @@ -1,38 +1,5 @@ --- -application_id: "matrix" +application_id: "matrix" database_type: "postgres" registration_file_folder: "/data/" -well_known_directory: "{{nginx.directories.data.well_known}}/matrix/" - -bridges: - - database_password: "{{ applications[application_id].credentials.mautrix_whatsapp_bridge_database_password }}" - database_username: "mautrix_whatsapp_bridge" - database_name: "mautrix_whatsapp_bridge" - bridge_name: "whatsapp" - - - database_password: "{{ applications[application_id].credentials.mautrix_telegram_bridge_database_password }}" - database_username: "mautrix_telegram_bridge" - database_name: "mautrix_telegram_bridge" - bridge_name: "telegram" - - - database_password: "{{ applications[application_id].credentials.mautrix_signal_bridge_database_password }}" - database_username: "mautrix_signal_bridge" - database_name: "mautrix_signal_bridge" - bridge_name: "signal" - -# Deactivated temporary, due to bug which is hard to find -# @todo Reactivate -# - database_password: "{{ applications[application_id].credentials.mautrix_slack_bridge_database_password }}" -# database_username: "mautrix_slack_bridge" -# database_name: "mautrix_slack_bridge" -# bridge_name: "slack" - - - database_password: "{{ applications[application_id].credentials.mautrix_facebook_bridge_database_password }}" - database_username: "mautrix_facebook_bridge" - database_name: "mautrix_facebook_bridge" - bridge_name: "facebook" - - - database_password: "{{ applications[application_id].credentials.mautrix_instagram_bridge_database_password }}" - database_username: "mautrix_instagram_bridge" - database_name: "mautrix_instagram_bridge" - bridge_name: "instagram" \ No newline at end of file +well_known_directory: "{{nginx.directories.data.well_known}}/matrix/" \ No newline at end of file diff --git a/roles/docker-openproject/tasks/ldap.yml b/roles/docker-openproject/tasks/ldap.yml index d9a6e6fd..9a529a7f 100644 --- a/roles/docker-openproject/tasks/ldap.yml +++ b/roles/docker-openproject/tasks/ldap.yml @@ -96,7 +96,7 @@ shell: > docker compose exec web bash -c " cd /app && - RAILS_ENV=production bundle exec rails runner \" + RAILS_ENV={{ CYMAIS_ENVIRONMENT | lower }} bundle exec rails runner \" user = User.find_by(mail: '{{ users.administrator.email }}'); if user.nil?; puts 'User with email {{ users.administrator.email }} not found.'; diff --git a/roles/docker-openproject/tasks/main.yml b/roles/docker-openproject/tasks/main.yml index a0029ce6..7f62780c 100644 --- a/roles/docker-openproject/tasks/main.yml +++ b/roles/docker-openproject/tasks/main.yml @@ -49,7 +49,7 @@ - name: Set settings in OpenProject shell: > docker compose exec web bash -c "cd /app && - RAILS_ENV=production bundle exec rails runner \"Setting[:{{ item.key }}] = '{{ item.value }}'\"" + RAILS_ENV={{ CYMAIS_ENVIRONMENT | lower }} bundle exec rails runner \"Setting[:{{ item.key }}] = '{{ item.value }}'\"" args: chdir: "{{ docker_compose.directories.instance }}" loop: "{{ openproject_rails_settings | dict2items }}" diff --git a/roles/update-docker/templates/update-docker.py.j2 b/roles/update-docker/templates/update-docker.py.j2 index 0e11e6cc..e64a8f71 100644 --- a/roles/update-docker/templates/update-docker.py.j2 +++ b/roles/update-docker/templates/update-docker.py.j2 @@ -149,7 +149,7 @@ def update_mastodon(): Runs the database migration for Mastodon to ensure all required tables are up to date. """ print("Starting Mastodon database migration.") - run_command("docker compose exec -T web bash -c 'RAILS_ENV=production bin/rails db:migrate'") + run_command("docker compose exec -T web bash -c 'RAILS_ENV={{ CYMAIS_ENVIRONMENT | lower }} bin/rails db:migrate'") print("Mastodon database migration complete.") def upgrade_listmonk(): diff --git a/tests/unit/test_bridge_filters.py b/tests/unit/test_bridge_filters.py new file mode 100644 index 00000000..2a882fdb --- /dev/null +++ b/tests/unit/test_bridge_filters.py @@ -0,0 +1,64 @@ +import os +import sys +import unittest + +# Add the filter_plugins directory from the docker-matrix role to the import path +sys.path.insert( + 0, + os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../roles/docker-matrix") + ), +) + +from filter_plugins.bridge_filters import filter_enabled_bridges + + +class TestBridgeFilters(unittest.TestCase): + def test_no_bridges_returns_empty_list(self): + # When there are no bridges defined, result should be an empty list + self.assertEqual(filter_enabled_bridges([], {}), []) + + def test_all_bridges_disabled(self): + # Define two bridges, but plugins dict has them disabled or missing + bridges = [ + {'bridge_name': 'whatsapp', 'config': {}}, + {'bridge_name': 'telegram', 'config': {}}, + ] + plugins = { + 'whatsapp': False, + 'telegram': False, + } + result = filter_enabled_bridges(bridges, plugins) + self.assertEqual(result, []) + + def test_some_bridges_enabled(self): + # Only bridges with True in plugins should be returned + bridges = [ + {'bridge_name': 'whatsapp', 'version': '1.0'}, + {'bridge_name': 'telegram', 'version': '1.0'}, + {'bridge_name': 'signal', 'version': '1.0'}, + ] + plugins = { + 'whatsapp': True, + 'telegram': False, + 'signal': True, + } + result = filter_enabled_bridges(bridges, plugins) + expected = [ + {'bridge_name': 'whatsapp', 'version': '1.0'}, + {'bridge_name': 'signal', 'version': '1.0'}, + ] + self.assertEqual(result, expected) + + def test_bridge_without_plugin_entry_defaults_to_disabled(self): + # If a bridge_name is not present in plugins, it should be treated as disabled + bridges = [ + {'bridge_name': 'facebook', 'enabled': True}, + ] + plugins = {} # no entries + result = filter_enabled_bridges(bridges, plugins) + self.assertEqual(result, []) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/unit/test_configuration_filters.py b/tests/unit/test_configuration_filters.py index b191b1f9..08aaf76b 100644 --- a/tests/unit/test_configuration_filters.py +++ b/tests/unit/test_configuration_filters.py @@ -1,6 +1,16 @@ # tests/unit/test_configuration_filters.py import unittest +import sys +import os + +sys.path.insert( + 0, + os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../") + ), +) + from filter_plugins.configuration_filters import ( is_feature_enabled, ) diff --git a/tests/unit/test_csp_filters.py b/tests/unit/test_csp_filters.py index 17cc5eb0..a03da385 100644 --- a/tests/unit/test_csp_filters.py +++ b/tests/unit/test_csp_filters.py @@ -1,6 +1,16 @@ import unittest import hashlib import base64 +import sys +import os + +sys.path.insert( + 0, + os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../") + ), +) + from filter_plugins.csp_filters import FilterModule, AnsibleFilterError class TestCspFilters(unittest.TestCase): diff --git a/tests/unit/test_redirect_filters.py b/tests/unit/test_redirect_filters.py index 3283dff3..aa87fbd1 100644 --- a/tests/unit/test_redirect_filters.py +++ b/tests/unit/test_redirect_filters.py @@ -2,8 +2,12 @@ import os import sys import unittest -PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) -sys.path.insert(0, PROJECT_ROOT) +sys.path.insert( + 0, + os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../") + ), +) from filter_plugins.redirect_filters import FilterModule