Successfully implemented matrix whatsapp bridge

group_vars/all

@@ -170,6 +170,7 @@ mastodon_single_user_mode:        false
 #### Matrix
 matrix_playbook_tags:   "setup-all,start" # For the initial update use: install-all,ensure-matrix-users-created,start
 matrix_role:            "compose"         # Role to setup Matrix. Valid values: ansible, compose
+matrix_admin_name:      "admin"
 
 #### Mailu
 version_mailu:                    "2.0"

roles/backup-docker-to-local/tasks/main.yml

@@ -46,24 +46,8 @@
     state: absent
   when: mode_reset | bool and run_once_backup_docker_to_local is not defined
 
-- name: Display all database variables
+- name: "include seed-database-to-backup.yml"
-  debug:
+  include_tasks: seed-database-to-backup.yml
-    msg: |
-      database_instance: "{{ database_instance | default('undefined') }}"
-      database_name: "{{ database_name | default('undefined') }}"
-      database_username: "{{ database_username | default('undefined') }}"
-      database_password: "{{ database_password | default('undefined') }}"
-  when: mode_verbose | bool
-
-- name: seed database values
-  command:
-    cmd: "python database_entry_seeder.py databases.csv {{database_instance}} {{database_name}} {{database_username}} {{database_password}}"
-    chdir: "{{backup_docker_to_local_folder}}"
-  when: >
-    database_instance is defined and
-    database_name is defined and
-    database_username is defined and
-    database_password is defined
 
 - name: Set file permissions for databases.csv to be readable, writable, and executable by root only
   ansible.builtin.file:

roles/docker-matrix-compose/tasks/main.yml

@@ -14,33 +14,40 @@
     domain:     "{{element_domain}}"
     http_port:  "{{element_http_port}}"
 
-#- name: "create {{docker_compose_instance_directory}}mautrix_whatsapp"
+- name: "create {{docker_compose_instance_directory}}mautrix_whatsapp"
-#  file:
+  file:
-#    path: "{{docker_compose_instance_directory}}mautrix_whatsapp"
+    path: "{{docker_compose_instance_directory}}mautrix_whatsapp"
-#    state: directory
+    state: directory
-#    mode: 0755
+    mode: 0755
-#
-#- name: add whatsapp-bridge-config.yml
-#  template: 
-#    src:  "whatsapp-bridge-config.yml.j2" 
-#    dest: "{{docker_compose_instance_directory}}mautrix_whatsapp/config.yml"
-#  notify: docker compose project setup
 
-- name: add element-config.json
+- name: "include create-and-seed-database.yml"
+  include_tasks: create-and-seed-database.yml
+  vars:
+    database_password:  "{{mautrix_whatsapp_bridge_database_password}}"
+    database_username:  "{{mautrix_whatsapp_bridge_database_username}}"
+    database_name:      "{{mautrix_whatsapp_bridge_database_name}}"
+
+- name: add mautrix-whatsapp configuration
   template: 
-    src:  "element-config.json.j2" 
+    src:  "mautrix-whatsapp.config.yml.j2" 
+    dest: "{{docker_compose_instance_directory}}mautrix_whatsapp/config.yaml"
+  notify: docker compose project setup
+
+- name: add element configuration
+  template: 
+    src:  "element.config.json.j2" 
     dest: "{{docker_compose_instance_directory}}element-config.json"
   notify: docker compose project setup
 
-- name: add homeserver.yaml
+- name: add synapse homeserver configuration
   template: 
-    src:  "homeserver.yaml.j2" 
+    src:  "synapse.homeserver.config.j2" 
     dest: "{{docker_compose_instance_directory}}homeserver.yaml"
   notify: docker compose project setup
 
-- name: add log.config
+- name: add synapse log configuration
   template: 
-    src:  "log.config.j2" 
+    src:  "synapse.log.config.j2" 
     dest: "{{docker_compose_instance_directory}}{{synapse_domain}}.log.config"
   notify: docker compose project setup
 
@@ -54,3 +61,21 @@
     src:  "docker-compose.yml.j2" 
     dest: "{{docker_compose_instance_directory}}docker-compose.yml"
   notify: docker compose project setup
+
+- name: flush docker service
+  meta: flush_handlers
+
+- name: create registration files
+  command:
+    cmd: docker compose exec -it mautrix-whatsapp /usr/bin/mautrix-whatsapp -g -c /data/config.yaml -r {{mautrix_registration_file}}
+    chdir: "{{docker_compose_instance_directory}}"
+
+- name: create registration files
+  command:
+    cmd: docker compose exec mautrix-whatsapp /usr/bin/mautrix-whatsapp -g -c /data/config.yaml -r {{ mautrix_registration_file }}
+    chdir: "{{ docker_compose_instance_directory }}"
+
+- name: set read and execute permissions for the registration file in the container
+  command:
+    cmd: docker compose exec mautrix-whatsapp chmod a+rx {{ mautrix_registration_file }}
+    chdir: "{{ docker_compose_instance_directory }}"

roles/docker-matrix-compose/templates/docker-compose.yml.j2

@@ -6,6 +6,7 @@ services:
 
   synapse:
     image: matrixdotorg/synapse:latest
+    container_name: mautrix-synapse
     restart: {{docker_restart_policy}}
     logging:
       driver: journald
@@ -13,6 +14,7 @@ services:
       - synapse_data:/data
       - ./homeserver.yaml:/data/homeserver.yaml:ro
       - ./{{synapse_domain}}.log.config:/data/{{synapse_domain}}.log.config:ro
+      - registration_files:{{registration_file_folder}}
     environment:
       - SYNAPSE_SERVER_NAME={{synapse_domain}}
       - SYNAPSE_REPORT_STATS=no
@@ -23,6 +25,7 @@ services:
   
   element:
     image: vectorim/element-web:latest
+    container_name: mautrix-element
     restart: {{docker_restart_policy}}
     volumes:
       - ./element-config.json:/app/config.json
@@ -38,12 +41,14 @@ services:
   #  volumes:
   #  - telegram_bridge_data:/data
 
-  #mautrix-whatsapp:
+  mautrix-whatsapp:
-  #  container_name: mautrix-whatsapp
+    container_name: mautrix-whatsapp
-  #  image: dock.mau.dev/mautrix/whatsapp:latest
+    image: dock.mau.dev/mautrix/whatsapp:latest
-  #  restart: {{docker_restart_policy}}
+    restart: {{docker_restart_policy}}
-  #  volumes:
+    volumes:
-  #  - ./mautrix_whatsapp:/data
+    - ./mautrix_whatsapp:/data
+    - registration_files:{{registration_file_folder}}
+{% include 'templates/docker-container-networks.yml.j2' %}
 
   #mautrix-facebook:
   #  container_name: mautrix-facebook
@@ -62,7 +67,8 @@ services:
 {% include 'templates/docker-compose-volumes.yml.j2' %}
   synapse_data:
   #telegram_bridge_data:
-  #whatsapp_bridge_data:
+  whatsapp_bridge_data:
+  registration_files:
   #facebook_bridge_data:
   #instagram_bridge_data:
 

roles/docker-matrix-compose/templates/mautrix-whatsapp.config.yml.j2

@@ -1,9 +1,9 @@
 # Homeserver details.
 homeserver:
     # The address that this appservice can use to connect to the homeserver.
-    address: https://{{synapse_domain}}
+    address: http://synapse:8008
-    # The domain of the homeserver (for MXIDs, etc).
+    # The domain of the homeserver (also known as server_name, used for MXIDs, etc).
-    domain: "{{synapse_domain}}"
+    domain: {{synapse_domain}}
     
     # What software is the homeserver running?
     # Standard Matrix homeservers like Synapse, Dendrite and Conduit should just use "standard" here.
@@ -27,7 +27,7 @@ homeserver:
 # Changing these values requires regeneration of the registration.
 appservice:
     # The address that the homeserver can use to connect to this appservice.
-    address: http://localhost:29318
+    address: http://mautrix-whatsapp:29318
     
     # The hostname and port where this appservice should listen.
     hostname: 0.0.0.0
@@ -42,7 +42,7 @@ appservice:
         #           https://github.com/mattn/go-sqlite3#connection-string
         #   Postgres: Connection string. For example, postgres://user:password@host/database?sslmode=disable
         #             To connect via Unix socket, use something like postgres:///dbname?host=/var/run/postgresql
-        uri: postgres://matrix:{{matrix_database_password}}@database/matrix?sslmode=disable
+        uri: postgres://{{mautrix_whatsapp_bridge_database_username}}:{{mautrix_whatsapp_bridge_database_password}}@{{database_host}}/{{mautrix_whatsapp_bridge_database_name}}?sslmode=disable
         # Maximum number of connections. Mostly relevant for Postgres.
         max_open_conns: 20
         max_idle_conns: 2
@@ -70,8 +70,8 @@ appservice:
     async_transactions: false
     
     # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
-    as_token: "GvOrixRxN8G2dO1KhUgWKL8yhC0lgSBtbr819nntLV2ZcPlWayjWj14YIVKMJHmA"
+    #as_token: "{{mautrix_whatsapp_bridge_as_token}}"
-    hs_token: "oQbRcd1KO2ggrF9lu13fElKnp1Zf0GDU9fiTFSwQQjnD3VmcRnVlZqhMpXu3criI"
+    #hs_token: "{{mautrix_whatsapp_bridge_hs_token}}"
 
 # Segment-compatible analytics endpoint for tracking some events, like provisioning API login and encryption errors.
 analytics:
@@ -100,6 +100,7 @@ whatsapp:
 
 # Bridge config
 bridge:
+{% raw %}
     # Localpart template of MXIDs for WhatsApp users.
     # {{.}} is replaced with the phone number of the WhatsApp user.
     username_template: whatsapp_{{.}}
@@ -111,6 +112,7 @@ bridge:
     # {{.FullName}}  - full name from contact list
     # {{.FirstName}} - first name from contact list
     displayname_template: "{{or .BusinessName .PushName .JID}} (WA)"
+{% endraw %}
     # Should the bridge create a space for each logged-in user and add bridged rooms to it?
     # Users who logged in before turning this on should run `!wa sync space` to create and fill the space for the first time.
     personal_filtering_spaces: false
@@ -234,7 +236,7 @@ bridge:
     force_active_delivery_receipts: false
     # Servers to always allow double puppeting from
     double_puppet_server_map:
-        example.com: https://example.com
+        {{synapse_domain}}: https://{{synapse_domain}}
     # Allow using double puppeting from any server with a valid client .well-known file.
     double_puppet_allow_discovery: false
     # Shared secrets for https://github.com/devture/matrix-synapse-shared-secret-auth
@@ -243,7 +245,7 @@ bridge:
     # instead of users having to find an access token and run `login-matrix`
     # manually.
     login_shared_secret_map:
-        example.com: foobar
+        {{synapse_domain}}: foobar
     # Whether to explicitly set the avatar and room name for private chat portal rooms.
     # If set to `default`, this will be enabled in encrypted rooms and disabled in unencrypted rooms.
     # If set to `always`, all DM rooms will have explicit names and avatars set.
@@ -417,7 +419,7 @@ bridge:
         prefix: /_matrix/provision
         # Shared secret for authentication. If set to "generate", a random secret will be generated,
         # or if set to "disable", the provisioning API will be disabled.
-        shared_secret: kY8Ez6M80YBpvGtMz5R1Mb1c7deb8d9zxmlvqwTv0HGCKPVqcrTmHu90Wi04828z
+        shared_secret: {{mautrix_whatsapp_bridge_provisioning_shared_secret}}
         # Enable debug API at /debug with provisioning authentication.
         debug_endpoints: false
     
@@ -432,8 +434,8 @@ bridge:
     #     mxid - Specific user
     permissions:
         "*": relay
-        "example.com": user
+        "{{synapse_domain}}": user
-        "@admin:example.com": admin
+        "@{{matrix_admin_name}}:{{synapse_domain}}": admin
     
     # Settings for relay mode
     relay:
@@ -444,6 +446,7 @@ bridge:
         admin_only: true
         # The formats to use when sending messages to WhatsApp via the relaybot.
         message_formats:
+{% raw %}
             m.text: "<b>{{ .Sender.Displayname }}</b>: {{ .Message }}"
             m.notice: "<b>{{ .Sender.Displayname }}</b>: {{ .Message }}"
             m.emote: "* <b>{{ .Sender.Displayname }}</b> {{ .Message }}"
@@ -452,10 +455,11 @@ bridge:
             m.audio: "<b>{{ .Sender.Displayname }}</b> sent an audio file"
             m.video: "<b>{{ .Sender.Displayname }}</b> sent a video"
             m.location: "<b>{{ .Sender.Displayname }}</b> sent a location"
+{% endraw %}
 
 # Logging config. See https://github.com/tulir/zeroconfig for details.
 logging:
     min_level: debug
     writers:
         - type: stdout
-          format: pretty-colored
+          format: pretty-colored

roles/docker-matrix-compose/templates/synapse.homeserver.config.j2

@@ -43,4 +43,7 @@ email:
   enable_notifs:                true
   notif_for_new_users:          false
   client_base_url:              "{{domain_matrix_synapse}}"
-  validation_token_lifetime:    15m
+  validation_token_lifetime:    15m
+
+app_service_config_files:
+  - "{{mautrix_registration_file}}"

roles/docker-matrix-compose/vars/main.yml

@@ -1,4 +1,9 @@
 ---
-docker_compose_project_name:  	    "matrix"
+docker_compose_project_name:  	            "matrix"
-database_password:  	              "{{matrix_database_password}}"
+database_password:  	                      "{{matrix_database_password}}"
-database_type:                      "postgres"
+database_type:                              "postgres"
+registration_file_folder:                   "/data/registration/"
+mautrix_whatsapp_bridge_database_name:      "mautrix_whatsapp_bridge"
+mautrix_whatsapp_bridge_database_username:  "mautrix_whatsapp_bridge"
+mautrix_registration_file:                  "{{registration_file_folder}}mautrix-whatsapp.registration.yaml"
+

roles/docker-openproject/files/Dockerfile

@@ -0,0 +1,14 @@
+FROM openproject/community:13
+
+# If installing a local plugin (using `path:` in the `Gemfile.plugins` above),
+# you will have to copy the plugin code into the container here and use the
+# path inside of the container. Say for `/app/vendor/plugins/openproject-slack`:
+# COPY /path/to/my/local/openproject-slack /app/vendor/plugins/openproject-slack
+
+COPY Gemfile.plugins /app/
+
+# If the plugin uses any external NPM dependencies you have to install them here.
+# RUN npm add npm <package-name>*
+
+RUN bundle config unset deployment && bundle install && bundle config set deployment 'true'
+RUN ./docker/prod/setup/postinstall.sh

roles/docker-openproject/files/Gemfile.plugins

@@ -0,0 +1,3 @@
+group :opf_plugins do
+  gem "openproject-gitlab-integration", git: "openproject-gitlab_integration", branch: "master"
+end

roles/docker-openproject/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+- name: rebuild docker image
+  command:
+    cmd: docker build --no-cache -t custom_openproject .
+    chdir: "{{docker_compose_instance_directory}}"
+  environment:
+    COMPOSE_HTTP_TIMEOUT: 600
+    DOCKER_CLIENT_TIMEOUT: 600

roles/docker-openproject/tasks/main.yml

@@ -8,6 +8,23 @@
 - name: "include tasks update-repository-with-docker-compose.yml"
   include_tasks: update-repository-with-docker-compose.yml
 
+
+- name: "Transfering Gemfile.plugins to {{docker_compose_instance_directory}}"
+  copy:
+    src: Gemfile.plugins
+    dest: "{{docker_compose_instance_directory}}Gemfile.plugins"
+  notify:
+    - docker compose project setup
+    - rebuild docker image
+
+- name: "Transfering Dockerfile to {{docker_compose_instance_directory}}"
+  copy:
+    src: Dockerfile
+    dest: "{{docker_compose_instance_directory}}Dockerfile"
+  notify:
+    - docker compose project setup
+    - rebuild docker image
+
 - name: "create {{dummy_volume}}"
   file:
     path: "{{dummy_volume}}"

roles/docker-openproject/templates/docker-compose.yml.j2

@@ -2,7 +2,7 @@ version: "3.7"
   
 x-op-app: &app
   restart: {{docker_restart_policy}}
-  image: openproject/community:${TAG:-13}
+  image: custom_openproject
   environment:
     OPENPROJECT_HTTPS: "${OPENPROJECT_HTTPS}"
     OPENPROJECT_HOST__NAME: "${OPENPROJECT_HOST__NAME}"
@@ -25,12 +25,14 @@ services:
 
   cache:
     image: memcached
+    container_name: openproject-memcached
     restart: {{docker_restart_policy}}
 {% include 'templates/docker-container-networks.yml.j2' %}
 
   proxy:
     restart: {{docker_restart_policy}}
-    image: openproject/community:${TAG:-13}
+    image: custom_openproject
+    container_name: openproject-proxy
     command: "./docker/prod/proxy"
     ports:
       - "${PORT}:80"
@@ -47,6 +49,7 @@ services:
   web:
     <<: *app
     command: "./docker/prod/web"
+    container_name: openproject-web
 {% include 'templates/docker-container-networks.yml.j2' %}
 {% include 'templates/docker-container-depends-on-also-database.yml.j2' %}
       cache:
@@ -64,6 +67,7 @@ services:
 
   autoheal:
     image: willfarrell/autoheal:1.2.0
+    container_name: openproject-autoheal
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock"
     environment:
@@ -74,6 +78,7 @@ services:
   worker:
     <<: *app
     command: "./docker/prod/worker"
+    container_name: openproject-worker
 {% include 'templates/docker-container-networks.yml.j2' %}
 {% include 'templates/docker-container-depends-on-also-database.yml.j2' %}
       cache:
@@ -84,6 +89,7 @@ services:
   cron:
     <<: *app
     command: "./docker/prod/cron"
+    container_name: openproject-cron
 {% include 'templates/docker-container-networks.yml.j2' %}
 {% include 'templates/docker-container-depends-on-also-database.yml.j2' %}
       cache:
@@ -94,6 +100,7 @@ services:
   seeder:
     <<: *app
     command: "./docker/prod/seeder"
+    container_name: openproject-seeder
     restart: on-failure
 {% include 'templates/docker-container-networks.yml.j2' %}
 

roles/docker-openproject/templates/env.j2

@@ -6,7 +6,6 @@
 # Please refer to our documentation to see all possible variables:
 #   https://www.openproject.org/docs/installation-and-operations/configuration/environment/
 #
-TAG=13
 OPENPROJECT_HTTPS=true
 OPENPROJECT_HOST__NAME={{domain}}
 PORT=127.0.0.1:{{http_port}}

roles/docker-wordpress/templates/docker-compose.yml.j2

@@ -8,6 +8,7 @@ services:
     logging:
       driver: journald
     image: custom_wordpress
+    container_name: wordpress-application
     build:
       context: .
     restart: {{docker_restart_policy}}

tasks/create-and-seed-database.yml

@@ -0,0 +1,13 @@
+# The following parameters need to be based:
+# 
+# - database_instance
+# - database_name
+# - database_username
+# - database_password
+- name: "create {{database_name}} database"
+  include_role:
+    name: docker-postgres
+  when: enable_central_database | bool
+
+- name: "include seed-database-to-backup.yml"
+  include_tasks: seed-database-to-backup.yml

tasks/seed-database-to-backup.yml

@@ -0,0 +1,18 @@
+- name: Display all database variables
+  debug:
+    msg: |
+      database_instance: "{{ database_instance | default('undefined') }}"
+      database_name: "{{ database_name | default('undefined') }}"
+      database_username: "{{ database_username | default('undefined') }}"
+      database_password: "{{ database_password | default('undefined') }}"
+  when: mode_verbose | bool
+
+- name: seed database values
+  command:
+    cmd: "python database_entry_seeder.py databases.csv {{database_instance}} {{database_name}} {{database_username}} {{database_password}}"
+    chdir: "{{backup_docker_to_local_folder}}"
+  when: >
+    database_instance is defined and
+    database_name is defined and
+    database_username is defined and
+    database_password is defined