feat(postgres): add split_postgres_connections filter and average pool fact

Compute POSTGRES_ALLOWED_AVG_CONNECTIONS once and propagate to app roles (gitlab, mastodon, listmonk, matrix, pretix, mobilizon, openproject, discourse). Fix docker-compose postgres command (-c flags split). Add unit tests. Minor env/locale tweaks and includes.

Conversation: https://chatgpt.com/share/68b48e72-cc28-800f-9c21-270cbc17d82a

roles/svc-db-postgres/filter_plugins/split_postgres_connections.py

@@ -0,0 +1,58 @@
+import os
+import yaml
+from ansible.errors import AnsibleFilterError
+
+def _iter_role_vars_files(roles_dir):
+    if not os.path.isdir(roles_dir):
+        raise AnsibleFilterError(f"roles_dir not found: {roles_dir}")
+    for name in os.listdir(roles_dir):
+        role_path = os.path.join(roles_dir, name)
+        if not os.path.isdir(role_path):
+            continue
+        vars_main = os.path.join(role_path, "vars", "main.yml")
+        if os.path.isfile(vars_main):
+            yield vars_main
+
+def _is_postgres_role(vars_file):
+    try:
+        with open(vars_file, "r", encoding="utf-8") as f:
+            data = yaml.safe_load(f) or {}
+        # only count roles with explicit database_type: postgres in VARS
+        return str(data.get("database_type", "")).strip().lower() == "postgres"
+    except Exception:
+        # ignore unreadable/broken YAML files quietly
+        return False
+
+def split_postgres_connections(total_connections, roles_dir="roles"):
+    """
+    Return an integer average: total_connections / number_of_roles_with_database_type_postgres.
+    Uses max(count, 1) to avoid division-by-zero.
+    """
+    try:
+        total = int(total_connections)
+    except Exception:
+        raise AnsibleFilterError(f"total_connections must be int-like, got: {total_connections!r}")
+
+    count = sum(1 for vf in _iter_role_vars_files(roles_dir) if _is_postgres_role(vf))
+    denom = max(count, 1)
+    return max(1, total // denom)
+
+def list_postgres_roles(roles_dir="roles"):
+    """
+    Helper: return a list of role names that declare database_type: postgres in vars/main.yml.
+    """
+    names = []
+    if not os.path.isdir(roles_dir):
+        return names
+    for name in os.listdir(roles_dir):
+        vars_main = os.path.join(roles_dir, name, "vars", "main.yml")
+        if os.path.isfile(vars_main) and _is_postgres_role(vars_main):
+            names.append(name)
+    return names
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            "split_postgres_connections": split_postgres_connections,
+            "list_postgres_roles": list_postgres_roles,
+        }

roles/svc-db-postgres/tasks/01_core.yml

@@ -1,3 +1,7 @@
+- name: Compute average allowed connections per Postgres app (once)
+  set_fact:
+    POSTGRES_ALLOWED_AVG_CONNECTIONS: "{{ (POSTGRES_MAX_CONNECTIONS | split_postgres_connections(playbook_dir ~ '/roles')) | int }}"
+  run_once: true
 
 - name: Include dependency 'sys-svc-docker'
   include_role:

roles/svc-db-postgres/templates/docker-compose.yml.j2

@@ -7,6 +7,19 @@
       context: .
       dockerfile: Dockerfile
     pull_policy: never
+    command:
+      - "postgres"
+      - "-c"
+      - "max_connections={{ POSTGRES_MAX_CONNECTIONS }}"
+      - "-c"
+      - "superuser_reserved_connections={{ POSTGRES_SUPERUSER_RESERVED_CONNECTIONS }}"
+      - "-c"
+      - "shared_buffers={{ POSTGRES_SHARED_BUFFERS }}"
+      - "-c"
+      - "work_mem={{ POSTGRES_WORK_MEM }}"
+      - "-c"
+      - "maintenance_work_mem={{ POSTGRES_MAINTENANCE_WORK_MEM }}"
+
 {% include 'roles/docker-container/templates/base.yml.j2' %}
 {% if POSTGRES_EXPOSE_LOCAL %}
     ports: 

roles/svc-db-postgres/vars/main.yml

@@ -1,25 +1,37 @@
 # General
-application_id:                 svc-db-postgres
+application_id:                           svc-db-postgres
 
 # Docker
-docker_compose_flush_handlers:  true
+docker_compose_flush_handlers:            true
 
 # Docker Compose
-database_type:                  "{{ application_id | get_entity_name }}"
+database_type:                            "{{ application_id | get_entity_name }}"
 
 ## Postgres
-POSTGRES_VOLUME:                "{{ applications | get_app_conf(application_id, 'docker.volumes.data', True) }}"
-POSTGRES_CONTAINER:             "{{ applications | get_app_conf(application_id, 'docker.services.postgres.name', True) }}"
-POSTGRES_IMAGE:                 "{{ applications | get_app_conf(application_id, 'docker.services.postgres.image', True) }}"
-POSTGRES_SUBNET:                "{{ networks.local['svc-db-postgres'].subnet }}"
-POSTGRES_NETWORK_NAME:          "{{ applications | get_app_conf(application_id, 'docker.network', True) }}"
-POSTGRES_VERSION:               "{{ applications | get_app_conf(application_id, 'docker.services.postgres.version', True) }}"
-POSTGRES_PASSWORD:              "{{ applications | get_app_conf(application_id, 'credentials.POSTGRES_PASSWORD', True) }}"
-POSTGRES_PORT:                  "{{ database_port | default(ports.localhost.database[ application_id ]) }}"
-POSTGRES_INIT:                  "{{ database_username is defined and database_password is defined and database_name is defined }}"
-POSTGRES_EXPOSE_LOCAL:          True # Exposes the db to localhost, almost everytime neccessary
-POSTGRES_CUSTOM_IMAGE_NAME:     "postgres_custom"
-POSTGRES_LOCAL_HOST:            "127.0.0.1"
-POSTGRES_VECTOR_ENABLED:        True # Required by discourse, propably in a later step it makes sense to define this as a configuration option in config/main.yml
-POSTGRES_RETRIES:               5
-POSTGRES_DELAY:                 2
+POSTGRES_VOLUME:                          "{{ applications | get_app_conf(application_id, 'docker.volumes.data') }}"
+POSTGRES_CONTAINER:                       "{{ applications | get_app_conf(application_id, 'docker.services.postgres.name') }}"
+POSTGRES_IMAGE:                           "{{ applications | get_app_conf(application_id, 'docker.services.postgres.image') }}"
+POSTGRES_SUBNET:                          "{{ networks.local['svc-db-postgres'].subnet }}"
+POSTGRES_NETWORK_NAME:                    "{{ applications | get_app_conf(application_id, 'docker.network') }}"
+POSTGRES_VERSION:                         "{{ applications | get_app_conf(application_id, 'docker.services.postgres.version') }}"
+POSTGRES_PASSWORD:                        "{{ applications | get_app_conf(application_id, 'credentials.POSTGRES_PASSWORD') }}"
+POSTGRES_PORT:                            "{{ database_port | default(ports.localhost.database[ application_id ]) }}"
+POSTGRES_INIT:                            "{{ database_username is defined and database_password is defined and database_name is defined }}"
+POSTGRES_EXPOSE_LOCAL:                    True # Exposes the db to localhost, almost everytime neccessary
+POSTGRES_CUSTOM_IMAGE_NAME:               "postgres_custom"
+POSTGRES_LOCAL_HOST:                      "127.0.0.1"
+POSTGRES_VECTOR_ENABLED:                  True # Required by discourse, propably in a later step it makes sense to define this as a configuration option in config/main.yml
+POSTGRES_RETRIES:                         5
+
+## Performance
+POSTGRES_TOTAL_RAM_MB:                    "{{ ansible_memtotal_mb | int }}"
+POSTGRES_VCPUS:                           "{{ ansible_processor_vcpus | int }}"
+POSTGRES_MAX_CONNECTIONS:                 "{{ [ ((POSTGRES_VCPUS | int) * 30 + 50), 400 ] | min }}"
+POSTGRES_SUPERUSER_RESERVED_CONNECTIONS:  3
+POSTGRES_SHARED_BUFFERS_MB:               "{{ ((POSTGRES_TOTAL_RAM_MB | int) * 25) // 100 }}"
+POSTGRES_SHARED_BUFFERS:                  "{{ POSTGRES_SHARED_BUFFERS_MB ~ 'MB' }}"
+POSTGRES_WORK_MEM_MB:                     "{{ [ ( (POSTGRES_TOTAL_RAM_MB | int) // ( [ (POSTGRES_MAX_CONNECTIONS | int), 1 ] | max ) // 2 ), 1 ] | max }}"
+POSTGRES_WORK_MEM:                        "{{ POSTGRES_WORK_MEM_MB ~ 'MB' }}"
+POSTGRES_MAINTENANCE_WORK_MEM_MB:         "{{ [ (((POSTGRES_TOTAL_RAM_MB | int) * 5) // 100), 64 ] | max }}"
+POSTGRES_MAINTENANCE_WORK_MEM:            "{{ POSTGRES_MAINTENANCE_WORK_MEM_MB ~ 'MB' }}"
+POSTGRES_DELAY:                           2

roles/web-app-discourse/templates/config.yml.j2

@@ -31,11 +31,16 @@ params:
   #version: tests-passed
 
 env:
-  LC_ALL: en_US.UTF-8
-  LANG: en_US.UTF-8
-  LANGUAGE: en_US.UTF-8
+  LC_ALL:   "{{ HOST_LL_CC }}.UTF-8"
+  LANG:     "{{ HOST_LL_CC }}.UTF-8"
+  LANGUAGE: "{{ HOST_LL_CC }}.UTF-8"
   #DISCOURSE_DEFAULT_LOCALE: {{ HOST_LL }} # Deactivated because not right format was selected @todo find right format
 
+  DB_POOL:            "{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+  RAILS_MAX_THREADS:  "{{ [ (POSTGRES_ALLOWED_AVG_CONNECTIONS | int), 5 ] | min }}"
+  UNICORN_WORKERS:    "{{ [ (POSTGRES_ALLOWED_AVG_CONNECTIONS | int) // 2, 1 ] | max }}"
+
+
   ## How many concurrent web requests are supported? Depends on memory and CPU cores.
   ## will be set automatically by bootstrap based on detected CPUs, or you can override
   UNICORN_WORKERS: 8

roles/web-app-funkwhale/templates/env.j2

@@ -119,4 +119,6 @@ LDAP_USER_ATTR_MAP='{"username":"uid","first_name":"givenName","last_name":"sn",
 
 FUNKWHALE_FRONTEND_PATH=/srv/funkwhale/front/dist
 
-DATABASE_URL = {{ database_url_full }}
+DATABASE_URL = {{ database_url_full }}
+WEB_CONCURRENCY="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+WORKER_THREADS="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS | int // 2 }}"

roles/web-app-gitlab/vars/main.yml

@@ -36,6 +36,7 @@ GITLAB_OMNIBUS_BASE:
   - "gitlab_rails['db_username']='{{ database_username }}'"
   - "gitlab_rails['db_password']='{{ database_password }}'"
   - "gitlab_rails['db_database']='{{ database_name }}'"
+  - "gitlab_rails['db_pool']={{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
   - "nginx['listen_port']=80"
   - "nginx['listen_https']=false"
   - ""

roles/web-app-listmonk/tasks/main.yml

@@ -40,7 +40,7 @@
          "client_secret": OIDC.CLIENT.SECRET
       } | to_json }}
 
-- name: Update administrator email and password login in Listmonk
+- name: Update administrator email and password login in Listmonk (as superuser)
   shell: |
     docker exec -i {{ database_host }} psql \
       -U {{ database_username }} \

roles/web-app-listmonk/templates/config.toml.j2

@@ -16,8 +16,8 @@ password = "{{ database_password }}"
 database = "{{ database_name }}"
 
 ssl_mode = "disable"
-max_open = 25
-max_idle = 25
+max_open = {{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}
+max_idle = {{ ( POSTGRES_ALLOWED_AVG_CONNECTIONS | int // 2 ) }}
 max_lifetime = "300s"
 
 # Optional space separated Postgres DSN params. eg: "application_name=listmonk gssencmode=disable"

roles/web-app-mastodon/templates/env.j2

@@ -38,6 +38,8 @@ DB_PORT={{ database_port }}
 DB_NAME={{ database_name }}
 DB_USER={{ database_username }}
 DB_PASS={{ database_password }}
+DB_POOL="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+RAILS_MAX_THREADS="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
 
 REDIS_HOST=redis
 REDIS_PORT=6379

roles/web-app-matrix/templates/nginx.conf.j2

@@ -6,7 +6,7 @@ server {
     listen {{ MATRIX_FEDERATION_PORT }} ssl default_server;
     listen [::]:{{ MATRIX_FEDERATION_PORT }} ssl default_server;
 
-    {% include 'roles/sys-srv-web-inj-compose/templates/server.conf.j2'%}
+    {% include 'roles/sys-front-inj-all/templates/server.conf.j2'%}
 
     {% include 'roles/srv-proxy-core/templates/location/html.conf.j2' %}
 

roles/web-app-matrix/templates/synapse/homeserver.yaml.j2

@@ -17,7 +17,7 @@ database:
     database: "{{ database_name }}"
     host: "{{ database_host }}"
     cp_min: 5
-    cp_max: 10
+    cp_max: {{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}
 log_config:                     "{{ MATRIX_SYNAPSE_LOG_PATH_CONTAINER }}"
 media_store_path:               "/data/media_store"
 registration_shared_secret:     "{{ MATRIX_REGISTRATION_SHARED_SECRET }}"

roles/web-app-mobilizon/templates/env.j2

@@ -60,6 +60,10 @@ MOBILIZON_DATABASE_PORT={{ database_port }}
 # Whether to use SSL to connect to the Mobilizon database. Useful if using an external database.
 # MOBILIZON_DATABASE_SSL=false
 
+# Not sure which of the following variables apply
+DATABASE_POOL_SIZE="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+POOL_SIZE="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+
 ######################################################
 # Secrets                                            #
 ######################################################

roles/web-app-openproject/templates/env.j2

@@ -6,15 +6,18 @@
 # Please refer to our documentation to see all possible variables:
 #   https://www.openproject.org/docs/installation-and-operations/configuration/environment/
 #
-OPENPROJECT_HTTPS=true
+OPENPROJECT_HTTPS={{ WEB_PORT == 433 | string | lower }}
 OPENPROJECT_HOST__NAME={{ domains | get_domain(application_id) }}
 OPENPROJECT_RAILS__RELATIVE__URL__ROOT=
 IMAP_ENABLED=false
-POSTGRES_PASSWORD="{{ database_password }}"
-DATABASE_URL="{{ database_url_full }}?pool=20&encoding=unicode&reconnect=true"
-RAILS_MIN_THREADS=4
-RAILS_MAX_THREADS=16
-OPENPROJECT_HSTS=true
+OPENPROJECT_HSTS={{ WEB_PORT == 433 | string | lower }}
 RAILS_CACHE_STORE: "memcache"
 OPENPROJECT_CACHE__MEMCACHE__SERVER: "cache:11211"
-OPENPROJECT_RAILS__RELATIVE__URL__ROOT: ""
+OPENPROJECT_RAILS__RELATIVE__URL__ROOT: ""
+
+# Database
+POSTGRES_PASSWORD="{{ database_password }}"
+DATABASE_URL="{{ database_url_full }}?pool=20&encoding=unicode&reconnect=true"
+DB_POOL="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+RAILS_MAX_THREADS="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+RAILS_MIN_THREADS=4

roles/web-app-pretix/templates/env.j2

@@ -13,6 +13,8 @@ PRETIX_DATABASE_USER="{{ database_username }}"
 PRETIX_DATABASE_PASSWORD="{{ database_password }}"
 PRETIX_DATABASE_HOST="{{ database_host }}"
 PRETIX_DATABASE_PORT="{{ database_port }}"
+PRETIX_WEB_CONCURRENCY="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+PRETIX_WORKER_THREADS="{{ (POSTGRES_ALLOWED_AVG_CONNECTIONS | int // 2 ) }}"
 
 ## Redis
 PRETIX_REDIS_LOCATION="redis://redis:6379/1"