feat(postgres): add split_postgres_connections filter and average pool fact

Compute POSTGRES_ALLOWED_AVG_CONNECTIONS once and propagate to app roles (gitlab, mastodon, listmonk, matrix, pretix, mobilizon, openproject, discourse). Fix docker-compose postgres command (-c flags split). Add unit tests. Minor env/locale tweaks and includes.

Conversation: https://chatgpt.com/share/68b48e72-cc28-800f-9c21-270cbc17d82a

roles/svc-db-postgres/filter_plugins/split_postgres_connections.py

@@ -0,0 +1,58 @@
+import os
+import yaml
+from ansible.errors import AnsibleFilterError
+
+def _iter_role_vars_files(roles_dir):
+    if not os.path.isdir(roles_dir):
+        raise AnsibleFilterError(f"roles_dir not found: {roles_dir}")
+    for name in os.listdir(roles_dir):
+        role_path = os.path.join(roles_dir, name)
+        if not os.path.isdir(role_path):
+            continue
+        vars_main = os.path.join(role_path, "vars", "main.yml")
+        if os.path.isfile(vars_main):
+            yield vars_main
+
+def _is_postgres_role(vars_file):
+    try:
+        with open(vars_file, "r", encoding="utf-8") as f:
+            data = yaml.safe_load(f) or {}
+        # only count roles with explicit database_type: postgres in VARS
+        return str(data.get("database_type", "")).strip().lower() == "postgres"
+    except Exception:
+        # ignore unreadable/broken YAML files quietly
+        return False
+
+def split_postgres_connections(total_connections, roles_dir="roles"):
+    """
+    Return an integer average: total_connections / number_of_roles_with_database_type_postgres.
+    Uses max(count, 1) to avoid division-by-zero.
+    """
+    try:
+        total = int(total_connections)
+    except Exception:
+        raise AnsibleFilterError(f"total_connections must be int-like, got: {total_connections!r}")
+
+    count = sum(1 for vf in _iter_role_vars_files(roles_dir) if _is_postgres_role(vf))
+    denom = max(count, 1)
+    return max(1, total // denom)
+
+def list_postgres_roles(roles_dir="roles"):
+    """
+    Helper: return a list of role names that declare database_type: postgres in vars/main.yml.
+    """
+    names = []
+    if not os.path.isdir(roles_dir):
+        return names
+    for name in os.listdir(roles_dir):
+        vars_main = os.path.join(roles_dir, name, "vars", "main.yml")
+        if os.path.isfile(vars_main) and _is_postgres_role(vars_main):
+            names.append(name)
+    return names
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            "split_postgres_connections": split_postgres_connections,
+            "list_postgres_roles": list_postgres_roles,
+        }

roles/svc-db-postgres/tasks/01_core.yml

@@ -1,3 +1,7 @@
+- name: Compute average allowed connections per Postgres app (once)
+  set_fact:
+    POSTGRES_ALLOWED_AVG_CONNECTIONS: "{{ (POSTGRES_MAX_CONNECTIONS | split_postgres_connections(playbook_dir ~ '/roles')) | int }}"
+  run_once: true
 
 - name: Include dependency 'sys-svc-docker'
   include_role:

roles/svc-db-postgres/templates/docker-compose.yml.j2

@@ -7,6 +7,19 @@
       context: .
       dockerfile: Dockerfile
     pull_policy: never
+    command:
+      - "postgres"
+      - "-c"
+      - "max_connections={{ POSTGRES_MAX_CONNECTIONS }}"
+      - "-c"
+      - "superuser_reserved_connections={{ POSTGRES_SUPERUSER_RESERVED_CONNECTIONS }}"
+      - "-c"
+      - "shared_buffers={{ POSTGRES_SHARED_BUFFERS }}"
+      - "-c"
+      - "work_mem={{ POSTGRES_WORK_MEM }}"
+      - "-c"
+      - "maintenance_work_mem={{ POSTGRES_MAINTENANCE_WORK_MEM }}"
+
 {% include 'roles/docker-container/templates/base.yml.j2' %}
 {% if POSTGRES_EXPOSE_LOCAL %}
     ports: 

roles/svc-db-postgres/vars/main.yml

@@ -1,25 +1,37 @@
 # General
-application_id:                 svc-db-postgres
+application_id:                           svc-db-postgres
 
 # Docker
-docker_compose_flush_handlers:  true
+docker_compose_flush_handlers:            true
 
 # Docker Compose
-database_type:                  "{{ application_id | get_entity_name }}"
+database_type:                            "{{ application_id | get_entity_name }}"
 
 ## Postgres
-POSTGRES_VOLUME:                "{{ applications | get_app_conf(application_id, 'docker.volumes.data', True) }}"
-POSTGRES_CONTAINER:             "{{ applications | get_app_conf(application_id, 'docker.services.postgres.name', True) }}"
-POSTGRES_IMAGE:                 "{{ applications | get_app_conf(application_id, 'docker.services.postgres.image', True) }}"
-POSTGRES_SUBNET:                "{{ networks.local['svc-db-postgres'].subnet }}"
-POSTGRES_NETWORK_NAME:          "{{ applications | get_app_conf(application_id, 'docker.network', True) }}"
-POSTGRES_VERSION:               "{{ applications | get_app_conf(application_id, 'docker.services.postgres.version', True) }}"
-POSTGRES_PASSWORD:              "{{ applications | get_app_conf(application_id, 'credentials.POSTGRES_PASSWORD', True) }}"
-POSTGRES_PORT:                  "{{ database_port | default(ports.localhost.database[ application_id ]) }}"
-POSTGRES_INIT:                  "{{ database_username is defined and database_password is defined and database_name is defined }}"
-POSTGRES_EXPOSE_LOCAL:          True # Exposes the db to localhost, almost everytime neccessary
-POSTGRES_CUSTOM_IMAGE_NAME:     "postgres_custom"
-POSTGRES_LOCAL_HOST:            "127.0.0.1"
-POSTGRES_VECTOR_ENABLED:        True # Required by discourse, propably in a later step it makes sense to define this as a configuration option in config/main.yml
-POSTGRES_RETRIES:               5
-POSTGRES_DELAY:                 2
+POSTGRES_VOLUME:                          "{{ applications | get_app_conf(application_id, 'docker.volumes.data') }}"
+POSTGRES_CONTAINER:                       "{{ applications | get_app_conf(application_id, 'docker.services.postgres.name') }}"
+POSTGRES_IMAGE:                           "{{ applications | get_app_conf(application_id, 'docker.services.postgres.image') }}"
+POSTGRES_SUBNET:                          "{{ networks.local['svc-db-postgres'].subnet }}"
+POSTGRES_NETWORK_NAME:                    "{{ applications | get_app_conf(application_id, 'docker.network') }}"
+POSTGRES_VERSION:                         "{{ applications | get_app_conf(application_id, 'docker.services.postgres.version') }}"
+POSTGRES_PASSWORD:                        "{{ applications | get_app_conf(application_id, 'credentials.POSTGRES_PASSWORD') }}"
+POSTGRES_PORT:                            "{{ database_port | default(ports.localhost.database[ application_id ]) }}"
+POSTGRES_INIT:                            "{{ database_username is defined and database_password is defined and database_name is defined }}"
+POSTGRES_EXPOSE_LOCAL:                    True # Exposes the db to localhost, almost everytime neccessary
+POSTGRES_CUSTOM_IMAGE_NAME:               "postgres_custom"
+POSTGRES_LOCAL_HOST:                      "127.0.0.1"
+POSTGRES_VECTOR_ENABLED:                  True # Required by discourse, propably in a later step it makes sense to define this as a configuration option in config/main.yml
+POSTGRES_RETRIES:                         5
+
+## Performance
+POSTGRES_TOTAL_RAM_MB:                    "{{ ansible_memtotal_mb | int }}"
+POSTGRES_VCPUS:                           "{{ ansible_processor_vcpus | int }}"
+POSTGRES_MAX_CONNECTIONS:                 "{{ [ ((POSTGRES_VCPUS | int) * 30 + 50), 400 ] | min }}"
+POSTGRES_SUPERUSER_RESERVED_CONNECTIONS:  3
+POSTGRES_SHARED_BUFFERS_MB:               "{{ ((POSTGRES_TOTAL_RAM_MB | int) * 25) // 100 }}"
+POSTGRES_SHARED_BUFFERS:                  "{{ POSTGRES_SHARED_BUFFERS_MB ~ 'MB' }}"
+POSTGRES_WORK_MEM_MB:                     "{{ [ ( (POSTGRES_TOTAL_RAM_MB | int) // ( [ (POSTGRES_MAX_CONNECTIONS | int), 1 ] | max ) // 2 ), 1 ] | max }}"
+POSTGRES_WORK_MEM:                        "{{ POSTGRES_WORK_MEM_MB ~ 'MB' }}"
+POSTGRES_MAINTENANCE_WORK_MEM_MB:         "{{ [ (((POSTGRES_TOTAL_RAM_MB | int) * 5) // 100), 64 ] | max }}"
+POSTGRES_MAINTENANCE_WORK_MEM:            "{{ POSTGRES_MAINTENANCE_WORK_MEM_MB ~ 'MB' }}"
+POSTGRES_DELAY:                           2

roles/web-app-discourse/templates/config.yml.j2

@@ -31,11 +31,16 @@ params:
   #version: tests-passed
 
 env:
-  LC_ALL: en_US.UTF-8
-  LANG: en_US.UTF-8
-  LANGUAGE: en_US.UTF-8
+  LC_ALL:   "{{ HOST_LL_CC }}.UTF-8"
+  LANG:     "{{ HOST_LL_CC }}.UTF-8"
+  LANGUAGE: "{{ HOST_LL_CC }}.UTF-8"
   #DISCOURSE_DEFAULT_LOCALE: {{ HOST_LL }} # Deactivated because not right format was selected @todo find right format
 
+  DB_POOL:            "{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+  RAILS_MAX_THREADS:  "{{ [ (POSTGRES_ALLOWED_AVG_CONNECTIONS | int), 5 ] | min }}"
+  UNICORN_WORKERS:    "{{ [ (POSTGRES_ALLOWED_AVG_CONNECTIONS | int) // 2, 1 ] | max }}"
+
+
   ## How many concurrent web requests are supported? Depends on memory and CPU cores.
   ## will be set automatically by bootstrap based on detected CPUs, or you can override
   UNICORN_WORKERS: 8

roles/web-app-funkwhale/templates/env.j2

@@ -119,4 +119,6 @@ LDAP_USER_ATTR_MAP='{"username":"uid","first_name":"givenName","last_name":"sn",
 
 FUNKWHALE_FRONTEND_PATH=/srv/funkwhale/front/dist
 
-DATABASE_URL = {{ database_url_full }}
+DATABASE_URL = {{ database_url_full }}
+WEB_CONCURRENCY="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+WORKER_THREADS="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS | int // 2 }}"

roles/web-app-gitlab/vars/main.yml

@@ -36,6 +36,7 @@ GITLAB_OMNIBUS_BASE:
   - "gitlab_rails['db_username']='{{ database_username }}'"
   - "gitlab_rails['db_password']='{{ database_password }}'"
   - "gitlab_rails['db_database']='{{ database_name }}'"
+  - "gitlab_rails['db_pool']={{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
   - "nginx['listen_port']=80"
   - "nginx['listen_https']=false"
   - ""

roles/web-app-listmonk/tasks/main.yml

@@ -40,7 +40,7 @@
          "client_secret": OIDC.CLIENT.SECRET
       } | to_json }}
 
-- name: Update administrator email and password login in Listmonk
+- name: Update administrator email and password login in Listmonk (as superuser)
   shell: |
     docker exec -i {{ database_host }} psql \
       -U {{ database_username }} \

roles/web-app-listmonk/templates/config.toml.j2

@@ -16,8 +16,8 @@ password = "{{ database_password }}"
 database = "{{ database_name }}"
 
 ssl_mode = "disable"
-max_open = 25
-max_idle = 25
+max_open = {{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}
+max_idle = {{ ( POSTGRES_ALLOWED_AVG_CONNECTIONS | int // 2 ) }}
 max_lifetime = "300s"
 
 # Optional space separated Postgres DSN params. eg: "application_name=listmonk gssencmode=disable"

roles/web-app-mastodon/templates/env.j2

@@ -38,6 +38,8 @@ DB_PORT={{ database_port }}
 DB_NAME={{ database_name }}
 DB_USER={{ database_username }}
 DB_PASS={{ database_password }}
+DB_POOL="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+RAILS_MAX_THREADS="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
 
 REDIS_HOST=redis
 REDIS_PORT=6379

roles/web-app-matrix/templates/nginx.conf.j2

@@ -6,7 +6,7 @@ server {
     listen {{ MATRIX_FEDERATION_PORT }} ssl default_server;
     listen [::]:{{ MATRIX_FEDERATION_PORT }} ssl default_server;
 
-    {% include 'roles/sys-srv-web-inj-compose/templates/server.conf.j2'%}
+    {% include 'roles/sys-front-inj-all/templates/server.conf.j2'%}
 
     {% include 'roles/srv-proxy-core/templates/location/html.conf.j2' %}
 

roles/web-app-matrix/templates/synapse/homeserver.yaml.j2

@@ -17,7 +17,7 @@ database:
     database: "{{ database_name }}"
     host: "{{ database_host }}"
     cp_min: 5
-    cp_max: 10
+    cp_max: {{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}
 log_config:                     "{{ MATRIX_SYNAPSE_LOG_PATH_CONTAINER }}"
 media_store_path:               "/data/media_store"
 registration_shared_secret:     "{{ MATRIX_REGISTRATION_SHARED_SECRET }}"

roles/web-app-mobilizon/templates/env.j2

@@ -60,6 +60,10 @@ MOBILIZON_DATABASE_PORT={{ database_port }}
 # Whether to use SSL to connect to the Mobilizon database. Useful if using an external database.
 # MOBILIZON_DATABASE_SSL=false
 
+# Not sure which of the following variables apply
+DATABASE_POOL_SIZE="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+POOL_SIZE="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+
 ######################################################
 # Secrets                                            #
 ######################################################

roles/web-app-openproject/templates/env.j2

@@ -6,15 +6,18 @@
 # Please refer to our documentation to see all possible variables:
 #   https://www.openproject.org/docs/installation-and-operations/configuration/environment/
 #
-OPENPROJECT_HTTPS=true
+OPENPROJECT_HTTPS={{ WEB_PORT == 433 | string | lower }}
 OPENPROJECT_HOST__NAME={{ domains | get_domain(application_id) }}
 OPENPROJECT_RAILS__RELATIVE__URL__ROOT=
 IMAP_ENABLED=false
-POSTGRES_PASSWORD="{{ database_password }}"
-DATABASE_URL="{{ database_url_full }}?pool=20&encoding=unicode&reconnect=true"
-RAILS_MIN_THREADS=4
-RAILS_MAX_THREADS=16
-OPENPROJECT_HSTS=true
+OPENPROJECT_HSTS={{ WEB_PORT == 433 | string | lower }}
 RAILS_CACHE_STORE: "memcache"
 OPENPROJECT_CACHE__MEMCACHE__SERVER: "cache:11211"
-OPENPROJECT_RAILS__RELATIVE__URL__ROOT: ""
+OPENPROJECT_RAILS__RELATIVE__URL__ROOT: ""
+
+# Database
+POSTGRES_PASSWORD="{{ database_password }}"
+DATABASE_URL="{{ database_url_full }}?pool=20&encoding=unicode&reconnect=true"
+DB_POOL="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+RAILS_MAX_THREADS="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+RAILS_MIN_THREADS=4

roles/web-app-pretix/templates/env.j2

@@ -13,6 +13,8 @@ PRETIX_DATABASE_USER="{{ database_username }}"
 PRETIX_DATABASE_PASSWORD="{{ database_password }}"
 PRETIX_DATABASE_HOST="{{ database_host }}"
 PRETIX_DATABASE_PORT="{{ database_port }}"
+PRETIX_WEB_CONCURRENCY="{{ POSTGRES_ALLOWED_AVG_CONNECTIONS }}"
+PRETIX_WORKER_THREADS="{{ (POSTGRES_ALLOWED_AVG_CONNECTIONS | int // 2 ) }}"
 
 ## Redis
 PRETIX_REDIS_LOCATION="redis://redis:6379/1"

tests/unit/roles/svc-db-postgres/filter_plugins/test_split_postgres_connections.py

@@ -0,0 +1,109 @@
+import os
+import shutil
+import tempfile
+import textwrap
+import importlib.util
+import unittest
+from types import ModuleType
+from ansible.errors import AnsibleFilterError
+
+
+def load_filter_module(repo_root: str) -> ModuleType:
+    """
+    Load the filter plugin from:
+      roles/svc-db-postgres/filter_plugins/split_postgres_connections.py
+    """
+    plugin_path = os.path.join(
+        repo_root, "roles", "svc-db-postgres", "filter_plugins", "split_postgres_connections.py"
+    )
+    if not os.path.isfile(plugin_path):
+        raise FileNotFoundError(f"Filter plugin not found at {plugin_path}")
+    spec = importlib.util.spec_from_file_location("split_postgres_connections_plugin", plugin_path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec and spec.loader
+    spec.loader.exec_module(module)  # type: ignore[attr-defined]
+    return module
+
+
+def write_role_vars(repo_root: str, role_name: str, database_type: str | None):
+    """
+    Create a minimal role with optional vars/main.yml containing database_type.
+    """
+    role_dir = os.path.join(repo_root, "roles", role_name)
+    vars_dir = os.path.join(role_dir, "vars")
+    os.makedirs(role_dir, exist_ok=True)
+    if database_type is not None:
+        os.makedirs(vars_dir, exist_ok=True)
+        with open(os.path.join(vars_dir, "main.yml"), "w", encoding="utf-8") as f:
+            f.write(textwrap.dedent(f"""\
+                # auto-generated for test
+                database_type: {database_type}
+            """))
+
+
+class SplitPostgresConnectionsTests(unittest.TestCase):
+    def setUp(self):
+        # Create an isolated temporary repository layout
+        self.repo = tempfile.mkdtemp(prefix="repo_")
+        self.roles_dir = os.path.join(self.repo, "roles")
+        os.makedirs(self.roles_dir, exist_ok=True)
+
+        # Create roles:
+        # - app_a (postgres)
+        # - app_b (postgres)
+        # - app_c (mysql)
+        # - app_d (no vars/main.yml)
+        write_role_vars(self.repo, "app_a", "postgres")
+        write_role_vars(self.repo, "app_b", "postgres")
+        write_role_vars(self.repo, "app_c", "mysql")
+        write_role_vars(self.repo, "app_d", None)
+
+        # Copy the real plugin into this temp repo structure, preserving your path layout.
+        # (Adjust src_plugin_path if your test runner runs from a different CWD.)
+        src_plugin_path = os.path.join(
+            os.getcwd(), "roles", "svc-db-postgres", "filter_plugins", "split_postgres_connections.py"
+        )
+        if not os.path.isfile(src_plugin_path):
+            self.skipTest(f"Source plugin not found at {src_plugin_path}")
+        dst_plugin_dir = os.path.join(self.repo, "roles", "svc-db-postgres", "filter_plugins")
+        os.makedirs(dst_plugin_dir, exist_ok=True)
+        shutil.copy2(src_plugin_path, os.path.join(dst_plugin_dir, "split_postgres_connections.py"))
+
+        self.mod = load_filter_module(self.repo)
+
+    def tearDown(self):
+        shutil.rmtree(self.repo, ignore_errors=True)
+
+    def test_registry_contains_filters(self):
+        registry = self.mod.FilterModule().filters()
+        self.assertIn("split_postgres_connections", registry)
+        self.assertIn("list_postgres_roles", registry)
+
+    def test_list_postgres_roles(self):
+        roles = self.mod.list_postgres_roles(self.roles_dir)
+        self.assertIsInstance(roles, list)
+        self.assertSetEqual(set(roles), {"app_a", "app_b"})
+
+    def test_split_postgres_connections_division(self):
+        # There are 2 postgres roles -> 200 / 2 = 100
+        avg = self.mod.split_postgres_connections(200, roles_dir=self.roles_dir)
+        self.assertEqual(avg, 100)
+
+        # 5 / 2 -> floor 2
+        self.assertEqual(self.mod.split_postgres_connections(5, roles_dir=self.roles_dir), 2)
+
+        # Safety floor: at least 1
+        self.assertEqual(self.mod.split_postgres_connections(1, roles_dir=self.roles_dir), 1)
+
+    def test_split_handles_non_int_input(self):
+        with self.assertRaises(AnsibleFilterError):
+            self.mod.split_postgres_connections("not-an-int", roles_dir=self.roles_dir)
+
+    def test_missing_roles_dir_raises(self):
+        # Current plugin behavior: raise if roles_dir does not exist
+        with self.assertRaises(AnsibleFilterError):
+            self.mod.split_postgres_connections(100, roles_dir="/does/not/exist")
+
+
+if __name__ == "__main__":
+    unittest.main()