kevinveenbirkenbach/computer-playbook
1
0
Fork 0
mirror of https://github.com/kevinveenbirkenbach/computer-playbook.git synced 2025-09-08 19:27:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(postgres): add split_postgres_connections filter and average pool fact

Browse Source 
Compute POSTGRES_ALLOWED_AVG_CONNECTIONS once and propagate to app roles (gitlab, mastodon, listmonk, matrix, pretix, mobilizon, openproject, discourse). Fix docker-compose postgres command (-c flags split). Add unit tests. Minor env/locale tweaks and includes.

Conversation: https://chatgpt.com/share/68b48e72-cc28-800f-9c21-270cbc17d82a
This commit is contained in:
Kevin Veen-Birkenbach
2025-08-31 20:04:14 +02:00
parent b3dfb8bf22
commit 5f66c1a622
18 changed files with 249 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
roles/svc-db-postgres/filter_plugins/split_postgres_connections.py Normal file
View File

@@ -0,0 +1,58 @@
import os
import yaml
from ansible.errors import AnsibleFilterError
def _iter_role_vars_files(roles_dir):
if not os.path.isdir(roles_dir):
raise AnsibleFilterError(f"roles_dir not found: {roles_dir}")
for name in os.listdir(roles_dir):
role_path = os.path.join(roles_dir, name)
if not os.path.isdir(role_path):
continue
vars_main = os.path.join(role_path, "vars", "main.yml")
if os.path.isfile(vars_main):
yield vars_main
def _is_postgres_role(vars_file):
try:
with open(vars_file, "r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
# only count roles with explicit database_type: postgres in VARS
return str(data.get("database_type", "")).strip().lower() == "postgres"
except Exception:
# ignore unreadable/broken YAML files quietly
return False
def split_postgres_connections(total_connections, roles_dir="roles"):
"""
Return an integer average: total_connections / number_of_roles_with_database_type_postgres.
Uses max(count, 1) to avoid division-by-zero.
"""
try:
total = int(total_connections)
except Exception:
raise AnsibleFilterError(f"total_connections must be int-like, got: {total_connections!r}")
count = sum(1 for vf in _iter_role_vars_files(roles_dir) if _is_postgres_role(vf))
denom = max(count, 1)
return max(1, total // denom)
def list_postgres_roles(roles_dir="roles"):
"""
Helper: return a list of role names that declare database_type: postgres in vars/main.yml.
"""
names = []
if not os.path.isdir(roles_dir):
return names
for name in os.listdir(roles_dir):
vars_main = os.path.join(roles_dir, name, "vars", "main.yml")
if os.path.isfile(vars_main) and _is_postgres_role(vars_main):
names.append(name)
return names
class FilterModule(object):
def filters(self):
return {
"split_postgres_connections": split_postgres_connections,
"list_postgres_roles": list_postgres_roles,
}

4
roles/svc-db-postgres/tasks/01_core.yml
View File

@@ -1,3 +1,7 @@
- name: Compute average allowed connections per Postgres app (once)
set_fact:
POSTGRES_ALLOWED_AVG_CONNECTIONS: "{{ (POSTGRES_MAX_CONNECTIONS | split_postgres_connections(playbook_dir ~ '/roles')) | int }}"
run_once: true
- name: Include dependency 'sys-svc-docker'
include_role:

13
roles/svc-db-postgres/templates/docker-compose.yml.j2
View File

@@ -7,6 +7,19 @@
context: .
dockerfile: Dockerfile
pull_policy: never
command:
- "postgres"
- "-c"
- "max_connections={{ POSTGRES_MAX_CONNECTIONS }}"
- "-c"
- "superuser_reserved_connections={{ POSTGRES_SUPERUSER_RESERVED_CONNECTIONS }}"
- "-c"
- "shared_buffers={{ POSTGRES_SHARED_BUFFERS }}"
- "-c"
- "work_mem={{ POSTGRES_WORK_MEM }}"
- "-c"
- "maintenance_work_mem={{ POSTGRES_MAINTENANCE_WORK_MEM }}"
{% include 'roles/docker-container/templates/base.yml.j2' %}
{% if POSTGRES_EXPOSE_LOCAL %}
ports:

48
roles/svc-db-postgres/vars/main.yml
View File

@@ -1,25 +1,37 @@
# General
application_id: svc-db-postgres
application_id: svc-db-postgres
# Docker
docker_compose_flush_handlers: true
docker_compose_flush_handlers: true
# Docker Compose
database_type: "{{ application_id | get_entity_name }}"
database_type: "{{ application_id | get_entity_name }}"
## Postgres
POSTGRES_VOLUME: "{{ applications | get_app_conf(application_id, 'docker.volumes.data', True) }}"
POSTGRES_CONTAINER: "{{ applications | get_app_conf(application_id, 'docker.services.postgres.name', True) }}"
POSTGRES_IMAGE: "{{ applications | get_app_conf(application_id, 'docker.services.postgres.image', True) }}"
POSTGRES_SUBNET: "{{ networks.local['svc-db-postgres'].subnet }}"
POSTGRES_NETWORK_NAME: "{{ applications | get_app_conf(application_id, 'docker.network', True) }}"
POSTGRES_VERSION: "{{ applications | get_app_conf(application_id, 'docker.services.postgres.version', True) }}"
POSTGRES_PASSWORD: "{{ applications | get_app_conf(application_id, 'credentials.POSTGRES_PASSWORD', True) }}"
POSTGRES_PORT: "{{ database_port | default(ports.localhost.database[ application_id ]) }}"
POSTGRES_INIT: "{{ database_username is defined and database_password is defined and database_name is defined }}"
POSTGRES_EXPOSE_LOCAL: True # Exposes the db to localhost, almost everytime neccessary
POSTGRES_CUSTOM_IMAGE_NAME: "postgres_custom"
POSTGRES_LOCAL_HOST: "127.0.0.1"
POSTGRES_VECTOR_ENABLED: True # Required by discourse, propably in a later step it makes sense to define this as a configuration option in config/main.yml
POSTGRES_RETRIES: 5
POSTGRES_DELAY: 2
POSTGRES_VOLUME: "{{ applications | get_app_conf(application_id, 'docker.volumes.data') }}"
POSTGRES_CONTAINER: "{{ applications | get_app_conf(application_id, 'docker.services.postgres.name') }}"
POSTGRES_IMAGE: "{{ applications | get_app_conf(application_id, 'docker.services.postgres.image') }}"
POSTGRES_SUBNET: "{{ networks.local['svc-db-postgres'].subnet }}"
POSTGRES_NETWORK_NAME: "{{ applications | get_app_conf(application_id, 'docker.network') }}"
POSTGRES_VERSION: "{{ applications | get_app_conf(application_id, 'docker.services.postgres.version') }}"
POSTGRES_PASSWORD: "{{ applications | get_app_conf(application_id, 'credentials.POSTGRES_PASSWORD') }}"
POSTGRES_PORT: "{{ database_port | default(ports.localhost.database[ application_id ]) }}"
POSTGRES_INIT: "{{ database_username is defined and database_password is defined and database_name is defined }}"
POSTGRES_EXPOSE_LOCAL: True # Exposes the db to localhost, almost everytime neccessary
POSTGRES_CUSTOM_IMAGE_NAME: "postgres_custom"
POSTGRES_LOCAL_HOST: "127.0.0.1"
POSTGRES_VECTOR_ENABLED: True # Required by discourse, propably in a later step it makes sense to define this as a configuration option in config/main.yml
POSTGRES_RETRIES: 5
## Performance
POSTGRES_TOTAL_RAM_MB: "{{ ansible_memtotal_mb | int }}"
POSTGRES_VCPUS: "{{ ansible_processor_vcpus | int }}"
POSTGRES_MAX_CONNECTIONS: "{{ [ ((POSTGRES_VCPUS | int) * 30 + 50), 400 ] | min }}"
POSTGRES_SUPERUSER_RESERVED_CONNECTIONS: 3
POSTGRES_SHARED_BUFFERS_MB: "{{ ((POSTGRES_TOTAL_RAM_MB | int) * 25) // 100 }}"
POSTGRES_SHARED_BUFFERS: "{{ POSTGRES_SHARED_BUFFERS_MB ~ 'MB' }}"
POSTGRES_WORK_MEM_MB: "{{ [ ( (POSTGRES_TOTAL_RAM_MB | int) // ( [ (POSTGRES_MAX_CONNECTIONS | int), 1 ] | max ) // 2 ), 1 ] | max }}"
POSTGRES_WORK_MEM: "{{ POSTGRES_WORK_MEM_MB ~ 'MB' }}"
POSTGRES_MAINTENANCE_WORK_MEM_MB: "{{ [ (((POSTGRES_TOTAL_RAM_MB | int) * 5) // 100), 64 ] | max }}"
POSTGRES_MAINTENANCE_WORK_MEM: "{{ POSTGRES_MAINTENANCE_WORK_MEM_MB ~ 'MB' }}"
POSTGRES_DELAY: 2