From 7330aeb8ecafcddc519ec599769b63ad0e5985a0 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 3 Nov 2025 21:47:38 +0100 Subject: [PATCH] =?UTF-8?q?feat(web-app-peertube):=20add=20dynamic=20perfo?= =?UTF-8?q?rmance=20tuning=20for=20heap=20and=20transcoding=20concurrency?= =?UTF-8?q?=20-=20Dynamically=20calculate=20PEERTUBE=5FMAX=5FOLD=5FSPACE?= =?UTF-8?q?=5FSIZE=20(~35%=20of=20container=20RAM,=20clamped=20between=207?= =?UTF-8?q?68=E2=80=933072=20MB)=20-=20Dynamically=20calculate=20PEERTUBE?= =?UTF-8?q?=5FTRANSCODING=5FCONCURRENCY=20(~=C2=BD=20vCPUs,=20min=201,=20m?= =?UTF-8?q?ax=208)=20-=20Added=20default=20resource=20limits=20for=20Redis?= =?UTF-8?q?=20and=20Peertube=20containers=20-=20Updated=20test=20suite=20t?= =?UTF-8?q?o=20include=20human=5Fto=5Fbytes=20filter=20in=20built-in=20fil?= =?UTF-8?q?ter=20list?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://chatgpt.com/share/690914d2-6100-800f-a850-94e6d226e7c9 --- roles/web-app-peertube/config/main.yml | 8 +++ .../templates/docker-compose.yml.j2 | 11 ++++ roles/web-app-peertube/vars/main.yml | 50 +++++++++++++++---- roles/web-app-shopware/vars/main.yml | 5 -- tests/integration/test_filters_are_defined.py | 2 +- 5 files changed, 60 insertions(+), 16 deletions(-) diff --git a/roles/web-app-peertube/config/main.yml b/roles/web-app-peertube/config/main.yml index df8d0fff..b8f865ca 100644 --- a/roles/web-app-peertube/config/main.yml +++ b/roles/web-app-peertube/config/main.yml @@ -30,6 +30,10 @@ docker: services: redis: enabled: true + cpus: "0.5" + mem_reservation: "256m" + mem_limit: "512m" + pids_limit: 512 database: enabled: true peertube: @@ -38,6 +42,10 @@ docker: image: "chocobozzz/peertube" backup: no_stop_required: true + cpus: 4 + mem_reservation: "4g" + mem_limit: "8g" + pids_limit: 2048 # ffmpeg spawnt Threads/Prozesse volumes: data: peertube_data config: peertube_config \ No newline at end of file diff --git a/roles/web-app-peertube/templates/docker-compose.yml.j2 b/roles/web-app-peertube/templates/docker-compose.yml.j2 index c4460ac6..1a33a257 100644 --- a/roles/web-app-peertube/templates/docker-compose.yml.j2 +++ b/roles/web-app-peertube/templates/docker-compose.yml.j2 @@ -12,6 +12,17 @@ - assets:/app/client/dist - data:/data - config:/config + environment: + - NODE_OPTIONS=--max-old-space-size={{ PEERTUBE_MAX_OLD_SPACE_SIZE }} + - PEERTUBE_TRANSCODING_CONCURRENCY={{ PEERTUBE_TRANSCODING_CONCURRENCY }} + shm_size: "512m" + tmpfs: + - /tmp:size=1g,exec + ulimits: + nofile: + soft: 131072 + hard: 131072 + nproc: 8192 {% include 'roles/docker-container/templates/depends_on/dmbs_excl.yml.j2' %} {% include 'roles/docker-container/templates/networks.yml.j2' %} {% include 'roles/docker-container/templates/healthcheck/tcp.yml.j2' %} diff --git a/roles/web-app-peertube/vars/main.yml b/roles/web-app-peertube/vars/main.yml index ea31f569..d56e5c40 100644 --- a/roles/web-app-peertube/vars/main.yml +++ b/roles/web-app-peertube/vars/main.yml @@ -1,17 +1,47 @@ # General -application_id: "web-app-peertube" -database_type: "postgres" +application_id: "web-app-peertube" +database_type: "postgres" # Docker -docker_compose_flush_handlers: true +docker_compose_flush_handlers: true # Role variables -PEERTUBE_VERSION: "{{ applications | get_app_conf(application_id, 'docker.services.peertube.version') }}" -PEERTUBE_IMAGE: "{{ applications | get_app_conf(application_id, 'docker.services.peertube.image') }}" -PEERTUBE_CONTAINER: "{{ applications | get_app_conf(application_id, 'docker.services.peertube.name') }}" -PEERTUBE_DATA_VOLUME: "{{ applications | get_app_conf(application_id, 'docker.volumes.data') }}" -PEERTUBE_CONFIG_VOLUME: "{{ applications | get_app_conf(application_id, 'docker.volumes.config') }}" +PEERTUBE_VERSION: "{{ applications | get_app_conf(application_id, 'docker.services.peertube.version') }}" +PEERTUBE_IMAGE: "{{ applications | get_app_conf(application_id, 'docker.services.peertube.image') }}" +PEERTUBE_CONTAINER: "{{ applications | get_app_conf(application_id, 'docker.services.peertube.name') }}" +PEERTUBE_DATA_VOLUME: "{{ applications | get_app_conf(application_id, 'docker.volumes.data') }}" +PEERTUBE_CONFIG_VOLUME: "{{ applications | get_app_conf(application_id, 'docker.volumes.config') }}" # OIDC -PEERTUBE_OIDC_PLUGIN: "peertube-plugin-auth-openid-connect" -PEERTUBE_OIDC_ENABLED: "{{ applications | get_app_conf(application_id, 'features.oidc', False) }}" \ No newline at end of file +PEERTUBE_OIDC_PLUGIN: "peertube-plugin-auth-openid-connect" +PEERTUBE_OIDC_ENABLED: "{{ applications | get_app_conf(application_id, 'features.oidc') }}" + +# === Dynamic performance defaults ========================================== + +# Raw Docker configuration values (with sane fallbacks) +peertube_cpus: "{{ applications | get_app_conf(application_id, 'docker.services.peertube.cpus') | float }}" +peertube_mem_limit_raw: "{{ applications | get_app_conf(application_id, 'docker.services.peertube.mem_limit') }}" +peertube_mem_bytes: "{{ peertube_mem_limit_raw | human_to_bytes }}" +peertube_mem_mb: "{{ ((peertube_mem_bytes | int) // (1024 * 1024)) | int }}" + +# --------------------------------------------------------------------------- +# Node heap size: +# ~35% of total RAM, but at least 768 MB, at most 3072 MB, +# and never more than 60% of total memory (safety cap for small containers) +# --------------------------------------------------------------------------- + +_peertube_heap_candidate_mb: "{{ ((peertube_mem_mb | float) * 0.35) | round(0, 'floor') | int }}" +_peertube_heap_cap_mb: "{{ ((peertube_mem_mb | float) * 0.60) | round(0, 'floor') | int }}" + +# Step 1: enforce minimum (≥768 MB) +_peertube_heap_min_applied: "{{ [ (_peertube_heap_candidate_mb | int), 768 ] | max }}" + +# Step 2: determine hard cap (min of 3072 MB and 60% of total memory) +_peertube_heap_hardcap: "{{ [ 3072, (_peertube_heap_cap_mb | int) ] | min }}" + +# Step 3: final heap = min(min-applied, hardcap) +PEERTUBE_MAX_OLD_SPACE_SIZE: "{{ [ (_peertube_heap_min_applied | int), (_peertube_heap_hardcap | int) ] | min }}" + +# Transcoding concurrency: half the vCPUs; min 1, max 8 +_peertube_concurrency_candidate: "{{ ((peertube_cpus | float) * 0.5) | round(0, 'floor') | int }}" +PEERTUBE_TRANSCODING_CONCURRENCY: "{{ [ ( [ (_peertube_concurrency_candidate | int), 1 ] | max ), 8 ] | min }}" diff --git a/roles/web-app-shopware/vars/main.yml b/roles/web-app-shopware/vars/main.yml index d937a5a0..b778f99c 100644 --- a/roles/web-app-shopware/vars/main.yml +++ b/roles/web-app-shopware/vars/main.yml @@ -32,12 +32,7 @@ SHOPWARE_WORKER_REPLICAS: "{{ applications | get_app_conf(application_id, SHOPWARE_REDIS_ENABLED: "{{ applications | get_app_conf(application_id, 'docker.services.redis.enabled') }}" SHOPWARE_REDIS_ADDRESS: "redis:6379" SHOPWARE_OPENSEARCH_ENABLED: "{{ applications | get_app_conf(application_id, 'docker.services.opensearch.enabled') }}" -SHOPWARE_OPENSEARCH_ENGINE: "{{ applications | get_app_conf(application_id, 'docker.services.opensearch.engine') }}" SHOPWARE_OPENSEARCH_IMAGE: "{{ applications | get_app_conf(application_id, 'docker.services.opensearch.image') }}" SHOPWARE_OPENSEARCH_VERSION: "{{ applications | get_app_conf(application_id, 'docker.services.opensearch.version') }}" SHOPWARE_OPENSEARCH_CONTAINER: "{{ applications | get_app_conf(application_id, 'docker.services.opensearch.name') }}" SHOPWARE_OPENSEARCH_MEM_RESERVATION: "{{ applications | get_app_conf(application_id, 'docker.services.opensearch.mem_reservation') }}" -SHOPWARE_OPENSEARCH_MEM_LIMIT: "{{ applications | get_app_conf(application_id, 'docker.services.opensearch.mem_limit') }}" - -# IAM (true if either OIDC or LDAP is enabled) -SHOPWARE_IAM_ENABLED: "{{ applications | get_app_conf(application_id, 'features.oidc') or applications | get_app_conf(application_id, 'features.ldap') }}" diff --git a/tests/integration/test_filters_are_defined.py b/tests/integration/test_filters_are_defined.py index 6a11fa47..c7ef78cc 100644 --- a/tests/integration/test_filters_are_defined.py +++ b/tests/integration/test_filters_are_defined.py @@ -37,7 +37,7 @@ BUILTIN_FILTERS: Set[str] = { "type_debug", "json_query", "mandatory", "hash", "checksum", "lower", "upper", "capitalize", "unique", "dict2items", "items2dict", "password_hash", "path_join", "product", "quote", "split", "ternary", "to_nice_yaml", - "tojson", "to_nice_json", + "tojson", "to_nice_json", "human_to_bytes", # Date/time-ish