mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-10 12:47:04 +00:00
fix(web-app-matrix): cap the usage exporter's open-files limit
MDAD's synapse-usage-exporter runs uwsgi with an HTTP router that writes an 8 byte slot per possible file descriptor, sized from RLIMIT_NOFILE. Inside the matrix-mdad runner the nested daemon hands its containers the kernel maximum of 2147483584, so the exporter pinned about 16 GiB of RAM and swap and the swarm watchdog aborted WordPress#0 and Nextcloud#7 in run 42239638356. The addon now declares config.nofile 65536 and the role passes it to MDAD as --ulimit=nofile=N:N for that container; a local compose deploy shows the exporter at nofile 65536 and 38 MiB. 🎯 Fix: 97 % 💥 Break: 2 % 💸 Cost: 1 % ⏱️ Time: – 🛡️ Security: 100 % 🏆 Score: 100 🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
60cc31cd5c
commit
ccaa075b54
4 files changed
+61
No files matched your search
@@ -7,3 +7,5 @@ mechanism: addon
|
||||
source: upstream
|
||||
bridges:
|
||||
- prometheus
|
||||
config:
|
||||
nofile: 65536 # uwsgi's HTTP router sizes an 8 B per fd table from RLIMIT_NOFILE; uncapped, the nested daemon hands it 2147483584 and the exporter pins 16 GiB (run 42239638356)
|
||||
@@ -196,6 +196,8 @@ matrix_bot_baibot_config_initial_global_config_handler_catch_all: "static/{{ MAT
|
||||
{% endif %}
|
||||
|
||||
matrix_synapse_usage_exporter_enabled: {{ MATRIX_ADDON_SYNAPSE_USAGE_EXPORTER_ENABLED | string | lower }}
|
||||
matrix_synapse_usage_exporter_container_extra_arguments:
|
||||
- "--ulimit=nofile={{ MATRIX_ADDON_SYNAPSE_USAGE_EXPORTER_NOFILE }}:{{ MATRIX_ADDON_SYNAPSE_USAGE_EXPORTER_NOFILE }}"
|
||||
matrix_synapse_metrics_enabled: true
|
||||
matrix_synapse_container_metrics_api_host_bind_port: "0.0.0.0:{{ MATRIX_SYNAPSE_METRICS_PORT }}"
|
||||
matrix_synapse_metrics_port: {{ MATRIX_SYNAPSE_METRICS_PORT }}
|
||||
|
||||
@@ -131,6 +131,7 @@ MATRIX_ADDON_BAIBOT_PASSWORD: "{{ lookup('config', application_id, 'addons.baibo
|
||||
MATRIX_ADDON_BAIBOT_AGENT_ID: "{{ lookup('config', application_id, 'addons.baibot.config.agent_id') }}"
|
||||
MATRIX_ADDON_SYNAPSE_USAGE_EXPORTER_ENABLED: "{{ (lookup('config', application_id, 'addons.synapse-usage-exporter.enabled') | bool) and (HOST_ARCHITECTURE_BY_KERNEL[hostvars[STACK_HOST].ansible_facts['architecture']]
|
||||
in lookup('config', application_id, 'addons.synapse-usage-exporter.architectures')) }}"
|
||||
MATRIX_ADDON_SYNAPSE_USAGE_EXPORTER_NOFILE: "{{ lookup('config', application_id, 'addons.synapse-usage-exporter.config.nofile') }}"
|
||||
|
||||
MATRIX_ADDON_HOOKSHOT_GITLAB_ENABLED: "{{ lookup('config', application_id, 'addons.hookshot-gitlab.enabled') | bool }}"
|
||||
MATRIX_ADDON_HOOKSHOT_JIRA_ENABLED: "{{ lookup('config', application_id, 'addons.hookshot-jira.enabled') | bool }}"
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""uwsgi's HTTP router writes 8 B per possible fd; uncapped under the mdad daemon (2147483584) it pinned 16 GiB in run 42239638356."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from utils.cache.files import PROJECT_ROOT, read_text
|
||||
from utils.cache.yaml import load_yaml
|
||||
from utils.roles.mapping import ROLE_FILE_VARS_MAIN
|
||||
|
||||
_ROLE = PROJECT_ROOT / "roles/web-app-matrix"
|
||||
_ADDON = _ROLE / "meta/addons/synapse-usage-exporter.yml"
|
||||
_VARS_MAIN = _ROLE / ROLE_FILE_VARS_MAIN
|
||||
_MDAD_VARS = _ROLE / "templates/flavor/ansible/vars.yml.j2"
|
||||
|
||||
_VAR = "MATRIX_ADDON_SYNAPSE_USAGE_EXPORTER_NOFILE"
|
||||
_UWSGI_GEVENT_CORES = 1000
|
||||
_FD_SLOT_BYTES = 8
|
||||
_TABLE_BUDGET_BYTES = 16 * 1024 * 1024
|
||||
|
||||
|
||||
class TestUsageExporterNofile(unittest.TestCase):
|
||||
def test_the_cap_fits_the_gevent_cores_and_a_small_table(self) -> None:
|
||||
nofile = load_yaml(_ADDON)["config"]["nofile"]
|
||||
self.assertIsInstance(nofile, int)
|
||||
self.assertGreaterEqual(
|
||||
nofile,
|
||||
_UWSGI_GEVENT_CORES,
|
||||
"below uwsgi's gevent = 1000 the exporter silently drops async cores",
|
||||
)
|
||||
self.assertLessEqual(
|
||||
nofile * _FD_SLOT_BYTES,
|
||||
_TABLE_BUDGET_BYTES,
|
||||
"the cap no longer bounds uwsgi's per-fd router table",
|
||||
)
|
||||
|
||||
def test_the_role_reads_the_cap_from_the_addon_config(self) -> None:
|
||||
self.assertIn(
|
||||
f"{_VAR}: \"{{{{ lookup('config', application_id, "
|
||||
"'addons.synapse-usage-exporter.config.nofile') }}\"",
|
||||
read_text(str(_VARS_MAIN)),
|
||||
)
|
||||
|
||||
def test_mdad_creates_the_exporter_with_the_cap(self) -> None:
|
||||
lines = read_text(str(_MDAD_VARS)).splitlines()
|
||||
key = "matrix_synapse_usage_exporter_container_extra_arguments:"
|
||||
self.assertTrue(key in lines, "mdad would create the exporter uncapped")
|
||||
cap = f"{{{{ {_VAR} }}}}"
|
||||
self.assertEqual(
|
||||
lines[lines.index(key) + 1].strip(),
|
||||
f'- "--ulimit=nofile={cap}:{cap}"',
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user