diff --git a/roles/web-app-matrix/meta/addons/synapse-usage-exporter.yml b/roles/web-app-matrix/meta/addons/synapse-usage-exporter.yml index 016cb15a9c..21a504e59c 100644 --- a/roles/web-app-matrix/meta/addons/synapse-usage-exporter.yml +++ b/roles/web-app-matrix/meta/addons/synapse-usage-exporter.yml @@ -7,3 +7,5 @@ mechanism: addon source: upstream bridges: - prometheus +config: + nofile: 65536 # uwsgi's HTTP router sizes an 8 B per fd table from RLIMIT_NOFILE; uncapped, the nested daemon hands it 2147483584 and the exporter pins 16 GiB (run 42239638356) diff --git a/roles/web-app-matrix/templates/flavor/ansible/vars.yml.j2 b/roles/web-app-matrix/templates/flavor/ansible/vars.yml.j2 index f67c66c37f..e7e7b0efaf 100644 --- a/roles/web-app-matrix/templates/flavor/ansible/vars.yml.j2 +++ b/roles/web-app-matrix/templates/flavor/ansible/vars.yml.j2 @@ -196,6 +196,8 @@ matrix_bot_baibot_config_initial_global_config_handler_catch_all: "static/{{ MAT {% endif %} matrix_synapse_usage_exporter_enabled: {{ MATRIX_ADDON_SYNAPSE_USAGE_EXPORTER_ENABLED | string | lower }} +matrix_synapse_usage_exporter_container_extra_arguments: + - "--ulimit=nofile={{ MATRIX_ADDON_SYNAPSE_USAGE_EXPORTER_NOFILE }}:{{ MATRIX_ADDON_SYNAPSE_USAGE_EXPORTER_NOFILE }}" matrix_synapse_metrics_enabled: true matrix_synapse_container_metrics_api_host_bind_port: "0.0.0.0:{{ MATRIX_SYNAPSE_METRICS_PORT }}" matrix_synapse_metrics_port: {{ MATRIX_SYNAPSE_METRICS_PORT }} diff --git a/roles/web-app-matrix/vars/main.yml b/roles/web-app-matrix/vars/main.yml index 900108fa9c..52def3410a 100644 --- a/roles/web-app-matrix/vars/main.yml +++ b/roles/web-app-matrix/vars/main.yml @@ -131,6 +131,7 @@ MATRIX_ADDON_BAIBOT_PASSWORD: "{{ lookup('config', application_id, 'addons.baibo MATRIX_ADDON_BAIBOT_AGENT_ID: "{{ lookup('config', application_id, 'addons.baibot.config.agent_id') }}" MATRIX_ADDON_SYNAPSE_USAGE_EXPORTER_ENABLED: "{{ (lookup('config', application_id, 'addons.synapse-usage-exporter.enabled') | bool) and (HOST_ARCHITECTURE_BY_KERNEL[hostvars[STACK_HOST].ansible_facts['architecture']] in lookup('config', application_id, 'addons.synapse-usage-exporter.architectures')) }}" +MATRIX_ADDON_SYNAPSE_USAGE_EXPORTER_NOFILE: "{{ lookup('config', application_id, 'addons.synapse-usage-exporter.config.nofile') }}" MATRIX_ADDON_HOOKSHOT_GITLAB_ENABLED: "{{ lookup('config', application_id, 'addons.hookshot-gitlab.enabled') | bool }}" MATRIX_ADDON_HOOKSHOT_JIRA_ENABLED: "{{ lookup('config', application_id, 'addons.hookshot-jira.enabled') | bool }}" diff --git a/tests/unit/python/roles/web-app-matrix/test_usage_exporter_nofile.py b/tests/unit/python/roles/web-app-matrix/test_usage_exporter_nofile.py new file mode 100644 index 0000000000..e32cd8fe23 --- /dev/null +++ b/tests/unit/python/roles/web-app-matrix/test_usage_exporter_nofile.py @@ -0,0 +1,56 @@ +"""uwsgi's HTTP router writes 8 B per possible fd; uncapped under the mdad daemon (2147483584) it pinned 16 GiB in run 42239638356.""" + +from __future__ import annotations + +import unittest + +from utils.cache.files import PROJECT_ROOT, read_text +from utils.cache.yaml import load_yaml +from utils.roles.mapping import ROLE_FILE_VARS_MAIN + +_ROLE = PROJECT_ROOT / "roles/web-app-matrix" +_ADDON = _ROLE / "meta/addons/synapse-usage-exporter.yml" +_VARS_MAIN = _ROLE / ROLE_FILE_VARS_MAIN +_MDAD_VARS = _ROLE / "templates/flavor/ansible/vars.yml.j2" + +_VAR = "MATRIX_ADDON_SYNAPSE_USAGE_EXPORTER_NOFILE" +_UWSGI_GEVENT_CORES = 1000 +_FD_SLOT_BYTES = 8 +_TABLE_BUDGET_BYTES = 16 * 1024 * 1024 + + +class TestUsageExporterNofile(unittest.TestCase): + def test_the_cap_fits_the_gevent_cores_and_a_small_table(self) -> None: + nofile = load_yaml(_ADDON)["config"]["nofile"] + self.assertIsInstance(nofile, int) + self.assertGreaterEqual( + nofile, + _UWSGI_GEVENT_CORES, + "below uwsgi's gevent = 1000 the exporter silently drops async cores", + ) + self.assertLessEqual( + nofile * _FD_SLOT_BYTES, + _TABLE_BUDGET_BYTES, + "the cap no longer bounds uwsgi's per-fd router table", + ) + + def test_the_role_reads_the_cap_from_the_addon_config(self) -> None: + self.assertIn( + f"{_VAR}: \"{{{{ lookup('config', application_id, " + "'addons.synapse-usage-exporter.config.nofile') }}\"", + read_text(str(_VARS_MAIN)), + ) + + def test_mdad_creates_the_exporter_with_the_cap(self) -> None: + lines = read_text(str(_MDAD_VARS)).splitlines() + key = "matrix_synapse_usage_exporter_container_extra_arguments:" + self.assertTrue(key in lines, "mdad would create the exporter uncapped") + cap = f"{{{{ {_VAR} }}}}" + self.assertEqual( + lines[lines.index(key) + 1].strip(), + f'- "--ulimit=nofile={cap}:{cap}"', + ) + + +if __name__ == "__main__": # pragma: no cover + unittest.main()