diff --git a/group_vars/all/04_webserver.yml b/group_vars/all/04_webserver.yml index 45f467e50c..8c0bbcf3f5 100644 --- a/group_vars/all/04_webserver.yml +++ b/group_vars/all/04_webserver.yml @@ -34,8 +34,9 @@ WEBSERVER_PROXY_WORKER_PROCESSES: >- ] | max }} -WEBSERVER_PROXY_WORKER_CONNECTIONS: >- - {{ lookup('applications') | resource_filter('svc-prx-openresty', 'pids_limit', '', RESOURCE_PIDS_LIMIT) | int }} +WEBSERVER_PROXY_WORKER_CONNECTIONS: "{{ lookup('config', 'svc-prx-openresty', 'services.openresty.worker_connections') | int }}" + +WEBSERVER_PROXY_WORKER_RLIMIT_NOFILE: "{{ (WEBSERVER_PROXY_WORKER_CONNECTIONS | int) * 2 }}" # Response security headers of every vhost; an app overrides any key under `security_headers` in its meta/server.yml. WEBSERVER_SECURITY_HEADERS: diff --git a/roles/svc-prx-openresty/meta/services.yml b/roles/svc-prx-openresty/meta/services.yml index a679a4ce2c..5e948a287e 100644 --- a/roles/svc-prx-openresty/meta/services.yml +++ b/roles/svc-prx-openresty/meta/services.yml @@ -35,6 +35,7 @@ openresty: custom: true lifecycle: beta pids_limit: 256 + worker_connections: 4096 # nocheck: variants-coverage — sso-variant-tor requires the onion on in every variant, so the false polarity has no variant left to live in tor: diff --git a/roles/svc-prx-openresty/templates/compose.yml.j2 b/roles/svc-prx-openresty/templates/compose.yml.j2 index b3bbe4721a..fa0dc37fea 100644 --- a/roles/svc-prx-openresty/templates/compose.yml.j2 +++ b/roles/svc-prx-openresty/templates/compose.yml.j2 @@ -23,6 +23,10 @@ network_mode: "host" {% endif %} {{ lookup('container_volumes', service_name) | indent(4) }} + ulimits: + nofile: + soft: {{ WEBSERVER_PROXY_WORKER_RLIMIT_NOFILE }} + hard: {{ WEBSERVER_PROXY_WORKER_RLIMIT_NOFILE }} command: ["openresty", "-g", "daemon off;"] {{ lookup('container_healthcheck', service_name) | indent(4) }} diff --git a/roles/sys-svc-webserver-core/templates/nginx.conf.j2 b/roles/sys-svc-webserver-core/templates/nginx.conf.j2 index 97b3e84dbf..c0ec1da989 100644 --- a/roles/sys-svc-webserver-core/templates/nginx.conf.j2 +++ b/roles/sys-svc-webserver-core/templates/nginx.conf.j2 @@ -1,4 +1,5 @@ worker_processes {{ WEBSERVER_PROXY_WORKER_PROCESSES }}; +worker_rlimit_nofile {{ WEBSERVER_PROXY_WORKER_RLIMIT_NOFILE }}; events { worker_connections {{ WEBSERVER_PROXY_WORKER_CONNECTIONS }}; diff --git a/tests/unit/python/group_vars/test_webserver_worker_limits.py b/tests/unit/python/group_vars/test_webserver_worker_limits.py index 16a9997311..71b0763446 100644 --- a/tests/unit/python/group_vars/test_webserver_worker_limits.py +++ b/tests/unit/python/group_vars/test_webserver_worker_limits.py @@ -8,6 +8,8 @@ proxy with that role's ``application_id``, so a 64-pid role capped every vhost a from __future__ import annotations +import re +import textwrap import unittest from pathlib import Path from typing import Any, ClassVar @@ -21,7 +23,8 @@ from utils.cache.carrier import ( merged_applications_cache_key, ) from utils.cache.files import PROJECT_ROOT, read_text -from utils.cache.yaml import load_yaml +from utils.cache.yaml import load_yaml, load_yaml_str +from utils.roles.mapping import ROLE_FILE_META_SERVICES, ROLE_FILE_TEMPL_COMPOSE _WEBSERVER_VARS = Path(PROJECT_ROOT) / "group_vars" / "all" / "04_webserver.yml" _NGINX_CONF = ( @@ -31,24 +34,45 @@ _NGINX_CONF = ( / "templates" / "nginx.conf.j2" ) +_OPENRESTY_ROLE = Path(PROJECT_ROOT) / "roles" / "svc-prx-openresty" +_OPENRESTY_SERVICES = _OPENRESTY_ROLE / ROLE_FILE_META_SERVICES +_OPENRESTY_COMPOSE = _OPENRESTY_ROLE / ROLE_FILE_TEMPL_COMPOSE _FIRST_LOADER = "web-svc-mirror" -def _applications(proxy_cpus: float) -> dict[str, Any]: - """Return a proxy with 256 pids and a first-loading role with 64 pids, 1 cpu. +def _applications(openresty: dict[str, Any]) -> dict[str, Any]: + """Return *openresty* as the proxy and a first-loading role with 64 pids, 1 cpu. Args: - proxy_cpus: the proxy's ``cpus``. + openresty: the proxy's ``services.openresty`` entry. """ return { - "svc-prx-openresty": { - "services": {"openresty": {"cpus": proxy_cpus, "pids_limit": 256}} - }, + "svc-prx-openresty": {"services": {"openresty": openresty}}, _FIRST_LOADER: {"services": {"mirror": {"cpus": 1, "pids_limit": 64}}}, } +def _proxy(cpus: float, pids_limit: int, worker_connections: int) -> dict[str, Any]: + """Return a ``services.openresty`` entry. + + Args: + cpus: the proxy's ``cpus``. + pids_limit: the proxy's ``pids_limit``. + worker_connections: the proxy's ``worker_connections``. + """ + return { + "cpus": cpus, + "pids_limit": pids_limit, + "worker_connections": worker_connections, + } + + +def _shipped() -> dict[str, Any]: + """Return ``services.openresty`` as ``svc-prx-openresty/meta/services.yml`` ships it.""" + return load_yaml(_OPENRESTY_SERVICES)["openresty"] + + class TestWebserverWorkerLimits(unittest.TestCase): group_vars: ClassVar[dict[str, Any]] @@ -57,7 +81,6 @@ class TestWebserverWorkerLimits(unittest.TestCase): cls.group_vars = load_yaml(_WEBSERVER_VARS) def setUp(self) -> None: - _reset_cache_for_tests() self.addCleanup(_reset_cache_for_tests) def _render( @@ -73,6 +96,7 @@ class TestWebserverWorkerLimits(unittest.TestCase): Returns: The rendered text, stripped. """ + _reset_cache_for_tests() variables: dict[str, Any] = { key: trust_as_template(value) if isinstance(value, str) else value for key, value in self.group_vars.items() @@ -91,25 +115,60 @@ class TestWebserverWorkerLimits(unittest.TestCase): templar = Templar(loader=DataLoader(), variables=variables) return str(templar.template(trust_as_template(expression))).strip() + def _nginx_worker_lines(self, openresty: dict[str, Any]) -> list[str]: + """Render the ``worker_*`` directives of the shared ``nginx.conf``. + + Args: + openresty: the proxy's ``services.openresty`` entry. + """ + worker_lines = [ + line.strip() + for line in read_text(str(_NGINX_CONF)).splitlines() + if line.strip().startswith("worker_") + ] + return self._render( + "\n".join(worker_lines), _applications(openresty) + ).splitlines() + def test_proxy_connections_follow_the_proxy_alone(self) -> None: for scope in ({}, {"service_name": "mirror"}): with self.subTest(**scope): self.assertEqual( self._render( "{{ WEBSERVER_PROXY_WORKER_CONNECTIONS }}", - _applications(4), + _applications(_proxy(4, 256, 4096)), **scope, ), - "256", + "4096", ) + def test_proxy_connections_ignore_pids_limit(self) -> None: + for pids_limit in (64, 256, 100000): + with self.subTest(pids_limit=pids_limit): + self.assertEqual( + self._render( + "{{ WEBSERVER_PROXY_WORKER_CONNECTIONS }}", + _applications(_proxy(0.5, pids_limit, 1234)), + ), + "1234", + ) + + def test_rlimit_nofile_doubles_the_connections(self) -> None: + self.assertEqual( + self._render( + "{{ WEBSERVER_PROXY_WORKER_RLIMIT_NOFILE }}", + _applications(_proxy(0.5, 256, 1234)), + ), + "2468", + ) + def test_proxy_processes_follow_the_proxy_alone(self) -> None: for scope in ({}, {"service_name": "mirror"}): with self.subTest(**scope): self.assertEqual( self._render( "{{ WEBSERVER_PROXY_WORKER_PROCESSES }}", - _applications(4), + _applications(_proxy(4, 256, 4096)), **scope, ), "4", @@ -117,12 +176,15 @@ class TestWebserverWorkerLimits(unittest.TestCase): def test_a_sub_cpu_proxy_still_runs_one_worker(self) -> None: self.assertEqual( - self._render("{{ WEBSERVER_PROXY_WORKER_PROCESSES }}", _applications(0.5)), + self._render( + "{{ WEBSERVER_PROXY_WORKER_PROCESSES }}", + _applications(_proxy(0.5, 256, 4096)), + ), "1", ) def test_app_internal_limits_still_pair_with_the_play(self) -> None: - applications = _applications(4) + applications = _applications(_proxy(4, 256, 4096)) self.assertEqual( self._render("{{ WEBSERVER_WORKER_CONNECTIONS }}", applications), "64" ) @@ -131,16 +193,44 @@ class TestWebserverWorkerLimits(unittest.TestCase): ) def test_shared_proxy_nginx_conf_uses_the_proxy_limits(self) -> None: - worker_lines = [ - line.strip() - for line in read_text(str(_NGINX_CONF)).splitlines() - if line.strip().startswith("worker_") - ] - rendered = self._render("\n".join(worker_lines), _applications(4)) self.assertEqual( - rendered.splitlines(), ["worker_processes 4;", "worker_connections 256;"] + self._nginx_worker_lines(_proxy(4, 256, 2048)), + [ + "worker_processes 4;", + "worker_rlimit_nofile 4096;", + "worker_connections 2048;", + ], ) + def test_shipped_proxy_limits(self) -> None: + self.assertEqual(_shipped()["pids_limit"], 256) + self.assertEqual( + self._nginx_worker_lines(_shipped()), + [ + "worker_processes 1;", + "worker_rlimit_nofile 8192;", + "worker_connections 4096;", + ], + ) + + def test_container_nofile_ulimit_allows_the_worker_rlimit(self) -> None: + match = re.search( + r"^( *)ulimits:\n(?:\1 .*\n)+", + read_text(str(_OPENRESTY_COMPOSE)), + re.MULTILINE, + ) + self.assertIsNotNone(match, f"{_OPENRESTY_COMPOSE} declares no ulimits") + applications = _applications(_shipped()) + nofile = load_yaml_str( + self._render(textwrap.dedent(match.group(0)), applications) + )["ulimits"]["nofile"] + rlimit = int( + self._render("{{ WEBSERVER_PROXY_WORKER_RLIMIT_NOFILE }}", applications) + ) + self.assertEqual(rlimit, 8192) + self.assertGreaterEqual(nofile["soft"], rlimit) + self.assertGreaterEqual(nofile["hard"], rlimit) + if __name__ == "__main__": unittest.main()