From f52da1c875ac0cf4053a594bcc829da133296119 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 31 Oct 2026 03:00:03 +0100 Subject: [PATCH] fix(svc-prx-openresty): size worker_connections independently of pids_limit The shared proxy's nginx.conf derived worker_connections from the openresty container's pids_limit. pids_limit caps processes and threads, worker_connections caps simultaneous sockets per worker (client and upstream), so with pids_limit 256 and one worker the whole host's proxy held about 256 sockets, roughly 128 proxied requests in flight across every vhost. Raising pids_limit to buy connections would loosen the fork-bomb guard. svc-prx-openresty now declares services.openresty.worker_connections (4096 per worker, the common nginx production range, enough for several chunk-heavy apps such as OpenWebUI at ~150-170 requests per page load). WEBSERVER_PROXY_WORKER_CONNECTIONS reads it through lookup('config'), without a default, so a missing key fails loudly. pids_limit stays 256. WEBSERVER_PROXY_WORKER_RLIMIT_NOFILE is derived as 2x worker_connections (8192), one descriptor for each client and upstream socket of a proxied request, and nginx.conf renders it as worker_rlimit_nofile. The openresty container pins its nofile ulimit (soft and hard) to the same value: workers call setrlimit as root without CAP_SYS_RESOURCE, so they can only lower the hard limit, and the inherited hard limit depends on the host daemon (Docker >= 25 defaults to 1024:524288, DiD nests whatever the outer container has). An explicit ulimit makes the rlimit succeed on every host instead of logging a setrlimit alert. The unit test pins worker_connections 4096, worker_rlimit_nofile 8192 and the container ulimit, and fails when worker_connections follows pids_limit again. Live proof (compose deploy of web-svc-html, both plays failed=0): the deployed nginx.conf renders worker_rlimit_nofile 8192 and worker_connections 4096; openresty's master, worker and cache manager run with "Max open files 8192 8192"; the openresty log carries no setrlimit or "worker_connections are not enough" line. Closes #1300 Co-Authored-By: Claude Opus 5.5 --- group_vars/all/04_webserver.yml | 5 +- roles/svc-prx-openresty/meta/services.yml | 1 + .../templates/compose.yml.j2 | 4 + .../templates/nginx.conf.j2 | 1 + .../test_webserver_worker_limits.py | 130 +++++++++++++++--- 5 files changed, 119 insertions(+), 22 deletions(-) diff --git a/group_vars/all/04_webserver.yml b/group_vars/all/04_webserver.yml index 45f467e50c..8c0bbcf3f5 100644 --- a/group_vars/all/04_webserver.yml +++ b/group_vars/all/04_webserver.yml @@ -34,8 +34,9 @@ WEBSERVER_PROXY_WORKER_PROCESSES: >- ] | max }} -WEBSERVER_PROXY_WORKER_CONNECTIONS: >- - {{ lookup('applications') | resource_filter('svc-prx-openresty', 'pids_limit', '', RESOURCE_PIDS_LIMIT) | int }} +WEBSERVER_PROXY_WORKER_CONNECTIONS: "{{ lookup('config', 'svc-prx-openresty', 'services.openresty.worker_connections') | int }}" + +WEBSERVER_PROXY_WORKER_RLIMIT_NOFILE: "{{ (WEBSERVER_PROXY_WORKER_CONNECTIONS | int) * 2 }}" # Response security headers of every vhost; an app overrides any key under `security_headers` in its meta/server.yml. WEBSERVER_SECURITY_HEADERS: diff --git a/roles/svc-prx-openresty/meta/services.yml b/roles/svc-prx-openresty/meta/services.yml index a679a4ce2c..5e948a287e 100644 --- a/roles/svc-prx-openresty/meta/services.yml +++ b/roles/svc-prx-openresty/meta/services.yml @@ -35,6 +35,7 @@ openresty: custom: true lifecycle: beta pids_limit: 256 + worker_connections: 4096 # nocheck: variants-coverage — sso-variant-tor requires the onion on in every variant, so the false polarity has no variant left to live in tor: diff --git a/roles/svc-prx-openresty/templates/compose.yml.j2 b/roles/svc-prx-openresty/templates/compose.yml.j2 index b3bbe4721a..fa0dc37fea 100644 --- a/roles/svc-prx-openresty/templates/compose.yml.j2 +++ b/roles/svc-prx-openresty/templates/compose.yml.j2 @@ -23,6 +23,10 @@ network_mode: "host" {% endif %} {{ lookup('container_volumes', service_name) | indent(4) }} + ulimits: + nofile: + soft: {{ WEBSERVER_PROXY_WORKER_RLIMIT_NOFILE }} + hard: {{ WEBSERVER_PROXY_WORKER_RLIMIT_NOFILE }} command: ["openresty", "-g", "daemon off;"] {{ lookup('container_healthcheck', service_name) | indent(4) }} diff --git a/roles/sys-svc-webserver-core/templates/nginx.conf.j2 b/roles/sys-svc-webserver-core/templates/nginx.conf.j2 index 97b3e84dbf..c0ec1da989 100644 --- a/roles/sys-svc-webserver-core/templates/nginx.conf.j2 +++ b/roles/sys-svc-webserver-core/templates/nginx.conf.j2 @@ -1,4 +1,5 @@ worker_processes {{ WEBSERVER_PROXY_WORKER_PROCESSES }}; +worker_rlimit_nofile {{ WEBSERVER_PROXY_WORKER_RLIMIT_NOFILE }}; events { worker_connections {{ WEBSERVER_PROXY_WORKER_CONNECTIONS }}; diff --git a/tests/unit/python/group_vars/test_webserver_worker_limits.py b/tests/unit/python/group_vars/test_webserver_worker_limits.py index 16a9997311..71b0763446 100644 --- a/tests/unit/python/group_vars/test_webserver_worker_limits.py +++ b/tests/unit/python/group_vars/test_webserver_worker_limits.py @@ -8,6 +8,8 @@ proxy with that role's ``application_id``, so a 64-pid role capped every vhost a from __future__ import annotations +import re +import textwrap import unittest from pathlib import Path from typing import Any, ClassVar @@ -21,7 +23,8 @@ from utils.cache.carrier import ( merged_applications_cache_key, ) from utils.cache.files import PROJECT_ROOT, read_text -from utils.cache.yaml import load_yaml +from utils.cache.yaml import load_yaml, load_yaml_str +from utils.roles.mapping import ROLE_FILE_META_SERVICES, ROLE_FILE_TEMPL_COMPOSE _WEBSERVER_VARS = Path(PROJECT_ROOT) / "group_vars" / "all" / "04_webserver.yml" _NGINX_CONF = ( @@ -31,24 +34,45 @@ _NGINX_CONF = ( / "templates" / "nginx.conf.j2" ) +_OPENRESTY_ROLE = Path(PROJECT_ROOT) / "roles" / "svc-prx-openresty" +_OPENRESTY_SERVICES = _OPENRESTY_ROLE / ROLE_FILE_META_SERVICES +_OPENRESTY_COMPOSE = _OPENRESTY_ROLE / ROLE_FILE_TEMPL_COMPOSE _FIRST_LOADER = "web-svc-mirror" -def _applications(proxy_cpus: float) -> dict[str, Any]: - """Return a proxy with 256 pids and a first-loading role with 64 pids, 1 cpu. +def _applications(openresty: dict[str, Any]) -> dict[str, Any]: + """Return *openresty* as the proxy and a first-loading role with 64 pids, 1 cpu. Args: - proxy_cpus: the proxy's ``cpus``. + openresty: the proxy's ``services.openresty`` entry. """ return { - "svc-prx-openresty": { - "services": {"openresty": {"cpus": proxy_cpus, "pids_limit": 256}} - }, + "svc-prx-openresty": {"services": {"openresty": openresty}}, _FIRST_LOADER: {"services": {"mirror": {"cpus": 1, "pids_limit": 64}}}, } +def _proxy(cpus: float, pids_limit: int, worker_connections: int) -> dict[str, Any]: + """Return a ``services.openresty`` entry. + + Args: + cpus: the proxy's ``cpus``. + pids_limit: the proxy's ``pids_limit``. + worker_connections: the proxy's ``worker_connections``. + """ + return { + "cpus": cpus, + "pids_limit": pids_limit, + "worker_connections": worker_connections, + } + + +def _shipped() -> dict[str, Any]: + """Return ``services.openresty`` as ``svc-prx-openresty/meta/services.yml`` ships it.""" + return load_yaml(_OPENRESTY_SERVICES)["openresty"] + + class TestWebserverWorkerLimits(unittest.TestCase): group_vars: ClassVar[dict[str, Any]] @@ -57,7 +81,6 @@ class TestWebserverWorkerLimits(unittest.TestCase): cls.group_vars = load_yaml(_WEBSERVER_VARS) def setUp(self) -> None: - _reset_cache_for_tests() self.addCleanup(_reset_cache_for_tests) def _render( @@ -73,6 +96,7 @@ class TestWebserverWorkerLimits(unittest.TestCase): Returns: The rendered text, stripped. """ + _reset_cache_for_tests() variables: dict[str, Any] = { key: trust_as_template(value) if isinstance(value, str) else value for key, value in self.group_vars.items() @@ -91,25 +115,60 @@ class TestWebserverWorkerLimits(unittest.TestCase): templar = Templar(loader=DataLoader(), variables=variables) return str(templar.template(trust_as_template(expression))).strip() + def _nginx_worker_lines(self, openresty: dict[str, Any]) -> list[str]: + """Render the ``worker_*`` directives of the shared ``nginx.conf``. + + Args: + openresty: the proxy's ``services.openresty`` entry. + """ + worker_lines = [ + line.strip() + for line in read_text(str(_NGINX_CONF)).splitlines() + if line.strip().startswith("worker_") + ] + return self._render( + "\n".join(worker_lines), _applications(openresty) + ).splitlines() + def test_proxy_connections_follow_the_proxy_alone(self) -> None: for scope in ({}, {"service_name": "mirror"}): with self.subTest(**scope): self.assertEqual( self._render( "{{ WEBSERVER_PROXY_WORKER_CONNECTIONS }}", - _applications(4), + _applications(_proxy(4, 256, 4096)), **scope, ), - "256", + "4096", ) + def test_proxy_connections_ignore_pids_limit(self) -> None: + for pids_limit in (64, 256, 100000): + with self.subTest(pids_limit=pids_limit): + self.assertEqual( + self._render( + "{{ WEBSERVER_PROXY_WORKER_CONNECTIONS }}", + _applications(_proxy(0.5, pids_limit, 1234)), + ), + "1234", + ) + + def test_rlimit_nofile_doubles_the_connections(self) -> None: + self.assertEqual( + self._render( + "{{ WEBSERVER_PROXY_WORKER_RLIMIT_NOFILE }}", + _applications(_proxy(0.5, 256, 1234)), + ), + "2468", + ) + def test_proxy_processes_follow_the_proxy_alone(self) -> None: for scope in ({}, {"service_name": "mirror"}): with self.subTest(**scope): self.assertEqual( self._render( "{{ WEBSERVER_PROXY_WORKER_PROCESSES }}", - _applications(4), + _applications(_proxy(4, 256, 4096)), **scope, ), "4", @@ -117,12 +176,15 @@ class TestWebserverWorkerLimits(unittest.TestCase): def test_a_sub_cpu_proxy_still_runs_one_worker(self) -> None: self.assertEqual( - self._render("{{ WEBSERVER_PROXY_WORKER_PROCESSES }}", _applications(0.5)), + self._render( + "{{ WEBSERVER_PROXY_WORKER_PROCESSES }}", + _applications(_proxy(0.5, 256, 4096)), + ), "1", ) def test_app_internal_limits_still_pair_with_the_play(self) -> None: - applications = _applications(4) + applications = _applications(_proxy(4, 256, 4096)) self.assertEqual( self._render("{{ WEBSERVER_WORKER_CONNECTIONS }}", applications), "64" ) @@ -131,16 +193,44 @@ class TestWebserverWorkerLimits(unittest.TestCase): ) def test_shared_proxy_nginx_conf_uses_the_proxy_limits(self) -> None: - worker_lines = [ - line.strip() - for line in read_text(str(_NGINX_CONF)).splitlines() - if line.strip().startswith("worker_") - ] - rendered = self._render("\n".join(worker_lines), _applications(4)) self.assertEqual( - rendered.splitlines(), ["worker_processes 4;", "worker_connections 256;"] + self._nginx_worker_lines(_proxy(4, 256, 2048)), + [ + "worker_processes 4;", + "worker_rlimit_nofile 4096;", + "worker_connections 2048;", + ], ) + def test_shipped_proxy_limits(self) -> None: + self.assertEqual(_shipped()["pids_limit"], 256) + self.assertEqual( + self._nginx_worker_lines(_shipped()), + [ + "worker_processes 1;", + "worker_rlimit_nofile 8192;", + "worker_connections 4096;", + ], + ) + + def test_container_nofile_ulimit_allows_the_worker_rlimit(self) -> None: + match = re.search( + r"^( *)ulimits:\n(?:\1 .*\n)+", + read_text(str(_OPENRESTY_COMPOSE)), + re.MULTILINE, + ) + self.assertIsNotNone(match, f"{_OPENRESTY_COMPOSE} declares no ulimits") + applications = _applications(_shipped()) + nofile = load_yaml_str( + self._render(textwrap.dedent(match.group(0)), applications) + )["ulimits"]["nofile"] + rlimit = int( + self._render("{{ WEBSERVER_PROXY_WORKER_RLIMIT_NOFILE }}", applications) + ) + self.assertEqual(rlimit, 8192) + self.assertGreaterEqual(nofile["soft"], rlimit) + self.assertGreaterEqual(nofile["hard"], rlimit) + if __name__ == "__main__": unittest.main()