From ffe23287d04f76beda45bbc114fb847e8b564dce Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 7 Oct 2026 18:09:10 +0200 Subject: [PATCH] fix(web-app-docs): size the sphinx concurrency from the container limit jobs and parallel were derived from the host CPU count with no memory term, so a 4g container could be asked to run two builds of two sphinx workers each. One sphinx process reached 2.39 GB anon-rss while still growing, and the kernel killed two of them ("Memory cgroup out of memory: Killed process ... (sphinx-build)", cgroup usage 4194304kB of limit 4194304kB). Every build died there, so every page kept answering 202 until the spec gave up. mem_limit now follows the host, and both knobs follow mem_limit: parallel takes the builds the limit can hold, then jobs takes the workers left once every concurrent build has reserved its overhead. Deriving jobs first let it claim all the memory and pinned parallel at one; reserving the overhead only once overcommitted an 8-core host by 683 MB. The two inputs are declared beside the limit they are measured against. Neither is a ceiling: no docs build has ever completed in CI, so sphinx_process_mb is a lower bound that the first green build can correct. mem_limit calls lookup("resource", ...) directly rather than through a group_var, the way svc-opt-swapfile does: a meta file renders with a flat variable dict, which cannot expand a group_var that itself holds a lookup and would have left the limit at 0. The generator env also lacked the PYTHONUNBUFFERED that _sphinx_env sets, so a generator killed under that memory pressure lost whatever its stdout buffer still held. That is why the integrations failure logged its argv and nothing else. Co-Authored-By: Claude Opus 5 (1M context) --- roles/web-app-docs/files/python/infinito_docs/builder.py | 1 + roles/web-app-docs/meta/services.yml | 6 +++--- roles/web-app-docs/vars/main.yml | 7 +++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/roles/web-app-docs/files/python/infinito_docs/builder.py b/roles/web-app-docs/files/python/infinito_docs/builder.py index 3c0260d51b..ad5852106e 100644 --- a/roles/web-app-docs/files/python/infinito_docs/builder.py +++ b/roles/web-app-docs/files/python/infinito_docs/builder.py @@ -88,6 +88,7 @@ class Builder: env = { **os.environ, "PYTHONPATH": os.pathsep.join([str(src), str(self.package_dir.parent)]), + "PYTHONUNBUFFERED": "1", } state["phase"] = "generate" for step, command in enumerate(generators, start=1): diff --git a/roles/web-app-docs/meta/services.yml b/roles/web-app-docs/meta/services.yml index d3864e60f0..79c0fc6ba2 100644 --- a/roles/web-app-docs/meta/services.yml +++ b/roles/web-app-docs/meta/services.yml @@ -38,8 +38,8 @@ docs: checksum_url: "https://cdn.jsdelivr.net/npm/mermaid@{version}/dist/mermaid.min.js" fetch_interval: 900 build: - jobs: "{{ [1, (RESOURCE_HOST_CPUS | int // 2)] | max }}" - parallel: "{{ [1, [2, (RESOURCE_HOST_CPUS | int // 2)] | min] | max }}" + sphinx_process_mb: 3072 + sphinx_overhead_mb: 2048 i18n: preload_languages: - "{{ HOST_LL }}" @@ -60,7 +60,7 @@ docs: lifecycle: beta cpus: "95%" mem_reservation: 512m - mem_limit: 4g + mem_limit: "{{ (lookup('resource', 'host_mem_mb') | int) * 2 // 3 }}M" pids_limit: 512 # nocheck: playwright-service-flag — design consumer, no Playwright surface (provider: web-svc-design) design: diff --git a/roles/web-app-docs/vars/main.yml b/roles/web-app-docs/vars/main.yml index ae933b6544..72e950640e 100644 --- a/roles/web-app-docs/vars/main.yml +++ b/roles/web-app-docs/vars/main.yml @@ -11,8 +11,11 @@ DOCS_SOURCE_REPO: "{{ lookup('config', application_id, 'services.docs.source_rep DOCS_MERMAID_VERSION: "{{ lookup('config', application_id, 'services.docs.mermaid_version') }}" DOCS_MERMAID_SHA256: "{{ lookup('config', application_id, 'services.docs.mermaid_sha256') }}" DOCS_FETCH_INTERVAL: "{{ lookup('config', application_id, 'services.docs.fetch_interval') }}" -DOCS_BUILD_JOBS: "{{ lookup('config', application_id, 'services.docs.build.jobs') | int }}" -DOCS_BUILD_PARALLEL: "{{ lookup('config', application_id, 'services.docs.build.parallel') | int }}" +DOCS_SPHINX_PROCESS_MB: "{{ lookup('config', application_id, 'services.docs.build.sphinx_process_mb') | int }}" +DOCS_SPHINX_OVERHEAD_MB: "{{ lookup('config', application_id, 'services.docs.build.sphinx_overhead_mb') | int }}" +DOCS_BUILD_MEM_MB: "{{ lookup('service_memory_mb', application_id, 'docs', 1.0) | int }}" +DOCS_BUILD_PARALLEL: "{{ [1, [2, (DOCS_BUILD_MEM_MB | int) // ((DOCS_SPHINX_OVERHEAD_MB | int) + (DOCS_SPHINX_PROCESS_MB | int))] | min] | max }}" +DOCS_BUILD_JOBS: "{{ [1, [(RESOURCE_HOST_CPUS | int // 2), ((DOCS_BUILD_MEM_MB | int) - (DOCS_BUILD_PARALLEL | int) * (DOCS_SPHINX_OVERHEAD_MB | int)) // ((DOCS_BUILD_PARALLEL | int) * (DOCS_SPHINX_PROCESS_MB | int))] | min] | max }}" DOCS_I18N_PRELOAD_LANGUAGES: "{{ lookup('config', application_id, 'services.docs.i18n.preload_languages') }}" DOCS_PORT: "{{ lookup('config', application_id, 'services.docs.ports.internal.http') }}" DOCS_LOCAL_BASE_URL: "http://{{ DOCKER_BIND_HOST }}:{{ lookup('config', application_id, 'services.docs.ports.local.http') }}"