From 8eda7a30bb340fafe5b78f1bf46c4baea2fa2804 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 28 Oct 2026 20:16:48 +0100 Subject: [PATCH] fix(web-app-docs): stop translated pages from saving a copy of the build environment Sphinx's i18n transform parses every msgstr into a temporary document whose settings carry the whole BuildEnvironment (sphinx/transforms/ i18n.py, Sphinx 9.2). MyST nodes patched into the page keep pointing at that document, and write_doctree only detaches the environment from the page's own settings, so every translated Markdown page was pickled with a full environment copy: a German doctree of drv-lid-switch grew from 21 KB to 5.5 MB, the whole doctree tree from 95 MB to 2.5 GB. The write phase loads them all back and the forked -j worker copies the parent, so the `deployed:de` build reached 5+ GB in the parent and was OOM-killed inside the docs container in CI run 40625679270 (Jenkins#1, Nextcloud#11), drove shared 16 GB swarm runners into swap and the watchdog's memory and I/O-wedge aborts (GitLab#2, Nextcloud#7/#9/#10), and left the daemon answering 500 to the queue task for the failed build. AdoptTranslatedNodes runs right after the Locale transform and points every node whose document is not the page back at the page; conf.py loads it for every build the image runs. No size, limit or budget changes, and docs stays deployed wherever a variant declares it. Measured locally on the full German build at -j 2: parent peak 685 MB (was 5.3 GB and climbing when stopped at about a quarter of the write phase), total 2.4 GB (was 8.35 GB), doctrees 92 MB (was 2.5 GB). The English HTML of all 5,373 pages is byte-identical before and after. A real German MyST build in the new test finds no foreign nodes and no BuildEnvironment in the saved page; without the transform it fails. Co-Authored-By: Claude Opus 5.5 --- .../files/python/infinito_docs/conf.py | 1 + .../extensions/translated_nodes.py | 23 +++ .../extensions/test_translated_nodes.py | 138 ++++++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 roles/web-app-docs/files/python/infinito_docs/extensions/translated_nodes.py create mode 100644 tests/unit/python/roles/web-app-docs/files/infinito_docs/extensions/test_translated_nodes.py diff --git a/roles/web-app-docs/files/python/infinito_docs/conf.py b/roles/web-app-docs/files/python/infinito_docs/conf.py index 8b8799141a..851100c1d7 100644 --- a/roles/web-app-docs/files/python/infinito_docs/conf.py +++ b/roles/web-app-docs/files/python/infinito_docs/conf.py @@ -63,6 +63,7 @@ extensions = [ "infinito_docs.extensions.parallel_postprocess", "infinito_docs.extensions.untranslated.yaml", "infinito_docs.extensions.untranslated.markup", + "infinito_docs.extensions.translated_nodes", "infinito_docs.extensions.yaml_source", "infinito_docs.extensions.directory_index", "infinito_docs.extensions.mermaid", diff --git a/roles/web-app-docs/files/python/infinito_docs/extensions/translated_nodes.py b/roles/web-app-docs/files/python/infinito_docs/extensions/translated_nodes.py new file mode 100644 index 0000000000..283ab198f9 --- /dev/null +++ b/roles/web-app-docs/files/python/infinito_docs/extensions/translated_nodes.py @@ -0,0 +1,23 @@ +"""Hand every node of a translated message over to the page it was patched into.""" + +from __future__ import annotations + +from sphinx.transforms import SphinxTransform +from sphinx.transforms.i18n import Locale + + +class AdoptTranslatedNodes(SphinxTransform): + """Point each node that still names its msgstr's own document at the page.""" + + default_priority = Locale.default_priority + 1 + + def apply(self, **kwargs) -> None: + """Rebind every node whose document is not the page being read.""" + for node in self.document.findall(): + if node.document is not self.document: + node.document = self.document + + +def setup(app): + app.add_transform(AdoptTranslatedNodes) + return {"version": "1.0", "parallel_read_safe": True} diff --git a/tests/unit/python/roles/web-app-docs/files/infinito_docs/extensions/test_translated_nodes.py b/tests/unit/python/roles/web-app-docs/files/infinito_docs/extensions/test_translated_nodes.py new file mode 100644 index 0000000000..c6e27ad6f6 --- /dev/null +++ b/tests/unit/python/roles/web-app-docs/files/infinito_docs/extensions/test_translated_nodes.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import importlib +import sys +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from babel.messages.catalog import Catalog +from babel.messages.pofile import write_po +from docutils import nodes +from docutils.frontend import get_default_settings +from docutils.parsers.rst import Parser +from docutils.utils import new_document + +from utils.cache.files import read_text + +from . import PROJECT_ROOT + +_TOOLING = str(PROJECT_ROOT / "roles" / "web-app-docs" / "files" / "python") +if _TOOLING not in sys.path: + sys.path.insert(0, _TOOLING) + +from infinito_docs.identity import docs_sphinx_identity # noqa: E402 + +translated_nodes = importlib.import_module("infinito_docs.extensions.translated_nodes") + +EXTENSION = "infinito_docs.extensions.translated_nodes" +MESSAGES = { + "Page": "Seite", + "Another sentence with *emphasis*.": "Ein weiterer Satz mit *Betonung*.", +} + + +def _build_german_page( + root: Path, extensions: list[str] +) -> tuple[nodes.document, bytes, str]: + """Build one translated Markdown page in German. + + Args: + root: empty scratch directory. + extensions: extensions loaded besides ``myst_parser``. + + Returns: + The page's doctree as Sphinx reads it back, its raw pickle and the HTML. + """ + from sphinx.application import Sphinx + + src = root / "src" + catalogs = src / "locale" / "de" / "LC_MESSAGES" + catalogs.mkdir(parents=True) + (src / "conf.py").write_text( + f"extensions = {['myst_parser', *extensions]!r}\n" + "locale_dirs = ['locale']\n" + "gettext_compact = 'docs'\n", + encoding="utf-8", + ) + (src / "index.md").write_text( + "# Page\n\nAnother sentence with *emphasis*.\n", encoding="utf-8" + ) + catalog = Catalog(locale="de") + for msgid, msgstr in MESSAGES.items(): + catalog.add(msgid, msgstr) + with (catalogs / "docs.po").open("wb") as handle: + write_po(handle, catalog) + app = Sphinx( + str(src), + str(src), + str(root / "html"), + str(root / "doctrees"), + "html", + confoverrides={"language": "de"}, + status=None, + warning=None, + freshenv=True, + ) + app.build() + with (root / "doctrees" / "index.doctree").open("rb") as handle: + raw = handle.read() + return ( + app.env.get_doctree("index"), + raw, + read_text(str(root / "html" / "index.html")), + ) + + +class TestAdoptTranslatedNodes(unittest.TestCase): + def test_nodes_of_a_foreign_document_are_handed_to_the_page(self) -> None: + settings = get_default_settings(Parser) + page = new_document("/src/page.md", settings) + msgstr = new_document("/src/page.md:3:", settings) + paragraph = nodes.paragraph() + text = nodes.Text("Ein weiterer Satz") + page += paragraph + paragraph += text + paragraph.document = msgstr + text.document = msgstr + self.assertIs(paragraph.document, msgstr) + self.assertIs(text.document, msgstr) + + translated_nodes.AdoptTranslatedNodes(page).apply() + + self.assertIs(paragraph.document, page) + self.assertIs(text.document, page) + + def test_it_runs_right_after_the_sphinx_locale_transform(self) -> None: + from sphinx.transforms.i18n import Locale + + self.assertEqual( + translated_nodes.AdoptTranslatedNodes.default_priority, + Locale.default_priority + 1, + ) + + def test_conf_loads_the_extension(self) -> None: + with ( + patch.object(sys, "argv", ["sphinx-build"]), + patch.dict("os.environ", docs_sphinx_identity(PROJECT_ROOT)), + ): + conf = importlib.import_module("infinito_docs.conf") + + self.assertIn(EXTENSION, conf.extensions) + + def test_a_translated_markdown_page_pickles_without_the_environment(self) -> None: + with TemporaryDirectory() as td: + doctree, raw, html = _build_german_page(Path(td), [EXTENSION]) + + self.assertEqual( + [node for node in doctree.findall() if node.document is not doctree], + [], + ) + self.assertNotIn(b"BuildEnvironment", raw) + self.assertIn("Betonung", html) + self.assertIn("Seite", html) + + +if __name__ == "__main__": + unittest.main()