mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-09 20:26:33 +00:00
fix(web-app-docs): stop translated pages from saving a copy of the build environment
Sphinx's i18n transform parses every msgstr into a temporary document whose settings carry the whole BuildEnvironment (sphinx/transforms/ i18n.py, Sphinx 9.2). MyST nodes patched into the page keep pointing at that document, and write_doctree only detaches the environment from the page's own settings, so every translated Markdown page was pickled with a full environment copy: a German doctree of drv-lid-switch grew from 21 KB to 5.5 MB, the whole doctree tree from 95 MB to 2.5 GB. The write phase loads them all back and the forked -j worker copies the parent, so the `deployed:de` build reached 5+ GB in the parent and was OOM-killed inside the docs container in CI run 40625679270 (Jenkins#1, Nextcloud#11), drove shared 16 GB swarm runners into swap and the watchdog's memory and I/O-wedge aborts (GitLab#2, Nextcloud#7/#9/#10), and left the daemon answering 500 to the queue task for the failed build. AdoptTranslatedNodes runs right after the Locale transform and points every node whose document is not the page back at the page; conf.py loads it for every build the image runs. No size, limit or budget changes, and docs stays deployed wherever a variant declares it. Measured locally on the full German build at -j 2: parent peak 685 MB (was 5.3 GB and climbing when stopped at about a quarter of the write phase), total 2.4 GB (was 8.35 GB), doctrees 92 MB (was 2.5 GB). The English HTML of all 5,373 pages is byte-identical before and after. A real German MyST build in the new test finds no foreign nodes and no BuildEnvironment in the saved page; without the transform it fails. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8a0e858f12
commit
8eda7a30bb
3 files changed
+162
No files matched your search
+138
@@ -0,0 +1,138 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import patch
|
||||
|
||||
from babel.messages.catalog import Catalog
|
||||
from babel.messages.pofile import write_po
|
||||
from docutils import nodes
|
||||
from docutils.frontend import get_default_settings
|
||||
from docutils.parsers.rst import Parser
|
||||
from docutils.utils import new_document
|
||||
|
||||
from utils.cache.files import read_text
|
||||
|
||||
from . import PROJECT_ROOT
|
||||
|
||||
_TOOLING = str(PROJECT_ROOT / "roles" / "web-app-docs" / "files" / "python")
|
||||
if _TOOLING not in sys.path:
|
||||
sys.path.insert(0, _TOOLING)
|
||||
|
||||
from infinito_docs.identity import docs_sphinx_identity # noqa: E402
|
||||
|
||||
translated_nodes = importlib.import_module("infinito_docs.extensions.translated_nodes")
|
||||
|
||||
EXTENSION = "infinito_docs.extensions.translated_nodes"
|
||||
MESSAGES = {
|
||||
"Page": "Seite",
|
||||
"Another sentence with *emphasis*.": "Ein weiterer Satz mit *Betonung*.",
|
||||
}
|
||||
|
||||
|
||||
def _build_german_page(
|
||||
root: Path, extensions: list[str]
|
||||
) -> tuple[nodes.document, bytes, str]:
|
||||
"""Build one translated Markdown page in German.
|
||||
|
||||
Args:
|
||||
root: empty scratch directory.
|
||||
extensions: extensions loaded besides ``myst_parser``.
|
||||
|
||||
Returns:
|
||||
The page's doctree as Sphinx reads it back, its raw pickle and the HTML.
|
||||
"""
|
||||
from sphinx.application import Sphinx
|
||||
|
||||
src = root / "src"
|
||||
catalogs = src / "locale" / "de" / "LC_MESSAGES"
|
||||
catalogs.mkdir(parents=True)
|
||||
(src / "conf.py").write_text(
|
||||
f"extensions = {['myst_parser', *extensions]!r}\n"
|
||||
"locale_dirs = ['locale']\n"
|
||||
"gettext_compact = 'docs'\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(src / "index.md").write_text(
|
||||
"# Page\n\nAnother sentence with *emphasis*.\n", encoding="utf-8"
|
||||
)
|
||||
catalog = Catalog(locale="de")
|
||||
for msgid, msgstr in MESSAGES.items():
|
||||
catalog.add(msgid, msgstr)
|
||||
with (catalogs / "docs.po").open("wb") as handle:
|
||||
write_po(handle, catalog)
|
||||
app = Sphinx(
|
||||
str(src),
|
||||
str(src),
|
||||
str(root / "html"),
|
||||
str(root / "doctrees"),
|
||||
"html",
|
||||
confoverrides={"language": "de"},
|
||||
status=None,
|
||||
warning=None,
|
||||
freshenv=True,
|
||||
)
|
||||
app.build()
|
||||
with (root / "doctrees" / "index.doctree").open("rb") as handle:
|
||||
raw = handle.read()
|
||||
return (
|
||||
app.env.get_doctree("index"),
|
||||
raw,
|
||||
read_text(str(root / "html" / "index.html")),
|
||||
)
|
||||
|
||||
|
||||
class TestAdoptTranslatedNodes(unittest.TestCase):
|
||||
def test_nodes_of_a_foreign_document_are_handed_to_the_page(self) -> None:
|
||||
settings = get_default_settings(Parser)
|
||||
page = new_document("/src/page.md", settings)
|
||||
msgstr = new_document("/src/page.md:3:<translated>", settings)
|
||||
paragraph = nodes.paragraph()
|
||||
text = nodes.Text("Ein weiterer Satz")
|
||||
page += paragraph
|
||||
paragraph += text
|
||||
paragraph.document = msgstr
|
||||
text.document = msgstr
|
||||
self.assertIs(paragraph.document, msgstr)
|
||||
self.assertIs(text.document, msgstr)
|
||||
|
||||
translated_nodes.AdoptTranslatedNodes(page).apply()
|
||||
|
||||
self.assertIs(paragraph.document, page)
|
||||
self.assertIs(text.document, page)
|
||||
|
||||
def test_it_runs_right_after_the_sphinx_locale_transform(self) -> None:
|
||||
from sphinx.transforms.i18n import Locale
|
||||
|
||||
self.assertEqual(
|
||||
translated_nodes.AdoptTranslatedNodes.default_priority,
|
||||
Locale.default_priority + 1,
|
||||
)
|
||||
|
||||
def test_conf_loads_the_extension(self) -> None:
|
||||
with (
|
||||
patch.object(sys, "argv", ["sphinx-build"]),
|
||||
patch.dict("os.environ", docs_sphinx_identity(PROJECT_ROOT)),
|
||||
):
|
||||
conf = importlib.import_module("infinito_docs.conf")
|
||||
|
||||
self.assertIn(EXTENSION, conf.extensions)
|
||||
|
||||
def test_a_translated_markdown_page_pickles_without_the_environment(self) -> None:
|
||||
with TemporaryDirectory() as td:
|
||||
doctree, raw, html = _build_german_page(Path(td), [EXTENSION])
|
||||
|
||||
self.assertEqual(
|
||||
[node for node in doctree.findall() if node.document is not doctree],
|
||||
[],
|
||||
)
|
||||
self.assertNotIn(b"BuildEnvironment", raw)
|
||||
self.assertIn("Betonung", html)
|
||||
self.assertIn("Seite", html)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user