fix(web-app-docs): preload the configured languages and wait for them

After a version build the builder queued every translated language, not the
configured ones. The list now reaches the library: server.py reads
DOCS_I18N_PRELOAD_LANGUAGES, Library freezes it into a set that drops the empty
string ''.split(',') yields when nothing is configured, and builder.py queues a
language only when it is in that set. Every other translation is built when a
visitor opens it.

The warm-up task accepted 202, which means queued, so the deploy walked on
while the sites were still building. A second task now waits for 200. Both read
DOCS_I18N_PRELOAD_PATHS, hoisted out of the loop it used to be written inline
in, because two tasks need the same language-code-to-URL-path mapping.

DOCS_BUILD_PARALLEL divides by overhead + 2 * process_mb instead of
overhead + process_mb: with a translation building beside the version build a
parallel slot holds two process-sized allocations. The new divisor is strictly
larger, so the value can only fall, never rise; with the configured 3072 and
2048 MB it differs from the old one only between 10 and 16 GB of service
memory, where it goes from 2 to 1.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

roles/web-app-docs/files/python/infinito_docs/builder.py

@@ -152,7 +152,8 @@ class Builder:
             self._publish(version, out / "html", ref)
             self._ready(version)
             for code in translated:
-                self.request(version, code, background=True)
+                if code in self.preload:
+                    self.request(version, code, background=True)
         except (OSError, subprocess.CalledProcessError, tarfile.TarError) as exc:
             self._failed(version, state, exc, ref)
         finally:

roles/web-app-docs/files/python/infinito_docs/library.py

@@ -31,9 +31,13 @@ class Library(Sites, Queue, Builder):
         jobs: parallel Sphinx processes per build.
         package_dir: the ``infinito_docs`` package directory.
         snapshot_dir: the deployed working tree, built as version ``deployed``.
+        preload: language codes a finished version build queues on its own;
+            every other translation is built when a visitor opens it.
     """
 
-    def __init__(self, repository, data_dir, jobs, package_dir, snapshot_dir):
+    def __init__(
+        self, repository, data_dir, jobs, package_dir, snapshot_dir, preload=()
+    ):
         data = Path(data_dir)
         self.repository = repository
         self.mirror = data / "repo.git"
@@ -45,6 +49,7 @@ class Library(Sites, Queue, Builder):
         self.prepared = data / "prepared"
         self.lock_file = data / "builder.lock"
         self.jobs = jobs
+        self.preload = frozenset(code for code in preload if code)
         self.package_dir = Path(package_dir)
         self.snapshot = Path(snapshot_dir)
         self._snapshot_ref = ""

roles/web-app-docs/files/python/infinito_docs/server.py

@@ -8,6 +8,8 @@ Environment:
     DOCS_FETCH_INTERVAL: seconds between two fetches of new commits and tags.
     DOCS_PORT: port to listen on.
     DOCS_SNAPSHOT_DIR: the deployed working tree, served as version ``deployed``.
+    DOCS_I18N_PRELOAD_LANGUAGES: comma-separated language codes a finished
+        version build queues on its own.
 """
 
 from __future__ import annotations
@@ -172,6 +174,7 @@ def main():
         int(os.environ["DOCS_BUILD_JOBS"]),
         PACKAGE_DIR,
         os.environ["DOCS_SNAPSHOT_DIR"],
+        os.environ["DOCS_I18N_PRELOAD_LANGUAGES"].split(","),
     )
     threading.Thread(
         target=library.run_builder,

roles/web-app-docs/tasks/main.yml

@@ -64,13 +64,21 @@
     url: "{{ DOCS_LOCAL_BASE_URL }}/deployed/{{ item }}"
     status_code: [200, 202]
     timeout: "{{ lookup('timeout', 30) | int }}"
-  loop: >-
-    {{ (DOCS_I18N_PRELOAD_LANGUAGES | reject('eq', HOST_LL)
-        | map('regex_replace', '$', '/') | list)
-       + (DOCS_I18N_PRELOAD_LANGUAGES | select('eq', HOST_LL)
-          | map('regex_replace', '.+', '') | list) }}
+  loop: "{{ DOCS_I18N_PRELOAD_PATHS }}"
   register: docs_deployed_warm
   until: docs_deployed_warm is not failed
   retries: 12
   delay: 10
   changed_when: false
+
+- name: "⏳ Wait for the preloaded sites of '{{ application_id }}'"
+  uri_retry:
+    url: "{{ DOCS_LOCAL_BASE_URL }}/deployed/{{ item }}"
+    status_code: [200]
+    timeout: "{{ lookup('timeout', 30) | int }}"
+  loop: "{{ DOCS_I18N_PRELOAD_PATHS }}"
+  register: docs_deployed_built
+  until: docs_deployed_built is not failed
+  retries: 120
+  delay: 30
+  changed_when: false

roles/web-app-docs/templates/env.j2

@@ -5,6 +5,7 @@ DOCS_DATA_DIR={{ DOCS_DATA_DIR }}
 DOCS_BUILD_JOBS={{ DOCS_BUILD_JOBS }}
 DOCS_BUILD_PARALLEL={{ DOCS_BUILD_PARALLEL }}
 DOCS_FETCH_INTERVAL={{ DOCS_FETCH_INTERVAL }}
+DOCS_I18N_PRELOAD_LANGUAGES={{ DOCS_I18N_PRELOAD_LANGUAGES | join(',') }}
 DOCS_PORT={{ DOCS_PORT }}
 DOCS_SNAPSHOT_DIR={{ DOCS_SNAPSHOT_DIR }}
 INFINITO_IMAGE={{ lookup('env', 'INFINITO_IMAGE') | dotenv_quote }}

roles/web-app-docs/vars/main.yml

@@ -14,10 +14,12 @@ DOCS_FETCH_INTERVAL: "{{ lookup('config', application_id, 'services.docs.fetch_i
 DOCS_SPHINX_PROCESS_MB: "{{ lookup('config', application_id, 'services.docs.build.sphinx_process_mb') | int }}"
 DOCS_SPHINX_OVERHEAD_MB: "{{ lookup('config', application_id, 'services.docs.build.sphinx_overhead_mb') | int }}"
 DOCS_BUILD_MEM_MB: "{{ lookup('service_memory_mb', application_id, 'docs', 1.0) | int }}"
-DOCS_BUILD_PARALLEL: "{{ [1, [2, (DOCS_BUILD_MEM_MB | int) // ((DOCS_SPHINX_OVERHEAD_MB | int) + (DOCS_SPHINX_PROCESS_MB | int))] | min] | max }}"
+DOCS_BUILD_PARALLEL: "{{ [1, [2, ((DOCS_BUILD_MEM_MB | int) // ((DOCS_SPHINX_OVERHEAD_MB | int) + 2 * (DOCS_SPHINX_PROCESS_MB | int)))] | min] | max }}"
 DOCS_BUILD_JOBS: "{{ [1, [(RESOURCE_HOST_CPUS | int // 2), ((DOCS_BUILD_MEM_MB | int) - (DOCS_BUILD_PARALLEL | int) * (DOCS_SPHINX_OVERHEAD_MB | int)) // ((DOCS_BUILD_PARALLEL
   | int) * (DOCS_SPHINX_PROCESS_MB | int))] | min] | max }}"
 DOCS_I18N_PRELOAD_LANGUAGES: "{{ lookup('config', application_id, 'services.docs.i18n.preload_languages') }}"
+DOCS_I18N_PRELOAD_PATHS: "{{ (DOCS_I18N_PRELOAD_LANGUAGES | reject('eq', HOST_LL) | map('regex_replace', '$', '/') | list) + (DOCS_I18N_PRELOAD_LANGUAGES
+  | select('eq', HOST_LL) | map('regex_replace', '.+', '') | list) }}"
 DOCS_PORT: "{{ lookup('config', application_id, 'services.docs.ports.internal.http') }}"
 DOCS_LOCAL_BASE_URL: "http://{{ DOCKER_BIND_HOST }}:{{ lookup('config', application_id, 'services.docs.ports.local.http') }}"
 DOCS_DATA_DIR: "/data"

tests/unit/python/roles/web-app-docs/files/infinito_docs/test_builder.py

@@ -167,6 +167,23 @@ class TestBuilder(LibraryFixture, unittest.TestCase):
                 encoding="utf-8",
             )
 
+    def test_a_language_outside_the_preload_set_is_not_prefetched(self) -> None:
+        self._translated_repo("de", "az")
+        _commit(self.repo, "two languages")
+        self.library.fetch()
+
+        self.library.build("latest")
+
+        self.assertTrue(self.library.translates("latest", "az"))
+        self.assertTrue((self.library.queue / "background" / "latest:de").exists())
+        self.assertFalse((self.library.queue / "background" / "latest:az").exists())
+
+        self.library.request("latest", "az")
+        self.assertTrue(
+            (self.library.queue / "latest:az").exists(),
+            "a visitor asking for an unpreloaded language must still queue it",
+        )
+
     def test_a_new_commit_rebuilds_the_translated_sites_too(self) -> None:
         self._translated_repo("de")
         _commit(self.repo, "first")

tests/unit/python/roles/web-app-docs/files/infinito_docs/test_library.py

@@ -116,9 +116,14 @@ class LibraryFixture(unittest.TestCase):
         self._env.stop()
         self._tmp.cleanup()
 
-    def _replica(self):
+    def _replica(self, preload=("de", "fr", "it")):
         return library.Library(
-            str(self.repo), self.data, 1, self.package, self.data / "no-snapshot"
+            str(self.repo),
+            self.data,
+            1,
+            self.package,
+            self.data / "no-snapshot",
+            preload,
         )
 
     def _site(self, version: str, name: str) -> str: