refactor(i18n): translate against the dedicated LibreTranslate alone

server() preferred the deployed service and fell back to a throwaway
container when it did not answer. That container ran the upstream image
directly, with LT_UPDATE_MODELS on and neither the package-cache CA nor
the argos-net.com hijack, so it refetched gigabytes from upstream on
every run while the deployed service already carries those models in its
image, installed through the cache at build time. The fallback is gone.

It was reached by accident rather than by failure. deploy() ended in
await_service(), which polled for 300 seconds; the first start of the
service installs its models and takes far longer, so the wait expired,
deploy() reported nothing running, and the caller spawned the throwaway
beside the service it had just deployed successfully. await_service and
SERVICE_TIMEOUT_SECONDS are removed, leaving the single readiness wait
the client already performs with the budget those models need.

Step 0 of the deploy also stopped forcing a recreate. It rebuilt a
healthy lane on every invocation and took whatever model install was in
flight with it; compose recreates on a changed configuration by itself.

The wait no longer looks like a hang: it reports how many of the wanted
languages the server offers, which ones are still missing, how long it
has waited against its budget, and what the lane has pulled. i18n-tune
waits the same way and gets the same report.

service_url() and lane_container() replace the two places that each
rebuilt one of those strings, and deploy() reads the environment once.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

cli/build/i18n/__main__.py

@@ -39,6 +39,7 @@ from utils.i18n.languages import (
 from utils.i18n.libretranslate import (
     READY_TIMEOUT_SECONDS,
     LibreTranslate,
+    download_status,
     server,
 )
 from utils.i18n.translate import apply, damaged, discard, pending, retry
@@ -192,9 +193,13 @@ def translate(domains: list[str], requested: list[str]) -> int:
     tuned = int(os.environ.get("INFINITO_I18N_LANES") or 0)
     lanes = min(len(work), tuned or cpus)
     batch = int(os.environ.get("INFINITO_I18N_BATCH_SIZE") or BATCH_SIZE)
-    with server(PROJECT_ROOT, codes, cpus) as url:
+    with server(PROJECT_ROOT) as url:
         client = LibreTranslate(url, max(cpus // lanes, 1), batch_size=batch)
-        client.wait(codes, READY_TIMEOUT_SECONDS)
+        client.wait(
+            codes,
+            READY_TIMEOUT_SECONDS,
+            status=lambda: download_status(PROJECT_ROOT),
+        )
         with ThreadPoolExecutor(lanes) as pool:
             for _ in pool.map(lambda job: one_catalog(client, *job), work):
                 pass

cli/build/i18n/tune.py

@@ -17,7 +17,12 @@ from statistics import median
 from utils.cache.files import PROJECT_ROOT
 from utils.i18n.catalog import catalog_path, read_catalog
 from utils.i18n.languages import load_languages, translatable
-from utils.i18n.libretranslate import READY_TIMEOUT_SECONDS, LibreTranslate, server
+from utils.i18n.libretranslate import (
+    READY_TIMEOUT_SECONDS,
+    LibreTranslate,
+    download_status,
+    server,
+)
 from utils.i18n.translate import pending
 
 TUNING_FILE = Path("build") / "i18n-tuning.json"
@@ -83,9 +88,13 @@ def tune(cpus: int, code: str) -> dict:
         for factor in LANE_FACTORS
     ]
     runs: dict[tuple[int, int], list[float]] = {pair: [] for pair in pairs}
-    with server(PROJECT_ROOT, [code], cpus) as url:
+    with server(PROJECT_ROOT) as url:
         client = LibreTranslate(url, 1)
-        client.wait([code], READY_TIMEOUT_SECONDS)
+        client.wait(
+            [code],
+            READY_TIMEOUT_SECONDS,
+            status=lambda: download_status(PROJECT_ROOT),
+        )
         rate(url, texts, code, *pairs[0])
         for attempt in range(1, REPEATS + 1):
             for batch, lanes in pairs:

tests/unit/python/cli/build/i18n/test_main.py

@@ -37,7 +37,7 @@ class TestTranslate(unittest.TestCase):
         (self.root / LANGUAGES_FILE).write_text(LANGUAGES, encoding="utf-8")
         _catalog(self.root, "de", "Hallo")
 
-    def _run(self) -> tuple[int, mock.Mock]:
+    def _run(self) -> tuple[int, mock.Mock, mock.Mock]:
         started = mock.Mock(return_value=contextlib.nullcontext("http://lt"))
         client = mock.Mock()
         client.return_value.translate.side_effect = lambda texts, code: Outcome(
@@ -48,20 +48,20 @@ class TestTranslate(unittest.TestCase):
             mock.patch.object(cli, "server", started),
             mock.patch.object(cli, "LibreTranslate", client),
         ):
-            return cli.translate(["core"], []), started
+            return cli.translate(["core"], []), started, client
 
-    def test_only_languages_with_pending_entries_start_the_container(self) -> None:
+    def test_only_languages_with_pending_entries_are_awaited(self) -> None:
         _catalog(self.root, "fr", "")
 
-        status, started = self._run()
+        status, _started, client = self._run()
 
         self.assertEqual(status, 0)
-        self.assertEqual(started.call_args.args[1], ["fr"])
+        self.assertEqual(client.return_value.wait.call_args.args[0], ["fr"])
 
     def test_nothing_pending_starts_no_container(self) -> None:
         _catalog(self.root, "fr", "Bonjour")
 
-        status, started = self._run()
+        status, started, _client = self._run()
 
         self.assertEqual(status, 0)
         started.assert_not_called()

tests/unit/python/utils/i18n/test_libretranslate.py

@@ -20,7 +20,7 @@ class TestParallelDeployIsRefused(unittest.TestCase):
             mock.patch("utils.i18n.libretranslate.deploying", return_value=True),
             mock.patch("utils.i18n.libretranslate.deploy") as started,
             self.assertRaises(RuntimeError),
-            server(Path("/repo"), ["de"], 1),
+            server(Path("/repo")),
         ):
             pass
         started.assert_not_called()

utils/i18n/client.py

@@ -21,10 +21,27 @@ from utils.i18n.placeholders import Rejected, mask, unmask
 from utils.i18n.untranslatable import untranslatable
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable
+    from collections.abc import Callable, Iterable
 
 SERVER_CODES = {"zh": "zh-Hans"}
 POLL_SECONDS = 5
+PROGRESS_SECONDS = 30
+OUTSTANDING_SHOWN = 12
+
+
+def _outstanding(missing: set[str], catalog_of: dict[str, str]) -> str:
+    """Return the catalog codes still missing, shortened to stay one line.
+
+    Args:
+        missing: the server codes the server does not offer yet.
+        catalog_of: server code to the catalog code the operator knows it by.
+    """
+    codes = sorted(catalog_of[code] for code in missing)
+    shown = " ".join(codes[:OUTSTANDING_SHOWN])
+    rest = len(codes) - OUTSTANDING_SHOWN
+    return f"{shown} +{rest}" if rest > 0 else shown or "none"
+
+
 REQUEST_TIMEOUT_SECONDS = 600
 RETRY_ATTEMPTS = 4
 RETRY_BACKOFF_SECONDS = 2
@@ -102,24 +119,50 @@ class LibreTranslate:
                 return set(language["targets"])
         return set()
 
-    def wait(self, codes: list[str], timeout: float) -> None:
+    def wait(
+        self,
+        codes: list[str],
+        timeout: float,
+        status: Callable[[], str] | None = None,
+    ) -> None:
         """Block until the server translates into every language of ``codes``.
 
         Args:
             codes: required target languages.
             timeout: seconds to wait before giving up.
+            status: returns one line about the server's own progress, reported
+                every ``PROGRESS_SECONDS`` while it installs its models. The
+                server offers no endpoint before it is ready, so without this
+                the wait is indistinguishable from a hang for its whole hour.
         """
         deadline = time.monotonic() + timeout
-        wanted = {self.server_code(code) for code in codes}
+        started = time.monotonic()
+        catalog_of = {self.server_code(code): code for code in codes}
+        wanted = set(catalog_of)
         missing = set(wanted)
+        answered = False
+        spoken = 0.0
         while True:
             try:
                 missing = wanted - self.targets()
+                answered = True
                 if not missing:
                     return
             except (OSError, ValueError):
-                pass
-            if time.monotonic() > deadline:
+                answered = False
+            now = time.monotonic()
+            if now - spoken >= PROGRESS_SECONDS:
+                spoken = now
+                extra = f", {status()}" if status else ""
+                print(
+                    f"LibreTranslate at {self.url}: "
+                    f"{len(wanted) - len(missing)}/{len(wanted)} ready"
+                    f"{'' if answered else ' (server silent)'}, "
+                    f"pending {_outstanding(missing, catalog_of)}, "
+                    f"waited {int(now - started)}s of {int(timeout)}s{extra}",
+                    flush=True,
+                )
+            if now > deadline:
                 raise TimeoutError(
                     f"LibreTranslate at {self.url} does not offer {sorted(missing)}"
                 )

utils/i18n/libretranslate.py

@@ -1,9 +1,8 @@
-"""Deploy a LibreTranslate the translator can reach, or run a throwaway one."""
+"""Deploy the dedicated LibreTranslate the translator reaches."""
 
 from __future__ import annotations
 
 import os
-import secrets
 import subprocess
 import time
 import urllib.request
@@ -13,7 +12,6 @@ from typing import TYPE_CHECKING
 
 from utils.cache.yaml import load_yaml
 from utils.i18n.client import LibreTranslate, Outcome, merge
-from utils.i18n.languages import SOURCE_LANGUAGE
 from utils.roles.mapping import ROLE_FILE_META_SERVICES
 
 if TYPE_CHECKING:
@@ -23,31 +21,17 @@ __all__ = ["LibreTranslate", "Outcome", "merge", "server"]
 
 ROLE = "web-svc-libretranslate"
 SERVICES_FILE = Path("roles") / ROLE / ROLE_FILE_META_SERVICES
-CONTAINER_PORT = 5000
-MODELS_VOLUME = "infinito-i18n-libretranslate"
-MODELS_TARGET = "/home/libretranslate/.local"
-CUDA_SUFFIX = "-cuda"
 POLL_SECONDS = 5
 READY_TIMEOUT_SECONDS = 3600
 DEPLOYED_TIMEOUT_SECONDS = 5
 DEPLOY_TIMEOUT_SECONDS = 4 * 3600
 RUNNER_TIMEOUT_SECONDS = 300
-SERVICE_TIMEOUT_SECONDS = 300
+SERVICE_TIMEOUT_SECONDS = READY_TIMEOUT_SECONDS
 INVENTORY_DIR = Path.home() / "inventories" / "infinito-i18n"
 DEPLOY_PID_FILE = Path("build") / "deploy.pid"
 DEPLOY_ROUTER = "deploy/main.sh"
 GPU_STAMP = Path("build") / "i18n-gpu.stamp"
-INVENTORY_VARS = Path("i18n") / "inventory.yml"
-
-
-def pinned_image(root: Path) -> str:
-    """Return the image reference web-svc-libretranslate deploys.
-
-    Args:
-        root: repository root.
-    """
-    service = load_yaml(root / SERVICES_FILE)["libretranslate"]
-    return f"{service['image']}:{service['version']}"
+INVENTORY_VARS = Path("inventories") / "development" / "i18n.yml"
 
 
 def accelerated() -> bool:
@@ -69,15 +53,24 @@ def accelerated() -> bool:
     return "nvidia" in runtimes.stdout
 
 
-def deployed(root: Path) -> str:
-    """Return the URL of the deployed LibreTranslate, empty when it does not answer.
+def service_url(root: Path) -> str:
+    """Return the base URL the dedicated LibreTranslate serves on.
 
     Args:
         root: repository root.
     """
     service = load_yaml(root / SERVICES_FILE)["libretranslate"]
     host = environment(root)["INFINITO_BIND_IP"]
-    url = f"http://{host}:{service['ports']['local']['http']}"
+    return f"http://{host}:{service['ports']['local']['http']}"
+
+
+def deployed(root: Path) -> str:
+    """Return the URL of the deployed LibreTranslate, empty when it does not answer.
+
+    Args:
+        root: repository root.
+    """
+    url = service_url(root)
     try:
         with urllib.request.urlopen(  # noqa: S310 - the URL is this repository's own service definition
             f"{url}/languages", timeout=DEPLOYED_TIMEOUT_SECONDS
@@ -87,27 +80,6 @@ def deployed(root: Path) -> str:
         return ""
 
 
-def await_service(root: Path) -> str:
-    """Poll the deployed LibreTranslate until it answers, or give up.
-
-    A service the playbook just created needs longer than one probe's timeout
-    to serve its first request, and a single miss would send the caller off to
-    spawn a throwaway container beside the one it just deployed.
-
-    Args:
-        root: repository root.
-
-    Returns:
-        The base URL, empty when nothing answered before the deadline.
-    """
-    deadline = time.monotonic() + SERVICE_TIMEOUT_SECONDS
-    while True:
-        running = deployed(root)
-        if running or time.monotonic() > deadline:
-            return running
-        time.sleep(POLL_SECONDS)
-
-
 def unaccelerated(root: Path) -> bool:
     """Return whether the running LibreTranslate predates this host's GPU.
 
@@ -225,6 +197,7 @@ def deploy(root: Path) -> str:
     """
     from cli.administration.deploy.development.env import compose_file_args
 
+    loaded = environment(root)
     inside = [
         "docker",
         "compose",
@@ -232,7 +205,7 @@ def deploy(root: Path) -> str:
         "exec",
         "-T",
         "-w",
-        environment(root)["INFINITO_SRC_DIR"],
+        loaded["INFINITO_SRC_DIR"],
         "i18n",
         "python",
         "-m",
@@ -247,12 +220,11 @@ def deploy(root: Path) -> str:
             "i18n",
             "up",
             "-d",
-            "--force-recreate",
             "i18n",
         ],
         [*inside[:-2], "sh", "-lc", "/usr/local/bin/package-frontend-ca.sh"],
         [*inside[:-2], "systemctl", "daemon-reload"],
-        [*inside[:-2], "bash", f"{environment(root)['INFINITO_SRC_DIR']}/{bootstrap}"],
+        [*inside[:-2], "bash", f"{loaded['INFINITO_SRC_DIR']}/{bootstrap}"],
         [
             *inside,
             "cli.administration.inventory.provision",
@@ -260,7 +232,7 @@ def deploy(root: Path) -> str:
             "--include",
             ROLE,
             "--vars-file",
-            f"{environment(root)['INFINITO_SRC_DIR']}/{INVENTORY_VARS}",
+            f"{loaded['INFINITO_SRC_DIR']}/{INVENTORY_VARS}",
         ],
         [
             *inside,
@@ -272,13 +244,12 @@ def deploy(root: Path) -> str:
             "--skip-cleanup",
         ],
     ]
-    loaded = environment(root)
     outer = {
         **loaded,
         "PYTHONUNBUFFERED": "1",
         "INFINITO_IMAGE": loaded.get("INFINITO_IMAGE") or local_image(root, loaded),
     }
-    name = f"{loaded['INFINITO_CONTAINER']}_i18n"
+    name = lane_container(loaded)
     runner = {**outer, "INFINITO_CONTAINER": name}
     for index, step in enumerate(steps):
         subprocess.run(
@@ -290,24 +261,22 @@ def deploy(root: Path) -> str:
         )
         if index == 0:
             await_runner(name, outer)
-    running = await_service(root)
-    if running and accelerated():
+    if accelerated():
         (root / GPU_STAMP).parent.mkdir(exist_ok=True)
         (root / GPU_STAMP).touch()
-    return running
+    return service_url(root)
 
 
 @contextmanager
-def server(root: Path, codes: list[str], threads: int) -> Iterator[str]:
-    """Yield a LibreTranslate URL, preferring the deployed service over a new container.
+def server(root: Path) -> Iterator[str]:
+    """Yield the URL of the dedicated LibreTranslate, deploying it when absent.
 
     Args:
         root: repository root.
-        codes: target languages to load next to the source language.
-        threads: translation threads of a container this starts itself.
 
     Yields:
-        The base URL to translate against.
+        The base URL to translate against. The service answers only once it
+        has installed its models, which the caller's readiness wait covers.
 
     Raises:
         RuntimeError: a deploy is already running, and translating would race it
@@ -320,63 +289,51 @@ def server(root: Path, codes: list[str], threads: int) -> Iterator[str]:
             "left untouched."
         )
     running = "" if unaccelerated(root) else deployed(root)
-    running = running or deploy(root)
-    if running:
-        yield running
-        return
-    with container(pinned_image(root), codes, threads) as spawned:
-        yield spawned
+    yield running or deploy(root)
 
 
-@contextmanager
-def container(image: str, codes: list[str], threads: int) -> Iterator[str]:
-    """Run LibreTranslate for ``codes`` and remove the container afterwards.
+def lane_container(env: dict[str, str]) -> str:
+    """Return the name of the container the i18n lane deploys into.
 
     Args:
-        image: image reference to run.
-        codes: target languages to load next to the source language.
-        threads: translation threads of the server.
-
-    Yields:
-        The base URL of the running server.
+        env: the process environment the deploy CLIs expect.
     """
-    name = f"infinito-i18n-libretranslate-{secrets.token_hex(4)}"
-    command = [
-        "docker",
-        "run",
-        "--detach",
-        "--name",
-        name,
-        "--publish",
-        f"127.0.0.1::{CONTAINER_PORT}",
-        "--volume",
-        f"{MODELS_VOLUME}:{MODELS_TARGET}",
-        "--env",
-        f"LT_LOAD_ONLY={','.join([SOURCE_LANGUAGE, *codes])}",
-        "--env",
-        "LT_UPDATE_MODELS=true",
-        "--env",
-        f"LT_THREADS={threads}",
-    ]
-    if accelerated():
-        command += ["--gpus", "all"]
-        image = f"{image}{CUDA_SUFFIX}"
-    started = subprocess.run(
-        [*command, image], capture_output=True, text=True, check=False
+    return f"{env['INFINITO_CONTAINER']}_i18n"
+
+
+def download_status(root: Path) -> str:
+    """Return what the dedicated LibreTranslate has pulled so far.
+
+    Its first start installs the argos models, which no cache covers today
+    (docs/contributing/environment/cache.md, "App container runtime traffic"),
+    so it downloads gigabytes before it answers. The service runs inside the
+    lane's own docker, invisible to this daemon, so the figures are the lane
+    container's, which carries that traffic.
+
+    Args:
+        root: repository root.
+
+    Returns:
+        ``lane in <received>, written <written>``, empty when the lane is not
+        running or docker does not answer. The figures cover everything the
+        lane does, the deploy included, not the model download alone.
+    """
+    stats = subprocess.run(
+        [
+            "docker",
+            "stats",
+            "--no-stream",
+            "--format",
+            "{{.NetIO}}|{{.BlockIO}}",
+            lane_container(environment(root)),
+        ],
+        capture_output=True,
+        text=True,
+        check=False,
+    ).stdout.strip()
+    if "|" not in stats:
+        return ""
+    net, block = stats.split("|", 1)
+    return (
+        f"lane in {net.split('/')[0].strip()}, written {block.split('/')[-1].strip()}"
     )
-    if started.returncode:
-        raise RuntimeError(
-            f"docker run exited with {started.returncode}: {started.stderr.strip()}"
-        )
-    try:
-        mapping = subprocess.run(
-            ["docker", "port", name, f"{CONTAINER_PORT}/tcp"],
-            check=True,
-            capture_output=True,
-            text=True,
-        ).stdout.split()[0]
-        yield f"http://{mapping}"
-    finally:
-        subprocess.run(
-            ["docker", "rm", "--force", name], capture_output=True, check=False
-        )