diff --git a/cli/build/i18n/__main__.py b/cli/build/i18n/__main__.py index f2ac7f871c..10bb727d2a 100644 --- a/cli/build/i18n/__main__.py +++ b/cli/build/i18n/__main__.py @@ -36,8 +36,7 @@ from utils.i18n.languages import ( from utils.i18n.libretranslate import ( READY_TIMEOUT_SECONDS, LibreTranslate, - container, - pinned_image, + server, ) from utils.i18n.translate import apply, damaged, discard, pending @@ -124,7 +123,7 @@ def translate(domain: str, requested: list[str]) -> int: print(f"{domain}: nothing to translate") return 0 threads = os.cpu_count() or 1 - with container(pinned_image(PROJECT_ROOT), codes, threads) as url: + with server(PROJECT_ROOT, codes, threads) as url: client = LibreTranslate(url, threads) client.wait(codes, READY_TIMEOUT_SECONDS) for code in codes: diff --git a/scripts/tests/deploy/local/deploy/main.sh b/scripts/tests/deploy/local/deploy/main.sh index c3e24ca609..225182099a 100755 --- a/scripts/tests/deploy/local/deploy/main.sh +++ b/scripts/tests/deploy/local/deploy/main.sh @@ -25,6 +25,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../../.." && pwd)" cd "${REPO_ROOT}" +mkdir -p build +echo $$ >build/deploy.pid + MODE="${mode:-initialize}" # nocheck: deploy router knob; routes to apps// subscripts PURGE="${purge:-false}" # nocheck: deploy router knob; gates entity pre-purge diff --git a/tests/unit/python/cli/build/i18n/test_main.py b/tests/unit/python/cli/build/i18n/test_main.py index 9d652437ee..b7d4b008d4 100644 --- a/tests/unit/python/cli/build/i18n/test_main.py +++ b/tests/unit/python/cli/build/i18n/test_main.py @@ -44,8 +44,7 @@ class TestTranslate(unittest.TestCase): ) with ( mock.patch.object(cli, "PROJECT_ROOT", self.root), - mock.patch.object(cli, "pinned_image", return_value="lt:1"), - mock.patch.object(cli, "container", started), + mock.patch.object(cli, "server", started), mock.patch.object(cli, "LibreTranslate", client), ): return cli.translate("core", []), started diff --git a/tests/unit/python/utils/i18n/test_libretranslate.py b/tests/unit/python/utils/i18n/test_libretranslate.py index 1b2a835b87..6698330480 100644 --- a/tests/unit/python/utils/i18n/test_libretranslate.py +++ b/tests/unit/python/utils/i18n/test_libretranslate.py @@ -1,9 +1,50 @@ +import os import subprocess import unittest import unittest.mock as mock import urllib.error +from pathlib import Path +from tempfile import TemporaryDirectory -from utils.i18n.libretranslate import LibreTranslate, accelerated +from utils.i18n.libretranslate import ( + LibreTranslate, + accelerated, + deploying, + server, +) + + +class TestParallelDeployIsRefused(unittest.TestCase): + def test_a_running_deploy_stops_the_translation(self) -> None: + with ( + mock.patch("utils.i18n.libretranslate.deploying", return_value=True), + mock.patch("utils.i18n.libretranslate.deploy") as started, + self.assertRaises(RuntimeError), + server(Path("/repo"), ["de"], 1), + ): + pass + started.assert_not_called() + + def test_a_live_router_pid_reports_a_deploy(self) -> None: + self.assertTrue(deploying(self._root_holding(os.getpid()))) + + def test_a_stale_pid_file_reports_no_deploy(self) -> None: + root = self._root_holding(4242) + with mock.patch("os.kill", side_effect=ProcessLookupError): + self.assertFalse(deploying(root)) + + def test_a_missing_pid_file_reports_no_deploy(self) -> None: + self.assertFalse(deploying(Path(self._tmp.name))) + + def _root_holding(self, pid: int) -> Path: + root = Path(self._tmp.name) + (root / "build").mkdir(exist_ok=True) + (root / "build" / "deploy.pid").write_text(str(pid)) + return root + + def setUp(self) -> None: + self._tmp = TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) class TestTranslateFailures(unittest.TestCase): diff --git a/utils/i18n/libretranslate.py b/utils/i18n/libretranslate.py index 8483abd962..4a02b85a23 100644 --- a/utils/i18n/libretranslate.py +++ b/utils/i18n/libretranslate.py @@ -5,6 +5,7 @@ from __future__ import annotations import json import os import subprocess +import sys import time import urllib.error import urllib.request @@ -21,7 +22,8 @@ from utils.roles.mapping import ROLE_FILE_META_SERVICES if TYPE_CHECKING: from collections.abc import Iterator -SERVICES_FILE = Path("roles") / "web-svc-libretranslate" / ROLE_FILE_META_SERVICES +ROLE = "web-svc-libretranslate" +SERVICES_FILE = Path("roles") / ROLE / ROLE_FILE_META_SERVICES CONTAINER_PORT = 5000 MODELS_VOLUME = "infinito-i18n-libretranslate" MODELS_TARGET = "/home/libretranslate/.local" @@ -30,6 +32,13 @@ BATCH_SIZE = 20 POLL_SECONDS = 5 READY_TIMEOUT_SECONDS = 3600 REQUEST_TIMEOUT_SECONDS = 600 +DEPLOYED_TIMEOUT_SECONDS = 5 +DEPLOY_TIMEOUT_SECONDS = 3600 +RUNNER_TIMEOUT_SECONDS = 300 +INVENTORY_DIR = Path.home() / "inventories" / "infinito-i18n" +DEPLOY_PID_FILE = Path("build") / "deploy.pid" +GPU_STAMP = Path("build") / "i18n-gpu.stamp" +INVENTORY_VARS = Path("i18n") / "inventory.yml" def pinned_image(root: Path) -> str: @@ -53,6 +62,234 @@ def accelerated() -> bool: return "nvidia" in runtimes.stdout +def deployed(root: Path) -> str: + """Return the URL of the deployed LibreTranslate, empty when it does not answer. + + Args: + root: repository root. + """ + service = load_yaml(root / SERVICES_FILE)["libretranslate"] + host = environment(root)["INFINITO_BIND_IP"] + url = f"http://{host}:{service['ports']['local']['http']}" + try: + with urllib.request.urlopen( # noqa: S310 - the URL is this repository's own service definition + f"{url}/languages", timeout=DEPLOYED_TIMEOUT_SECONDS + ): + return url + except OSError: + return "" + + +def unaccelerated(root: Path) -> bool: + """Return whether the running LibreTranslate predates this host's GPU. + + A container built before the GPU wiring landed stays healthy and answers + every request, so the probe alone cannot tell the two apart; the stamp the + last accelerated deploy left behind can. + + Args: + root: repository root. + """ + return accelerated() and not (root / GPU_STAMP).is_file() + + +def deploying(root: Path) -> bool: + """Return whether the deploy router still holds the machine. + + Args: + root: repository root. + """ + try: + raw = ( + root / DEPLOY_PID_FILE + ).read_text() # nocheck: cache-read - the router rewrites this file per run; the cached reader is an lru_cache without mtime invalidation + os.kill(int(raw), 0) + except (OSError, ValueError): + return False + return True + + +def environment(root: Path) -> dict[str, str]: + """Return the process environment the deploy CLIs expect. + + Args: + root: repository root. + + Returns: + ``os.environ`` widened by every assignment ``.env`` makes; an already + exported value wins, matching what ``scripts/meta/env/load.sh`` does. + """ + loaded = dict(os.environ) + dotenv = root / ".env" + if dotenv.is_file(): + for line in dotenv.read_text( + encoding="utf-8" + ).splitlines(): # nocheck: cache-read - `make dotenv` regenerates .env mid-session and utils.cache.files.read_text is an lru_cache without mtime invalidation + if line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + loaded.setdefault(key, value) + return loaded + + +def local_image(root: Path, env: dict[str, str]) -> str: + """Return the image tag the infinito service builds, so the runner reuses it. + + Args: + root: repository root. + env: environment the resolver script reads its distro from. + """ + resolver = root / "scripts" / "meta" / "resolve" / "image" / "local.sh" + return subprocess.run( + ["bash", str(resolver)], + cwd=root, + env=env, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + +def await_runner(name: str, env: dict[str, str]) -> None: + """Block until the i18n runner is up, so no exec falls back to another container. + + Args: + name: container name of the runner. + env: environment the docker CLI runs with. + + Raises: + RuntimeError: the runner never reached the running state. + """ + deadline = time.monotonic() + RUNNER_TIMEOUT_SECONDS + while time.monotonic() < deadline: + state = subprocess.run( + ["docker", "inspect", name, "--format", "{{.State.Running}}"], + capture_output=True, + text=True, + check=False, + env=env, + ) + if state.stdout.strip() == "true": + return + time.sleep(POLL_SECONDS) + raise RuntimeError(f"The i18n runner {name} did not reach the running state.") + + +def deploy(root: Path) -> str: + """Deploy web-svc-libretranslate alone into its own inventory and return its URL. + + Args: + root: repository root. + + Returns: + The base URL, empty when the service stays unreachable. + """ + from cli.administration.deploy.development.env import compose_file_args + + development = [sys.executable, "-m", "cli.administration.deploy.development"] + inside = [ + "docker", + "compose", + *compose_file_args(), + "exec", + "-T", + "-w", + environment(root)["INFINITO_SRC_DIR"], + "i18n", + "python", + "-m", + ] + bootstrap = "scripts/tests/deploy/local/utils/entry-bootstrap.sh" + steps = [ + [ + "docker", + "compose", + *compose_file_args(), + "--profile", + "i18n", + "up", + "-d", + "--force-recreate", + "i18n", + ], + [*inside[:-2], "bash", f"{environment(root)['INFINITO_SRC_DIR']}/{bootstrap}"], + [ + *inside, + "cli.administration.inventory.provision", + str(INVENTORY_DIR), + "--include", + ROLE, + "--vars-file", + f"{environment(root)['INFINITO_SRC_DIR']}/{INVENTORY_VARS}", + ], + [ + *inside, + "cli.administration.deploy.dedicated", + str(INVENTORY_DIR / "devices.yml"), + "--id", + ROLE, + "--password-file", + str(INVENTORY_DIR / ".password"), + "--skip-backup", + "--skip-cleanup", + ], + ] + loaded = environment(root) + outer = { + **loaded, + "PYTHONUNBUFFERED": "1", + "INFINITO_IMAGE": loaded.get("INFINITO_IMAGE") or local_image(root, loaded), + } + name = f"{loaded['INFINITO_CONTAINER']}_i18n" + runner = {**outer, "INFINITO_CONTAINER": name} + for index, step in enumerate(steps): + subprocess.run( + step, + cwd=root, + check=True, + timeout=DEPLOY_TIMEOUT_SECONDS, + env=outer if index == 0 else runner, + ) + if index == 0: + await_runner(name, outer) + running = deployed(root) + if running and accelerated(): + (root / GPU_STAMP).parent.mkdir(exist_ok=True) + (root / GPU_STAMP).touch() + return running + + +@contextmanager +def server(root: Path, codes: list[str], threads: int) -> Iterator[str]: + """Yield a LibreTranslate URL, preferring the deployed service over a new container. + + Args: + root: repository root. + codes: target languages to load next to the source language. + threads: translation threads of a container this starts itself. + + Yields: + The base URL to translate against. + + Raises: + RuntimeError: a deploy is already running, and translating would race it + for the same container stack. + """ + if deploying(root): + raise RuntimeError( + "A deploy is running; it shares the container stack this translation " + "would touch. Wait for it to finish and re-run. The running deploy is " + "left untouched." + ) + running = "" if unaccelerated(root) else deployed(root) + running = running or deploy(root) + if running: + yield running + return + with container(pinned_image(root), codes, threads) as spawned: + yield spawned + + @contextmanager def container(image: str, codes: list[str], threads: int) -> Iterator[str]: """Run LibreTranslate for ``codes`` and remove the container afterwards.