mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-09 12:17:13 +00:00
kevinveenbirkenbachandClaude Opus 5
d194b77f76
The gateway routes every request on its own, so a catalog holds the work of several engines, while the mark apply() wrote was the constant translated-by: libretranslate and attributed all of it to one backend. The catalogs on disk show the drift: 1021537 entries carry that constant and 13 carry ollama, written once the gateway began answering from more than one place. The constant becomes machine_translation(engine), which names what the gateway reported and writes unnamed where it reported nothing, plus is_machine_translation(), which recognises any of them by prefix so every mark already on disk keeps matching. Outcome carries an engines tuple aligned with its values, merge pads it, and _reask returns the name alongside the text: a successful re-ask records the engine that answered, not the one whose rejection it replaced, because the catalog does not carry that rejected text. apply() now strips an earlier mark before stamping the current one. clear() removes only refusal prefixes, so an entry re-translated after i18n-prune would otherwise accumulate one mark per engine and the record would be ambiguous again, which is the defect this change exists to remove. This is the half that makes the attribution true. Recording latency in the decider history is the other half; together they let the router weigh quality against speed per engine, which a catalog claiming one engine everywhere cannot support. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
310 lines
12 KiB
Python
310 lines
12 KiB
Python
"""HTTP client for the LibreTranslate API.
|
|
|
|
SERVER_CODES maps the catalog's bare ISO 639-1 codes onto the script-qualified
|
|
names LibreTranslate serves some languages under. A code missing from it never
|
|
turns up in /languages, and the readiness wait then spins until its timeout.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from dataclasses import dataclass
|
|
from typing import TYPE_CHECKING
|
|
|
|
from utils.i18n.languages import SOURCE_LANGUAGE
|
|
from utils.i18n.limits import BATCH_SIZE
|
|
from utils.i18n.placeholders import Rejected, mask, mask_tokens, unmask
|
|
from utils.i18n.untranslatable import untranslatable
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Callable, Iterable
|
|
|
|
SERVER_CODES = {"zh": "zh-Hans"}
|
|
POLL_SECONDS = 5
|
|
PROGRESS_SECONDS = 30
|
|
OUTSTANDING_SHOWN = 12
|
|
|
|
|
|
def _outstanding(missing: set[str], catalog_of: dict[str, str]) -> str:
|
|
"""Return the catalog codes still missing, shortened to stay one line.
|
|
|
|
Args:
|
|
missing: the server codes the server does not offer yet.
|
|
catalog_of: server code to the catalog code the operator knows it by.
|
|
"""
|
|
codes = sorted(catalog_of[code] for code in missing)
|
|
shown = " ".join(codes[:OUTSTANDING_SHOWN])
|
|
rest = len(codes) - OUTSTANDING_SHOWN
|
|
return f"{shown} +{rest}" if rest > 0 else shown or "none"
|
|
|
|
|
|
REQUEST_TIMEOUT_SECONDS = 600
|
|
RETRY_ATTEMPTS = 4
|
|
RETRY_BACKOFF_SECONDS = 2
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Outcome:
|
|
"""What one translation call produced, and why anything is missing.
|
|
|
|
Deliberately not a tuple: a caller that forgets ``.values`` would
|
|
otherwise zip against the four fields instead of the translations.
|
|
|
|
Args:
|
|
values: one entry per source: the translation, a ``Rejected`` carrying
|
|
what came back and why it was turned down, or None when the server
|
|
never answered.
|
|
refused: requests the server turned down, retries included.
|
|
damaged: translations that altered a protected span.
|
|
refusal: the last refusal's message, empty when none happened.
|
|
engines: the engine that produced each value, aligned with ``values``
|
|
and empty where none answered. The gateway routes per request, so
|
|
a batch is rarely the work of one engine, and the catalog records
|
|
which one earned each entry.
|
|
"""
|
|
|
|
values: list[str | Rejected | None]
|
|
refused: int
|
|
damaged: int
|
|
refusal: str
|
|
engines: tuple[str, ...] = ()
|
|
|
|
|
|
def merge(outcomes: Iterable[Outcome]) -> Outcome:
|
|
"""Fold per-text outcomes back into one for the whole batch."""
|
|
values: list[str | Rejected | None] = []
|
|
engines: list[str] = []
|
|
refused = damaged = 0
|
|
refusal = ""
|
|
for outcome in outcomes:
|
|
values += outcome.values
|
|
engines += outcome.engines or [""] * len(outcome.values)
|
|
refused += outcome.refused
|
|
damaged += outcome.damaged
|
|
refusal = outcome.refusal or refusal
|
|
return Outcome(values, refused, damaged, refusal, tuple(engines))
|
|
|
|
|
|
class LibreTranslate:
|
|
"""Client for the LibreTranslate HTTP API.
|
|
|
|
Args:
|
|
url: base URL of the server.
|
|
workers: parallel translation requests.
|
|
"""
|
|
|
|
def __init__(self, url: str, workers: int, batch_size: int = BATCH_SIZE):
|
|
self.url = url.rstrip("/")
|
|
self.workers = workers
|
|
self.batch_size = batch_size
|
|
|
|
def _call(self, path: str, payload: dict | None = None):
|
|
data = None if payload is None else json.dumps(payload).encode("utf-8")
|
|
request = urllib.request.Request( # noqa: S310 - the URL is the LibreTranslate container this process started
|
|
f"{self.url}{path}",
|
|
data=data,
|
|
headers={"Content-Type": "application/json"},
|
|
)
|
|
with urllib.request.urlopen( # noqa: S310 - the URL is the LibreTranslate container this process started
|
|
request, timeout=REQUEST_TIMEOUT_SECONDS
|
|
) as response:
|
|
return json.load(response)
|
|
|
|
@staticmethod
|
|
def server_code(code: str) -> str:
|
|
"""Return the target code the server knows this catalog code by."""
|
|
return SERVER_CODES.get(code, code)
|
|
|
|
def targets(self) -> set[str]:
|
|
"""Return the languages the server translates English into."""
|
|
for language in self._call("/languages"):
|
|
if language["code"] == SOURCE_LANGUAGE:
|
|
return set(language["targets"])
|
|
return set()
|
|
|
|
def wait(
|
|
self,
|
|
codes: list[str],
|
|
timeout: float,
|
|
status: Callable[[], str] | None = None,
|
|
) -> None:
|
|
"""Block until the server translates into every language of ``codes``.
|
|
|
|
Args:
|
|
codes: required target languages.
|
|
timeout: seconds to wait before giving up.
|
|
status: returns one line about the server's own progress, reported
|
|
every ``PROGRESS_SECONDS`` while it installs its models. The
|
|
server offers no endpoint before it is ready, so without this
|
|
the wait is indistinguishable from a hang for its whole hour.
|
|
"""
|
|
deadline = time.monotonic() + timeout
|
|
started = time.monotonic()
|
|
catalog_of = {self.server_code(code): code for code in codes}
|
|
wanted = set(catalog_of)
|
|
missing = set(wanted)
|
|
answered = False
|
|
spoken = 0.0
|
|
while True:
|
|
try:
|
|
missing = wanted - self.targets()
|
|
answered = True
|
|
if not missing:
|
|
return
|
|
except (OSError, ValueError):
|
|
answered = False
|
|
now = time.monotonic()
|
|
if now - spoken >= PROGRESS_SECONDS:
|
|
spoken = now
|
|
extra = f", {status()}" if status else ""
|
|
print(
|
|
f"LibreTranslate at {self.url}: "
|
|
f"{len(wanted) - len(missing)}/{len(wanted)} ready"
|
|
f"{'' if answered else ' (server silent)'}, "
|
|
f"pending {_outstanding(missing, catalog_of)}, "
|
|
f"waited {int(now - started)}s of {int(timeout)}s{extra}",
|
|
flush=True,
|
|
)
|
|
if now > deadline:
|
|
raise TimeoutError(
|
|
f"LibreTranslate at {self.url} does not offer {sorted(missing)}"
|
|
)
|
|
time.sleep(POLL_SECONDS)
|
|
|
|
def _post(self, masked: list, target: str, exclude: tuple = ()) -> list | None:
|
|
"""Return the server's translations and who wrote them, or None.
|
|
|
|
The server refuses a share of the requests while every lane hammers it
|
|
at once, and those refusals used to discard their entries for good.
|
|
|
|
Args:
|
|
masked: the messages, already masked.
|
|
target: ISO 639-1 code.
|
|
exclude: engines a previous pass over these messages produced an
|
|
unusable answer with.
|
|
"""
|
|
refusals = 0
|
|
refusal = ""
|
|
for attempt in range(RETRY_ATTEMPTS):
|
|
engines: list = []
|
|
try:
|
|
body = self._call(
|
|
"/translate",
|
|
{
|
|
"q": [item.text for item in masked],
|
|
"source": SOURCE_LANGUAGE,
|
|
"target": self.server_code(target),
|
|
"format": "html",
|
|
"protected": [mask_tokens(item) for item in masked],
|
|
"exclude": list(exclude),
|
|
},
|
|
)
|
|
result = body["translatedText"]
|
|
answered = body.get("engine")
|
|
engines = answered if isinstance(answered, list) else []
|
|
except (urllib.error.HTTPError, ValueError, KeyError, TypeError) as exc:
|
|
refusals += 1
|
|
refusal = f"{type(exc).__name__}: {exc}"
|
|
result = None
|
|
if isinstance(result, list) and len(result) == len(masked):
|
|
return result, engines, refusals, refusal
|
|
time.sleep(RETRY_BACKOFF_SECONDS * 2**attempt)
|
|
return None, [], refusals, refusal
|
|
|
|
def _batch(self, texts: list[str], target: str, exclude: tuple = ()) -> Outcome:
|
|
"""Translate one batch, re-asking what came back unusable.
|
|
|
|
The protected spans are masked before the request, so the gateway
|
|
judges a string this client never shows it and cannot tell that the
|
|
restored markup moved. Naming the engine that answered and asking
|
|
again without it is how this side's verdict reaches the candidate
|
|
set; without it a saturated signature returns the same damage for
|
|
every retry. One extra pass per engine, bounded by the candidate set.
|
|
"""
|
|
masked = [mask(text) for text in texts]
|
|
result, engines, refused, refusal = self._post(masked, target, exclude)
|
|
if result is None:
|
|
if len(texts) == 1:
|
|
return Outcome([None], refused, 0, refusal, ("",))
|
|
return merge(self._batch([text], target, exclude) for text in texts)
|
|
values = [
|
|
unmask(translated, item, text, target)
|
|
for translated, item, text in zip(result, masked, texts, strict=True)
|
|
]
|
|
retried, names = self._reask(texts, target, values, engines, exclude)
|
|
damaged = sum(1 for text in retried if isinstance(text, Rejected))
|
|
return Outcome(retried, refused, damaged, refusal, tuple(names))
|
|
|
|
def _reask(
|
|
self, texts: list[str], target: str, values: list, engines: list, exclude: tuple
|
|
) -> tuple[list, list[str]]:
|
|
"""Replace every rejected value with one from another engine.
|
|
|
|
Returns:
|
|
``values`` with each rejection re-translated where another engine
|
|
was left to ask, and the original rejection where none was, paired
|
|
with the engine each surviving value came from. A re-ask moves the
|
|
name along with the text: recording the engine that was replaced
|
|
would describe a translation the catalog does not carry.
|
|
"""
|
|
if not engines or len(engines) != len(values):
|
|
return values, [""] * len(values)
|
|
answered = list(values)
|
|
names = list(engines)
|
|
for index, value in enumerate(values):
|
|
if not isinstance(value, Rejected):
|
|
continue
|
|
engine = engines[index]
|
|
if not engine or engine in exclude:
|
|
continue
|
|
retry = self._batch([texts[index]], target, (*exclude, engine))
|
|
if not isinstance(retry.values[0], Rejected) and retry.values[0]:
|
|
answered[index] = retry.values[0]
|
|
names[index] = (retry.engines or ("",))[0]
|
|
return answered, names
|
|
|
|
def translate(self, texts: list[str], target: str) -> Outcome:
|
|
"""Translate ``texts`` from English into ``target``.
|
|
|
|
Args:
|
|
texts: source messages.
|
|
target: ISO 639-1 code.
|
|
|
|
Returns:
|
|
One translation per source, ``None`` where it was discarded, plus
|
|
the counts that say which of the two reasons applied. The counts
|
|
belong to this call: lanes share the client, so a field on the
|
|
client would mix another catalog's numbers into this one's.
|
|
|
|
Raises:
|
|
OSError: the server is unreachable; every further request would
|
|
fail too, so the run stops instead of discarding the rest.
|
|
"""
|
|
results: list[str | Rejected | None] = list(texts)
|
|
names = [""] * len(texts)
|
|
wordy = [index for index, text in enumerate(texts) if not untranslatable(text)]
|
|
size = self.batch_size
|
|
batches = [wordy[i : i + size] for i in range(0, len(wordy), size)]
|
|
refused = damaged = 0
|
|
refusal = ""
|
|
with ThreadPoolExecutor(self.workers) as pool:
|
|
translated = pool.map(
|
|
lambda batch: self._batch([texts[index] for index in batch], target),
|
|
batches,
|
|
)
|
|
for batch, outcome in zip(batches, translated, strict=True):
|
|
engines = outcome.engines or ("",) * len(outcome.values)
|
|
for index, text, engine in zip(
|
|
batch, outcome.values, engines, strict=True
|
|
):
|
|
results[index] = text
|
|
names[index] = engine
|
|
refused += outcome.refused
|
|
damaged += outcome.damaged
|
|
refusal = outcome.refusal or refusal
|
|
return Outcome(results, refused, damaged, refusal, tuple(names))
|