diff --git a/roles/web-svc-translate/files/python/translate_gateway/app.py b/roles/web-svc-translate/files/python/translate_gateway/app.py
index 01d4a4ae46..5de69d7911 100644
--- a/roles/web-svc-translate/files/python/translate_gateway/app.py
+++ b/roles/web-svc-translate/files/python/translate_gateway/app.py
@@ -4,13 +4,17 @@
from __future__ import annotations
+import asyncio
+
from fastapi import FastAPI, Request
-from fastapi.responses import JSONResponse, RedirectResponse
+from fastapi.responses import JSONResponse
from translate_gateway.errors import GatewayError, NoBackendError
+NAME = "Infinito.Nexus Translation Gateway"
UNPROCESSABLE = 422
UNAVAILABLE = 503
+BATCH_CONCURRENCY = 8
def create_app(gateway, engines, *, detector=None):
@@ -25,16 +29,25 @@ def create_app(gateway, engines, *, detector=None):
Returns:
The FastAPI application.
"""
- app = FastAPI(title="Infinito.Nexus Translation Gateway")
+ app = FastAPI(title=NAME)
def _detector():
if detector is not None:
return detector
return next((e for e in engines.values() if hasattr(e, "detect")), None)
- @app.get("/", include_in_schema=False)
- async def _root() -> RedirectResponse:
- return RedirectResponse(url="/docs")
+ @app.api_route("/", methods=["GET", "HEAD"], include_in_schema=False)
+ async def _root() -> JSONResponse:
+ """Name the service and where its schema lives.
+
+ It answers rather than redirecting to ``/docs``, and it serves HEAD
+ beside GET, because both health checks start here: the webserver one
+ HEAD-requests every vhost and accepts only 200, 301 or 302, and the
+ CSP one follows redirects and renders what it lands on. Swagger UI
+ loads its assets from an external CDN this deployment forbids, so a
+ redirect into it failed the deploy on a gateway that was healthy.
+ """
+ return JSONResponse(content={"service": NAME, "schema": "/docs"})
@app.exception_handler(GatewayError)
async def _refuse(_request: Request, exc: GatewayError) -> JSONResponse:
@@ -56,9 +69,38 @@ def create_app(gateway, engines, *, detector=None):
status_code=UNPROCESSABLE,
content={"error": "'q' and 'target' are required"},
)
- engine, answer = await gateway.translate(body.get("source"), target, text)
+ source = body.get("source")
+ fmt = body.get("format") or "text"
+ protected = tuple(body.get("protected") or ())
+ exclude = tuple(body.get("exclude") or ())
+ if not isinstance(text, list):
+ engine, answer = await gateway.translate(
+ source, target, text, fmt=fmt, protected=protected, exclude=exclude
+ )
+ return JSONResponse(content={"translatedText": answer, "engine": engine})
+
+ if len(protected) == len(text) and all(
+ isinstance(entry, list) for entry in protected
+ ):
+ per_item = [tuple(entry) for entry in protected]
+ else:
+ per_item = [protected] * len(text)
+ limit = asyncio.Semaphore(BATCH_CONCURRENCY)
+
+ async def _one(item, terms):
+ async with limit:
+ return await gateway.translate(
+ source, target, item, fmt=fmt, protected=terms, exclude=exclude
+ )
+
+ answered = await asyncio.gather(
+ *(_one(item, terms) for item, terms in zip(text, per_item, strict=True))
+ )
return JSONResponse(
- content={"translatedText": answer, "engine": engine},
+ content={
+ "translatedText": [answer for _, answer in answered],
+ "engine": [engine for engine, _ in answered],
+ }
)
@app.post("/detect")
@@ -79,15 +121,30 @@ def create_app(gateway, engines, *, detector=None):
content=[{"language": engine.detect(text), "confidence": 1}]
)
+ async def _pairs_of(engine):
+ """What *engine* offers: its declaration, else what it reports itself.
+
+ A backend that declares nothing still serves every pair it is asked
+ for, so leaving it out of the catalogue made the gateway advertise
+ an empty language list while translating fine.
+ """
+ declared = getattr(engine, "pairs", None)
+ if declared:
+ return set(declared)
+ reader = getattr(engine, "catalogue", None)
+ if reader is None:
+ return set()
+ try:
+ return await asyncio.to_thread(reader)
+ except Exception:
+ return set()
+
@app.get("/languages")
async def languages() -> JSONResponse:
- pairs = sorted(
- {
- (source, target)
- for engine in engines.values()
- for source, target in getattr(engine, "pairs", None) or ()
- }
+ reported = await asyncio.gather(
+ *(_pairs_of(engine) for engine in engines.values())
)
+ pairs = sorted(set().union(*reported) if reported else set())
catalogue: dict[str, list[str]] = {}
for source, target in pairs:
catalogue.setdefault(source, []).append(target)
diff --git a/roles/web-svc-translate/files/python/translate_gateway/cache.py b/roles/web-svc-translate/files/python/translate_gateway/cache.py
index a5a7a0b86d..e08f3b2638 100644
--- a/roles/web-svc-translate/files/python/translate_gateway/cache.py
+++ b/roles/web-svc-translate/files/python/translate_gateway/cache.py
@@ -48,22 +48,22 @@ class TranslationCache:
await self._client.execute_raw(self.ddl)
self._ready = True
- async def get(self, engine, source, target, text):
+ async def get(self, engine, source, target, text, fmt="text"):
"""The stored translation, or None when this request is new."""
await self._ensure()
rows = await self._client.query_raw(
f"SELECT translation FROM {self._table} WHERE key = $1", # noqa: S608 table validated against IDENTIFIER in __init__
- cache_key(engine, source, target, text),
+ cache_key(engine, source, target, text, fmt),
)
return (rows or [{}])[0].get("translation") if rows else None
- async def set(self, engine, source, target, text, translation):
+ async def set(self, engine, source, target, text, translation, fmt="text"):
"""Store *translation*, replacing any earlier answer for this key."""
await self._ensure()
await self._client.execute_raw(
f"INSERT INTO {self._table} (key, translation) VALUES ($1, $2) " # noqa: S608 table validated against IDENTIFIER in __init__
f"ON CONFLICT (key) DO UPDATE SET translation = EXCLUDED.translation, "
f"answered_at = now()",
- cache_key(engine, source, target, text),
+ cache_key(engine, source, target, text, fmt),
translation,
)
diff --git a/roles/web-svc-translate/files/python/translate_gateway/engines.py b/roles/web-svc-translate/files/python/translate_gateway/engines.py
index 53980d1662..510832f15c 100644
--- a/roles/web-svc-translate/files/python/translate_gateway/engines.py
+++ b/roles/web-svc-translate/files/python/translate_gateway/engines.py
@@ -56,6 +56,27 @@ class LibreTranslateEngine:
return True
return (source or "auto", target) in self._languages
+ def catalogue(self):
+ """The pairs the backend itself reports, as ``{(source, target)}``.
+
+ Asked only when nothing was declared for this backend. The engine
+ knows its own models, so reading them back beats a second copy in
+ the role's configuration that would go stale when a model lands.
+
+ Raises:
+ OSError: when the backend cannot be reached.
+ """
+ request = urllib.request.Request( # noqa: S310 configured internal origin
+ f"{self._base_url}/languages", method="GET"
+ )
+ with urllib.request.urlopen(request, timeout=self._timeout) as response: # noqa: S310 configured internal origin
+ body = json.loads(response.read())
+ return {
+ (entry.get("code"), target)
+ for entry in body or []
+ for target in entry.get("targets") or ()
+ }
+
def _post(self, path, payload):
request = urllib.request.Request( # noqa: S310 configured internal origin
f"{self._base_url}{path}",
@@ -66,7 +87,7 @@ class LibreTranslateEngine:
with urllib.request.urlopen(request, timeout=self._timeout) as response: # noqa: S310 configured internal origin
return json.loads(response.read())
- def translate(self, source, target, text, protected=()):
+ def translate(self, source, target, text, protected=(), fmt="text"):
"""The translation of *text*, as this backend returns it.
Args:
@@ -75,6 +96,10 @@ class LibreTranslateEngine:
text: the string to translate.
protected: glossary terms; the LibreTranslate API takes none, so
they are checked on the answer instead of sent.
+ fmt: the LibreTranslate payload format, ``text`` or ``html``. A
+ caller that masks its protected spans as tags MUST ask for
+ ``html``, because in text mode the engine translates the tag
+ itself and the mask never comes back.
Raises:
OSError: when the backend cannot be reached.
@@ -84,7 +109,7 @@ class LibreTranslateEngine:
"q": text,
"source": source or "auto",
"target": target,
- "format": "text",
+ "format": fmt,
}
if self._api_key:
payload["api_key"] = self._api_key
@@ -151,7 +176,7 @@ class ChatModelEngine:
return True
return (source or "auto", target) in self._languages
- def translate(self, source, target, text, protected=()):
+ def translate(self, source, target, text, protected=(), fmt="text"):
"""The translation the model generated.
Args:
@@ -159,6 +184,8 @@ class ChatModelEngine:
target: target language code.
text: the string to translate.
protected: glossary terms the prompt tells the model to keep.
+ fmt: accepted for signature parity with the LibreTranslate
+ backend; a prompt carries no payload format.
Raises:
OSError: when the model gateway cannot be reached.
diff --git a/roles/web-svc-translate/files/python/translate_gateway/gateway.py b/roles/web-svc-translate/files/python/translate_gateway/gateway.py
index 0727f07075..1c660f5b5d 100644
--- a/roles/web-svc-translate/files/python/translate_gateway/gateway.py
+++ b/roles/web-svc-translate/files/python/translate_gateway/gateway.py
@@ -43,9 +43,25 @@ class Gateway:
except Exception:
return ()
- async def translate(self, source, target, text):
+ async def translate(
+ self, source, target, text, *, fmt="text", protected=(), exclude=()
+ ):
"""Translate *text*, preferring a reviewed human string.
+ Args:
+ source: source language code, or None for auto-detection.
+ target: target language code.
+ text: the string to translate.
+ fmt: the payload format the engines are asked with, ``text`` or
+ ``html``.
+ protected: spans the caller needs back verbatim, on top of the
+ glossary. A caller that masks its placeholders knows which
+ tokens must survive and the glossary does not.
+ exclude: engines the caller already rejected for this string.
+ A reviewed string still wins over all of them, because the
+ precedence rule is about who wrote the translation, not about
+ which engine last disappointed the caller.
+
Returns:
``(origin, translation)`` where origin is ``weblate`` or the
engine name.
@@ -63,6 +79,7 @@ class Gateway:
if reviewed:
return REVIEWED, reviewed
+ terms = (*await self._protected(target, text), *protected)
return await self._router.translate(
- source, target, text, protected=await self._protected(target, text)
+ source, target, text, protected=terms, fmt=fmt, exclude=exclude
)
diff --git a/roles/web-svc-translate/files/python/translate_gateway/router.py b/roles/web-svc-translate/files/python/translate_gateway/router.py
index 9851e013e1..ad8338d90f 100644
--- a/roles/web-svc-translate/files/python/translate_gateway/router.py
+++ b/roles/web-svc-translate/files/python/translate_gateway/router.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
import time
+from collections import Counter
from .errors import EngineRefusedError, MangledTermError, NoBackendError
from .signature import signature
@@ -11,6 +12,7 @@ from .signature import signature
SYSTEM_ONE = "system_one"
PREFERENCE = "preference"
CHOICE_INSTRUCTIONS = "Which engine should answer this translation request?"
+MARKUP_CHARACTERS = "[]{}`*()"
def saturated(tally, minimum, share):
@@ -92,12 +94,23 @@ class Router:
self._pairs = dict(pairs or {})
self._latency = {}
- def candidates(self, source, target):
- """The engines that serve this language pair, in preference order."""
+ def candidates(self, source, target, exclude=()):
+ """The engines that serve this language pair, in preference order.
+
+ Args:
+ source: source language code, or None.
+ target: target language code.
+ exclude: engines the caller has already found wanting for this
+ string. A caller that masks its placeholders judges the
+ restored answer, which the gateway never sees, so naming the
+ engine that failed that judgement is the only way its own
+ verdict can reach the candidate set.
+ """
+ barred = set(exclude)
serving = [
name
for name, engine in self._engines.items()
- if engine.serves(source, target)
+ if name not in barred and engine.serves(source, target)
]
ranked = [name for name in self._order if name in serving]
return ranked + sorted(set(serving) - set(ranked))
@@ -120,11 +133,11 @@ class Router:
except Exception:
return {}
- async def _ask_one(self, engine, source, target, text, protected=()):
+ async def _ask_one(self, engine, source, target, text, protected=(), fmt="text"):
started = time.monotonic()
try:
answer = await asyncio.to_thread(
- self._engines[engine].translate, source, target, text, protected
+ self._engines[engine].translate, source, target, text, protected, fmt
)
except Exception as exc:
await self._failed(engine)
@@ -191,7 +204,9 @@ class Router:
choice = (answer or {}).get("choice")
return choice if choice in offered else None
- async def translate(self, source, target, text, *, protected=()):
+ async def translate(
+ self, source, target, text, *, protected=(), fmt="text", exclude=()
+ ):
"""Translate *text*, comparing engines when the sampler says so.
Args:
@@ -201,6 +216,9 @@ class Router:
protected: glossary terms the answer has to keep; they are handed
to every engine that takes them and checked before the answer
is cached.
+ fmt: the payload format every engine is asked with, ``text`` or
+ ``html``.
+ exclude: engines the caller already rejected for this string.
Returns:
``(engine, translation)``.
@@ -216,7 +234,7 @@ class Router:
request ever again, and nothing would clear a streak because nothing
would be asked.
"""
- serving = self.candidates(source, target)
+ serving = self.candidates(source, target, exclude)
if not serving:
raise NoBackendError(f"no engine serves {source or 'auto'} to {target}")
@@ -241,38 +259,55 @@ class Router:
)
if comparing:
return await self._compare(
- offered, request_class, source, target, text, protected
+ offered, request_class, source, target, text, protected, fmt
)
chosen = settled
if chosen is None and self._strategy == SYSTEM_ONE:
chosen = await self._choose(offered, tally, source, target, text)
return await self._serve(
- chosen or offered[0], offered, source, target, text, protected
+ chosen or offered[0], offered, source, target, text, protected, fmt
)
@staticmethod
- def _mangled(answer, protected):
- """The protected terms *answer* lost, in declaration order."""
- return [term for term in protected if term not in answer]
+ def _mangled(answer, protected, text=""):
+ """What *answer* broke: protected terms it lost, then the markup.
- async def _serve(self, engine, offered, source, target, text, protected=()):
+ The markup skeleton is checked as a multiset of ``MARKUP_CHARACTERS``
+ rather than term presence, because the damage that matters is a count
+ that moved: an engine that renders ``**Raw / Community**`` as
+ ``**/ Community** *`` keeps every character and still emits a stray
+ emphasis marker. Catching it here is what lets the caller's next
+ engine be tried, which is the whole reason the candidate set exists.
+ """
+ lost = [term for term in protected if term not in answer]
+ if text and Counter(
+ character for character in answer if character in MARKUP_CHARACTERS
+ ) != Counter(
+ character for character in text if character in MARKUP_CHARACTERS
+ ):
+ lost.append("the markup skeleton")
+ return lost
+
+ async def _serve(
+ self, engine, offered, source, target, text, protected=(), fmt="text"
+ ):
refused = []
mangled = []
for name in [engine, *[o for o in offered if o != engine]]:
- cached = await self._cached(name, source, target, text)
+ cached = await self._cached(name, source, target, text, fmt)
if cached is not None:
return name, cached
try:
- answer = await self._ask_one(name, source, target, text, protected)
+ answer = await self._ask_one(name, source, target, text, protected, fmt)
except EngineRefusedError as exc:
refused.append(str(exc))
continue
- missing = self._mangled(answer, protected)
+ missing = self._mangled(answer, protected, text)
if missing:
mangled.append(f"{name} dropped {', '.join(missing)}")
continue
- await self._store(name, source, target, text, answer)
+ await self._store(name, source, target, text, answer, fmt)
return name, answer
if mangled and not refused:
raise MangledTermError("; ".join(mangled))
@@ -281,16 +316,16 @@ class Router:
)
async def _compare(
- self, offered, request_class, source, target, text, protected=()
+ self, offered, request_class, source, target, text, protected=(), fmt="text"
):
replies = {}
mangled = []
for name in offered:
try:
- answer = await self._ask_one(name, source, target, text, protected)
+ answer = await self._ask_one(name, source, target, text, protected, fmt)
except EngineRefusedError:
continue
- missing = self._mangled(answer, protected)
+ missing = self._mangled(answer, protected, text)
if missing:
mangled.append(f"{name} dropped {', '.join(missing)}")
continue
@@ -300,15 +335,37 @@ class Router:
if not replies:
raise NoBackendError("every engine refused the request")
if len(replies) == 1:
- winner = next(iter(replies))
+ winner, judged = next(iter(replies)), True
else:
+ verdict = await self._judged(text, replies)
+ judged = verdict is not None
+ winner = verdict or next(
+ (name for name in self._order if name in replies), next(iter(replies))
+ )
+ for name, answer in replies.items():
+ await self._store(name, source, target, text, answer, fmt)
+ if judged:
+ await self._record(request_class, winner, sorted(replies))
+ return winner, replies[winner]
+
+ async def _judged(self, text, replies):
+ """The engine the decider preferred, or None when it did not say.
+
+ A decider that times out must not fail a request whose translations
+ are already in hand: ``_choose`` degrades the same way, and leaving
+ this call unguarded turned a slow decider into a 500 that threw the
+ replies away. None is the answer in that case rather than a stand-in,
+ because the caller records a row only for a verdict that was taken:
+ charging the fallback engine a win the decider never awarded would
+ let ``saturated`` freeze a signature on the configured order.
+ """
+ try:
winner = await asyncio.to_thread(
self._decider.judge_answers, text, replies, self._question
)
- for name, answer in replies.items():
- await self._store(name, source, target, text, answer)
- await self._record(request_class, winner, sorted(replies))
- return winner, replies[winner]
+ except Exception:
+ return None
+ return winner if winner in replies else None
async def _record(self, request_class, winner, offered):
if self._history is None:
@@ -318,18 +375,18 @@ class Router:
except Exception:
return
- async def _cached(self, engine, source, target, text):
+ async def _cached(self, engine, source, target, text, fmt="text"):
if self._cache is None:
return None
try:
- return await self._cache.get(engine, source, target, text)
+ return await self._cache.get(engine, source, target, text, fmt)
except Exception:
return None
- async def _store(self, engine, source, target, text, answer):
+ async def _store(self, engine, source, target, text, answer, fmt="text"):
if self._cache is None:
return
try:
- await self._cache.set(engine, source, target, text, answer)
+ await self._cache.set(engine, source, target, text, answer, fmt)
except Exception:
return
diff --git a/roles/web-svc-translate/files/python/translate_gateway/signature.py b/roles/web-svc-translate/files/python/translate_gateway/signature.py
index 36c64ce4d8..fa919d156a 100644
--- a/roles/web-svc-translate/files/python/translate_gateway/signature.py
+++ b/roles/web-svc-translate/files/python/translate_gateway/signature.py
@@ -43,12 +43,15 @@ def signature(source, target, text):
return f"{source or 'auto'}:{target}:{length_class(text)}:{shape}"
-def cache_key(engine, source, target, text):
+def cache_key(engine, source, target, text, fmt="text"):
"""What identifies a cached translation.
The engine is part of the key: two engines answer the same request
differently, and serving one's answer as the other's would make the
- learning log describe something that never happened.
+ learning log describe something that never happened. The format is part
+ of it for the same reason: an engine asked in text mode translates a tag
+ that the same engine in html mode leaves alone, so the two answers are
+ not interchangeable.
"""
digest = hashlib.sha256((text or "").encode()).hexdigest()
- return f"{engine}:{source or 'auto'}:{target}:{digest}"
+ return f"{engine}:{source or 'auto'}:{target}:{fmt}:{digest}"
diff --git a/tests/unit/python/roles/web-svc-translate/files/translate_gateway/test_gateway.py b/tests/unit/python/roles/web-svc-translate/files/translate_gateway/test_gateway.py
index a93667d8d2..eed672bbfc 100644
--- a/tests/unit/python/roles/web-svc-translate/files/translate_gateway/test_gateway.py
+++ b/tests/unit/python/roles/web-svc-translate/files/translate_gateway/test_gateway.py
@@ -18,10 +18,16 @@ class FakeRouter:
self.raises = raises
self.calls = 0
self.protected = []
+ self.formats = []
+ self.excluded = []
- async def translate(self, source, target, text, *, protected=()):
+ async def translate(
+ self, source, target, text, *, protected=(), fmt="text", exclude=()
+ ):
self.calls += 1
self.protected.append(tuple(protected))
+ self.formats.append(fmt)
+ self.excluded.append(tuple(exclude))
if self.raises:
raise self.raises
return self.answer
@@ -99,6 +105,22 @@ class GlossaryTestCase(unittest.TestCase):
self.assertEqual(engine, "alpha")
self.assertEqual(router.protected, [("Infinito.Nexus",)])
+ def test_the_callers_own_spans_join_the_glossary_terms(self) -> None:
+ """A masked placeholder is protected without being a glossary entry."""
+ router = FakeRouter(("alpha", "Haus"))
+ gateway = Gateway(router, glossary=FakeGlossary(("Infinito.Nexus",)))
+
+ run(gateway.translate("en", "de", "House", protected=('',)))
+
+ self.assertEqual(router.protected, [("Infinito.Nexus", '')])
+
+ def test_the_requested_format_reaches_the_router(self) -> None:
+ router = FakeRouter(("alpha", "Haus"))
+
+ run(Gateway(router).translate("en", "de", "House", fmt="html"))
+
+ self.assertEqual(router.formats, ["html"])
+
def test_a_mangled_answer_reaches_the_caller_as_a_refusal(self) -> None:
gateway = Gateway(
FakeRouter(raises=MangledTermError("alpha dropped Infinito.Nexus")),
diff --git a/tests/unit/python/roles/web-svc-translate/files/translate_gateway/test_router.py b/tests/unit/python/roles/web-svc-translate/files/translate_gateway/test_router.py
index 82c5b0c58a..3f90f5d8fe 100644
--- a/tests/unit/python/roles/web-svc-translate/files/translate_gateway/test_router.py
+++ b/tests/unit/python/roles/web-svc-translate/files/translate_gateway/test_router.py
@@ -27,13 +27,15 @@ class FakeEngine:
self.keeps_terms = keeps_terms
self.calls = 0
self.protected = []
+ self.formats = []
def serves(self, source, target):
return self.pairs is None or (source, target) in self.pairs
- def translate(self, source, target, text, protected=()):
+ def translate(self, source, target, text, protected=(), fmt="text"):
self.calls += 1
self.protected.append(tuple(protected))
+ self.formats.append(fmt)
if self.raises:
raise self.raises
kept = " ".join(protected) if self.keeps_terms else ""
@@ -145,6 +147,58 @@ class VerdictIsBlindTestCase(unittest.TestCase):
_text, replies, _question = decider.seen[0]
self.assertEqual(sorted(replies), ["alpha", "beta"])
+ def test_a_decider_that_raises_still_answers_the_request(self) -> None:
+ """Its timeout used to reach the caller as a 500, replies and all."""
+
+ class Unreachable:
+ def judge_answers(self, *_args):
+ raise TimeoutError("timed out")
+
+ router = Router(
+ {"alpha": FakeEngine("alpha"), "beta": FakeEngine("beta")},
+ decider=Unreachable(),
+ history=FakeHistory(),
+ sampler=AlwaysSampler(),
+ order=("beta", "alpha"),
+ )
+
+ engine, answer = run(router.translate("de", "en", "Haus"))
+
+ self.assertEqual((engine, answer), ("beta", "beta:Haus"))
+
+ def test_a_round_the_decider_never_judged_is_not_recorded(self) -> None:
+ """A fallback winner is not evidence that it translates the pair better."""
+
+ class Unreachable:
+ def judge_answers(self, *_args):
+ raise TimeoutError("timed out")
+
+ history = FakeHistory()
+ router = Router(
+ {"alpha": FakeEngine("alpha"), "beta": FakeEngine("beta")},
+ decider=Unreachable(),
+ history=history,
+ sampler=AlwaysSampler(),
+ order=("beta", "alpha"),
+ )
+
+ run(router.translate("de", "en", "Haus"))
+
+ self.assertEqual(history.recorded, [])
+
+ def test_a_verdict_naming_an_engine_that_did_not_answer_is_discarded(self) -> None:
+ router = Router(
+ {"alpha": FakeEngine("alpha"), "beta": FakeEngine("beta")},
+ decider=FakeDecider(pick="gamma"),
+ history=FakeHistory(),
+ sampler=AlwaysSampler(),
+ order=("alpha", "beta"),
+ )
+
+ engine, _answer = run(router.translate("de", "en", "Haus"))
+
+ self.assertEqual(engine, "alpha")
+
def test_the_decider_is_asked_under_the_configured_question(self) -> None:
decider = FakeDecider(pick="alpha")
router = Router(
@@ -371,11 +425,11 @@ class FakeCache:
self.stored = dict(stored or {})
self.reads = 0
- async def get(self, engine, source, target, text):
+ async def get(self, engine, source, target, text, fmt="text"):
self.reads += 1
return self.stored.get((engine, source, target, text))
- async def set(self, engine, source, target, text, translation):
+ async def set(self, engine, source, target, text, translation, fmt="text"):
self.stored[(engine, source, target, text)] = translation
@@ -390,6 +444,68 @@ class CacheTestCase(unittest.TestCase):
self.assertEqual(run(router.translate("de", "en", "Haus")), ("alpha", "House"))
self.assertEqual(alpha.calls, 0, "a cached answer must not call the engine")
+ def test_an_engine_that_moves_an_emphasis_marker_is_passed_over(self) -> None:
+ """The count is what moves; every character is still present."""
+
+ class Stray:
+ name = "stray"
+ pairs = frozenset()
+
+ def serves(self, *_args):
+ return True
+
+ def translate(self, _source, _target, text, _protected=(), _fmt="text"):
+ return text.replace("**Raw**", "**/ Raw** *")
+
+ router = Router(
+ {"stray": Stray(), "beta": FakeEngine("beta")},
+ order=("stray", "beta"),
+ sampler=NeverSampler(),
+ )
+
+ engine, _answer = run(router.translate("en", "de", "The **Raw** edition"))
+
+ self.assertEqual(engine, "beta")
+
+ def test_an_excluded_engine_is_not_asked(self) -> None:
+ """The caller judges the restored answer the gateway never sees."""
+ alpha = FakeEngine("alpha")
+ router = Router(
+ {"alpha": alpha, "beta": FakeEngine("beta")},
+ order=("alpha", "beta"),
+ sampler=NeverSampler(),
+ )
+
+ engine, _answer = run(router.translate("en", "de", "House", exclude=("alpha",)))
+
+ self.assertEqual(engine, "beta")
+ self.assertEqual(alpha.calls, 0)
+
+ def test_excluding_every_engine_fails_loudly(self) -> None:
+ router = Router(
+ {"alpha": FakeEngine("alpha")}, order=("alpha",), sampler=NeverSampler()
+ )
+
+ with self.assertRaises(NoBackendError):
+ run(router.translate("en", "de", "House", exclude=("alpha",)))
+
+ def test_a_translation_that_keeps_the_markup_is_served(self) -> None:
+ router = Router(
+ {"alpha": FakeEngine("alpha")}, order=("alpha",), sampler=NeverSampler()
+ )
+
+ engine, _answer = run(router.translate("en", "de", "plain prose"))
+
+ self.assertEqual(engine, "alpha")
+
+ def test_the_requested_format_reaches_the_engine(self) -> None:
+ alpha = FakeEngine("alpha")
+ router = Router({"alpha": alpha}, order=("alpha",), sampler=NeverSampler())
+
+ run(router.translate("en", "de", '', fmt="html"))
+
+ self.assertEqual(alpha.formats, ["html"])
+
def test_a_new_request_is_stored_under_its_engine(self) -> None:
cache = FakeCache()
router = Router(
diff --git a/tests/unit/python/roles/web-svc-translate/files/translate_gateway/test_signature.py b/tests/unit/python/roles/web-svc-translate/files/translate_gateway/test_signature.py
index 9f97ef518e..c1a36f5d5b 100644
--- a/tests/unit/python/roles/web-svc-translate/files/translate_gateway/test_signature.py
+++ b/tests/unit/python/roles/web-svc-translate/files/translate_gateway/test_signature.py
@@ -85,6 +85,12 @@ class CacheKeyTestCase(unittest.TestCase):
def test_the_text_itself_does_not_travel_in_the_key(self) -> None:
self.assertNotIn("Haus", cache_key("alpha", "de", "en", "Haus"))
+ def test_another_format_is_another_entry(self) -> None:
+ self.assertNotEqual(
+ cache_key("alpha", "de", "en", '', "text"),
+ cache_key("alpha", "de", "en", '', "html"),
+ )
+
if __name__ == "__main__":
unittest.main()