mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-08 11:46:35 +00:00
fix(i18n): judge a translation by one predicate on both ends
unmask() accepted a translation as soon as protected_spans matched, while damaged() also rejected it for a dropped name, a halved body or changed markup. Three of the four criteria lived only in the gate, so every run wrote entries the gate then condemned, make i18n-prune emptied them and the next run wrote them back. It never converged: ko carries 331 such entries, 3.7 percent of what it has translated, against 0.1 percent or less for Latin scripts. Introduce harms(source, translation) in placeholders.py as the single source of truth and call it from both ends. truncated() and structure() move there with it, since translate.py already imports placeholders and the other direction would be a cycle. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
91a2582ea4
commit
fe80a23725
3 files changed
+101
-40
No files matched your search
@@ -9,15 +9,18 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from typing import ClassVar
|
||||
|
||||
from utils.i18n.placeholders import (
|
||||
collapse,
|
||||
harms,
|
||||
mask,
|
||||
recapitalise,
|
||||
resegment,
|
||||
terminate,
|
||||
truncated,
|
||||
unmask,
|
||||
)
|
||||
from utils.i18n.translate import truncated
|
||||
|
||||
|
||||
def _round_trip(source: str, translated: str) -> str | None:
|
||||
@@ -145,5 +148,50 @@ class TestTruncated(unittest.TestCase):
|
||||
self.assertFalse(truncated(source, "B" * 180))
|
||||
|
||||
|
||||
class TestWriterAndGateAgree(unittest.TestCase):
|
||||
"""The client must reject exactly what the release gate reports.
|
||||
|
||||
A criterion on one side only makes every run rewrite what the other side
|
||||
then condemns, so `make i18n-prune` and `make i18n-translate` never
|
||||
converge.
|
||||
"""
|
||||
|
||||
SOURCE: ClassVar[str] = (
|
||||
"The deployment registers each service with the reverse proxy and then "
|
||||
"waits until every container reports itself healthy before it continues."
|
||||
)
|
||||
|
||||
def test_the_client_drops_a_translation_the_gate_would_report(self) -> None:
|
||||
halved = "Der Dienst wartet."
|
||||
|
||||
self.assertTrue(harms(self.SOURCE, halved))
|
||||
self.assertIsNone(_round_trip(self.SOURCE, halved))
|
||||
|
||||
def test_the_client_keeps_a_translation_the_gate_accepts(self) -> None:
|
||||
faithful = (
|
||||
"Die Bereitstellung meldet jeden Dienst beim Reverse Proxy an und "
|
||||
"wartet dann, bis jeder Container sich als gesund meldet."
|
||||
)
|
||||
|
||||
self.assertFalse(harms(self.SOURCE, faithful))
|
||||
self.assertEqual(_round_trip(self.SOURCE, faithful), faithful)
|
||||
|
||||
def test_every_criterion_of_the_gate_reaches_the_client(self) -> None:
|
||||
named = "Deploy web-app-docs before the documentation site answers anything."
|
||||
for label, source, translation in (
|
||||
("truncated", self.SOURCE, "Kurz."),
|
||||
("structure", self.SOURCE, f"{self.SOURCE} ["),
|
||||
(
|
||||
"protected span",
|
||||
self.SOURCE,
|
||||
self.SOURCE.replace("reverse proxy", "``rp``"),
|
||||
),
|
||||
("missing name", named, "Stelle die Dokumentationsseite bereit."),
|
||||
):
|
||||
with self.subTest(label):
|
||||
self.assertTrue(harms(source, translation))
|
||||
self.assertIsNone(_round_trip(source, translation))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -76,6 +76,9 @@ def matches(text: str, with_names: bool = True) -> list[tuple[int, int]]:
|
||||
|
||||
|
||||
MARKUP = '[]`*{}()"'
|
||||
STRUCTURE = "[]{}`*()"
|
||||
TRUNCATION_FLOOR = 120
|
||||
TRUNCATION_RATIO = 0.5
|
||||
EMPHASIS = re.compile(r"(\*\*)[ \t]*([^\s]|[^\s].*?[^\s])[ \t]*\1", re.DOTALL)
|
||||
|
||||
|
||||
@@ -203,6 +206,52 @@ def missing_names(source: str, translation: str) -> set[str]:
|
||||
return {name for name in carried if name not in translation}
|
||||
|
||||
|
||||
def truncated(source: str, translation: str) -> bool:
|
||||
"""Return whether ``translation`` kept too little of ``source`` to be one.
|
||||
|
||||
The floor keeps a short entry out: ``Situation`` becomes ``Lage`` and loses
|
||||
half its characters while saying the same thing. A passage past it that
|
||||
comes back halved has dropped a clause.
|
||||
|
||||
Args:
|
||||
source: the source message.
|
||||
translation: what came back for it.
|
||||
"""
|
||||
return (
|
||||
len(source) > TRUNCATION_FLOOR
|
||||
and len(translation) < len(source) * TRUNCATION_RATIO
|
||||
)
|
||||
|
||||
|
||||
def structure(text: str) -> Counter:
|
||||
"""Return the markup characters of ``text`` with their multiplicity.
|
||||
|
||||
Args:
|
||||
text: a source message or its translation.
|
||||
"""
|
||||
return Counter(character for character in text if character in STRUCTURE)
|
||||
|
||||
|
||||
def harms(source: str, translation: str) -> bool:
|
||||
"""Return whether ``translation`` broke something ``source`` carried.
|
||||
|
||||
Single source of truth for both ends of the pipeline: the client drops a
|
||||
translation this rejects, and the release gate reports one that survived
|
||||
anyway. A check on one side only lets every run rewrite what the other
|
||||
side then condemns, which never converges.
|
||||
|
||||
Args:
|
||||
source: the source message.
|
||||
translation: what came back for it.
|
||||
"""
|
||||
return bool(
|
||||
protected_spans(translation) != protected_spans(source)
|
||||
or missing_names(source, translation)
|
||||
or truncated(source, translation)
|
||||
or structure(translation) != structure(source)
|
||||
)
|
||||
|
||||
|
||||
def has_words(text: str) -> bool:
|
||||
"""Return whether ``text`` holds any letter outside its protected spans.
|
||||
|
||||
@@ -259,6 +308,6 @@ def unmask(translated: str, masked: Masked, source: str) -> str | None:
|
||||
restored = recapitalise(terminate(collapse(restored, source), source), source)
|
||||
if sorted(seen) != list(range(len(masked.spans))):
|
||||
return None
|
||||
if not restored or protected_spans(restored) != protected_spans(source):
|
||||
if not restored or harms(source, restored):
|
||||
return None
|
||||
return restored
|
||||
+2
-38
@@ -2,45 +2,14 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from utils.i18n.catalog import MACHINE_TRANSLATION
|
||||
from utils.i18n.placeholders import missing_names, protected_spans
|
||||
from utils.i18n.placeholders import harms
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from babel.messages.catalog import Catalog, Message
|
||||
|
||||
STRUCTURE = "[]{}`*()"
|
||||
TRUNCATION_FLOOR = 120
|
||||
TRUNCATION_RATIO = 0.5
|
||||
|
||||
|
||||
def truncated(source: str, translation: str) -> bool:
|
||||
"""Return whether ``translation`` kept too little of ``source`` to be one.
|
||||
|
||||
The floor keeps a short entry out: ``Situation`` becomes ``Lage`` and loses
|
||||
half its characters while saying the same thing. A passage past it that
|
||||
comes back halved has dropped a clause.
|
||||
|
||||
Args:
|
||||
source: the source message.
|
||||
translation: what came back for it.
|
||||
"""
|
||||
return (
|
||||
len(source) > TRUNCATION_FLOOR
|
||||
and len(translation) < len(source) * TRUNCATION_RATIO
|
||||
)
|
||||
|
||||
|
||||
def structure(text: str) -> Counter:
|
||||
"""Return the markup characters of ``text`` with their multiplicity.
|
||||
|
||||
Args:
|
||||
text: a source message or its translation.
|
||||
"""
|
||||
return Counter(character for character in text if character in STRUCTURE)
|
||||
|
||||
|
||||
def pending(catalog: Catalog) -> list[Message]:
|
||||
"""Return the entries of ``catalog`` that still need a translation.
|
||||
@@ -75,12 +44,7 @@ def damaged(catalog: Catalog) -> list[Message]:
|
||||
if isinstance(message.id, str)
|
||||
and message.id
|
||||
and message.string
|
||||
and (
|
||||
protected_spans(str(message.string)) != protected_spans(message.id)
|
||||
or missing_names(message.id, str(message.string))
|
||||
or truncated(message.id, str(message.string))
|
||||
or structure(str(message.string)) != structure(message.id)
|
||||
)
|
||||
and harms(message.id, str(message.string))
|
||||
]
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user