diff --git a/cli/build/i18n/__main__.py b/cli/build/i18n/__main__.py index cc02f9dd39..2d8686d677 100644 --- a/cli/build/i18n/__main__.py +++ b/cli/build/i18n/__main__.py @@ -44,7 +44,14 @@ from utils.i18n.libretranslate import ( server, ) from utils.i18n.placeholders import Rejected -from utils.i18n.translate import apply, damaged, discard, pending, retry +from utils.i18n.translate import ( + apply, + damaged, + discard, + pending, + retry, + taken_translations, +) CHUNK_SIZE = 500 SPHINX_JOBS_FLOOR = 2 @@ -153,6 +160,7 @@ def one_catalog(client: LibreTranslate, domain: str, code: str) -> None: path = catalog_path(PROJECT_ROOT, code, domain) catalog = read_catalog(path) todo = pending(catalog) + taken = taken_translations(catalog) discarded = refused = damaged = 0 criteria: Counter[str] = Counter() refusal = "" @@ -162,7 +170,7 @@ def one_catalog(client: LibreTranslate, domain: str, code: str) -> None: criteria.update( value.reason for value in outcome.values if isinstance(value, Rejected) ) - discarded += apply(chunk, outcome.values) + discarded += apply(chunk, outcome.values, taken, criteria) refused += outcome.refused damaged += outcome.damaged refusal = outcome.refusal or refusal diff --git a/tests/unit/python/utils/i18n/test_translate.py b/tests/unit/python/utils/i18n/test_translate.py index 438b3508b1..d62224fa4f 100644 --- a/tests/unit/python/utils/i18n/test_translate.py +++ b/tests/unit/python/utils/i18n/test_translate.py @@ -20,7 +20,14 @@ from utils.i18n.catalog import ( ) from utils.i18n.libretranslate import LibreTranslate from utils.i18n.placeholders import Rejected -from utils.i18n.translate import URL_SUPPRESSION, apply, damaged, pending, retry +from utils.i18n.translate import ( + URL_SUPPRESSION, + apply, + damaged, + pending, + retry, + taken_translations, +) TOKEN = re.compile(r'') @@ -318,5 +325,50 @@ class TestDamagedMarkup(unittest.TestCase): self.assertEqual(damaged(catalog), []) +class TestDuplicateTranslation(unittest.TestCase): + LONG_A = "Central identity provider issuing tokens for every service" + LONG_B = "Issue and project tracking platform for software teams today" + SHORT_A = "Save the file" + SHORT_B = "Open the file" + + def _messages(self, *sources: str) -> list: + catalog = Catalog(locale="de") + for source in sources: + catalog.add(source, "") + return [message for message in catalog if message.id] + + def test_a_second_long_source_may_not_reuse_a_translation(self) -> None: + messages = self._messages(self.LONG_A, self.LONG_B) + + discarded = apply( + messages, + ["Beschreibung eines Dienstes", "Beschreibung eines Dienstes"], + set(), + ) + + self.assertEqual(discarded, 1) + self.assertEqual(str(messages[1].string), "") + + def test_short_sources_may_share_one(self) -> None: + messages = self._messages(self.SHORT_A, self.SHORT_B) + + discarded = apply(messages, ["Speichern", "Speichern"], set()) + + self.assertEqual(discarded, 0) + self.assertEqual(str(messages[1].string), "Speichern") + + def test_without_the_set_the_check_is_skipped(self) -> None: + messages = self._messages(self.LONG_A, self.LONG_B) + + self.assertEqual(apply(messages, ["Gleich", "Gleich"], None), 0) + + def test_what_the_catalog_already_spends_is_seeded(self) -> None: + catalog = Catalog(locale="de") + catalog.add(self.LONG_A, "Schon vergeben") + catalog.add(self.SHORT_A, "Kurz") + + self.assertEqual(taken_translations(catalog), {"Schon vergeben"}) + + if __name__ == "__main__": unittest.main() diff --git a/utils/i18n/translate.py b/utils/i18n/translate.py index 8f7200b3fa..04af41deba 100644 --- a/utils/i18n/translate.py +++ b/utils/i18n/translate.py @@ -11,13 +11,22 @@ from utils.i18n.catalog import ( REJECTED_PREFIX, TRANSLATION_REFUSED, ) -from utils.i18n.placeholders import Rejected, harms, protected_spans +from utils.i18n.placeholders import ( + COLLAPSE_FLOOR, + Rejected, + harms, + protected_spans, +) +from utils.i18n.spans import prose if TYPE_CHECKING: + from collections import Counter + from babel.messages.catalog import Catalog, Message REFUSAL_PREFIXES = (REFUSAL_PREFIX, REJECTED_PREFIX) URL_SUPPRESSION = "nocheck: url" +DUPLICATE = "duplicate" def refusals(message: Message) -> list[str]: @@ -193,13 +202,53 @@ def record(message: Message, rejected: Rejected | None) -> None: ) -def apply(messages: list[Message], results: list[str | Rejected | None]) -> int: +def _own_translation_due(source: str) -> bool: + """Return whether ``source`` is long enough to deserve its own translation. + + Short entries legitimately share one: several buttons are all ``Save``. + The floor is the one the collapse check reads, and inclusive as it is + there, so the writer turns down exactly what the release gate reports + rather than leaving it a source of exactly that length. + + Args: + source: the source message. + """ + return len(prose(source).strip()) >= COLLAPSE_FLOOR + + +def taken_translations(catalog: Catalog) -> set[str]: + """Return the translations ``catalog`` already spends on a long source. + + Args: + catalog: the catalog about to be translated into. + """ + return { + str(message.string) + for message in catalog + if message.id and message.string and _own_translation_due(str(message.id)) + } + + +def apply( + messages: list[Message], + results: list[str | Rejected | None], + taken: set[str] | None = None, + criteria: Counter[str] | None = None, +) -> int: """Store machine translations on ``messages``. Args: messages: entries returned by ``pending``. results: one entry per message: the translation, a ``Rejected``, or ``None`` where the server never answered. + criteria: tally the caller prints, which counts what the client + rejected. A duplicate is turned down here rather than there, so + without this the run reports the discard and no reason for it. + taken: translations this catalog already carries for a source long + enough to deserve its own. A repeat is turned down: what is wrong + with it is not its shape, which every predicate passes, but that + another entry already says it. Pass None to skip the check, which + is what a caller without the whole catalog must do. Returns: The number of discarded translations; their entries keep no ``msgstr``. @@ -210,6 +259,14 @@ def apply(messages: list[Message], results: list[str | Rejected | None]) -> int: discarded += 1 record(message, text) continue + if taken is not None and _own_translation_due(str(message.id)): + if text in taken: + discarded += 1 + record(message, Rejected(text, DUPLICATE)) + if criteria is not None: + criteria[DUPLICATE] += 1 + continue + taken.add(text) message.string = text message.flags.discard("fuzzy") clear(message)