From 0c541cbb2fa1a41ff0c8f2d5b517cb954d4ae47b Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 24 Sep 2026 23:10:47 +0200 Subject: [PATCH] fix(i18n): stop resegment from splitting a name an agglutinative language suffixed 2aa7325eca put role names and product titles into the spans mask() produces, and unmask hands those same spans to resegment. That was safe for every span the protection had before, because each ends in a delimiter, so a letter right behind it can only be the text a lost boundary was taken from. A bare name ends in a word character, and Bengali writes `Baserowo` where the source says `Baserow`, so the substring search found the name inside the longer word and put the punctuation in the middle of it: resegment('Baserowo-r shathe', ('Baserow',), 'Adapter to Baserow: ...') -> 'Baserow: o-r shathe' Names are skipped. The repair still fires for the spans it was written for, and the 85 entries the boundary lint reported across et, fi, eu, cs, ja and bn drop to none. Co-Authored-By: Claude Opus 5 (1M context) --- tests/unit/python/utils/i18n/test_repairs.py | 29 +++++++++++++++++++- utils/i18n/placeholders.py | 9 ++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/unit/python/utils/i18n/test_repairs.py b/tests/unit/python/utils/i18n/test_repairs.py index 12adc16f33..b3348ba69a 100644 --- a/tests/unit/python/utils/i18n/test_repairs.py +++ b/tests/unit/python/utils/i18n/test_repairs.py @@ -9,7 +9,14 @@ from __future__ import annotations import unittest -from utils.i18n.placeholders import collapse, mask, recapitalise, terminate, unmask +from utils.i18n.placeholders import ( + collapse, + mask, + recapitalise, + resegment, + terminate, + unmask, +) from utils.i18n.translate import truncated @@ -103,6 +110,26 @@ class TestUnmaskAppliesTheRepairs(unittest.TestCase): self.assertEqual(restored, "Alle `alpha` Kriterien.") +class TestResegmentLeavesNamesAlone(unittest.TestCase): + def test_a_suffixed_name_keeps_its_word(self) -> None: + source = "Adapter to Baserow: the endpoint key" + + self.assertEqual( + resegment("Baserowo-r shathe", mask(source).spans, source), + "Baserowo-r shathe", + ) + + def test_a_swallowed_boundary_behind_a_delimiter_is_still_put_back(self) -> None: + source = "See `compose.yml`. The next sentence." + + self.assertEqual( + resegment( + "Siehe `compose.yml`Der naechste Satz.", mask(source).spans, source + ), + "Siehe `compose.yml`. Der naechste Satz.", + ) + + class TestTruncated(unittest.TestCase): def test_a_short_entry_may_shrink(self) -> None: self.assertFalse(truncated("Situation", "Lage")) diff --git a/utils/i18n/placeholders.py b/utils/i18n/placeholders.py index 9aa99bc404..a4169ed346 100644 --- a/utils/i18n/placeholders.py +++ b/utils/i18n/placeholders.py @@ -82,12 +82,21 @@ EMPHASIS = re.compile(r"(\*\*)[ \t]*([^\s]|[^\s].*?[^\s])[ \t]*\1", re.DOTALL) def resegment(restored: str, spans: tuple[str, ...], source: str) -> str: """Put back the sentence boundary a translator dropped behind a protected span. + A name is skipped. Every other protected span ends in a delimiter, so a + letter right behind it can only be the text the boundary was lost from, but + a name ends in a word character and an agglutinative language suffixes it: + ``Baserow`` is found inside the Bengali ``Baserowo`` and the punctuation + would land inside the word. + Args: restored: the translation with every protected span put back. spans: the protected spans of the source. source: the source message, which holds the boundary that went missing. """ + named = {source[start:end] for start, end in names.spans(source)} for span in spans: + if span in named: + continue start = source.find(span) if start < 0: continue