mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-10 12:47:04 +00:00
fix(i18n): stop resegment from splitting a name an agglutinative language suffixed
2aa7325eca put role names and product titles into the spans mask() produces, and
unmask hands those same spans to resegment. That was safe for every span the
protection had before, because each ends in a delimiter, so a letter right
behind it can only be the text a lost boundary was taken from. A bare name ends
in a word character, and Bengali writes `Baserowo` where the source says
`Baserow`, so the substring search found the name inside the longer word and put
the punctuation in the middle of it:
resegment('Baserowo-r shathe', ('Baserow',), 'Adapter to Baserow: ...')
-> 'Baserow: o-r shathe'
Names are skipped. The repair still fires for the spans it was written for, and
the 85 entries the boundary lint reported across et, fi, eu, cs, ja and bn drop
to none.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
da4dba4618
commit
0c541cbb2f
2 files changed
+37
-1
No files matched your search
@@ -9,7 +9,14 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from utils.i18n.placeholders import collapse, mask, recapitalise, terminate, unmask
|
||||
from utils.i18n.placeholders import (
|
||||
collapse,
|
||||
mask,
|
||||
recapitalise,
|
||||
resegment,
|
||||
terminate,
|
||||
unmask,
|
||||
)
|
||||
from utils.i18n.translate import truncated
|
||||
|
||||
|
||||
@@ -103,6 +110,26 @@ class TestUnmaskAppliesTheRepairs(unittest.TestCase):
|
||||
self.assertEqual(restored, "Alle `alpha` Kriterien.")
|
||||
|
||||
|
||||
class TestResegmentLeavesNamesAlone(unittest.TestCase):
|
||||
def test_a_suffixed_name_keeps_its_word(self) -> None:
|
||||
source = "Adapter to Baserow: the endpoint key"
|
||||
|
||||
self.assertEqual(
|
||||
resegment("Baserowo-r shathe", mask(source).spans, source),
|
||||
"Baserowo-r shathe",
|
||||
)
|
||||
|
||||
def test_a_swallowed_boundary_behind_a_delimiter_is_still_put_back(self) -> None:
|
||||
source = "See `compose.yml`. The next sentence."
|
||||
|
||||
self.assertEqual(
|
||||
resegment(
|
||||
"Siehe `compose.yml`Der naechste Satz.", mask(source).spans, source
|
||||
),
|
||||
"Siehe `compose.yml`. Der naechste Satz.",
|
||||
)
|
||||
|
||||
|
||||
class TestTruncated(unittest.TestCase):
|
||||
def test_a_short_entry_may_shrink(self) -> None:
|
||||
self.assertFalse(truncated("Situation", "Lage"))
|
||||
|
||||
Reference in new issue
Block a user