diff --git a/tests/unit/python/utils/i18n/test_damage.py b/tests/unit/python/utils/i18n/test_damage.py index 3668b2c13c..1b597da383 100644 --- a/tests/unit/python/utils/i18n/test_damage.py +++ b/tests/unit/python/utils/i18n/test_damage.py @@ -67,6 +67,34 @@ class TestWrongScript(unittest.TestCase): def test_a_translation_without_a_word_is_kept(self) -> None: self.assertEqual(reason("Status", "100 %", "be"), "") + def test_a_separatorless_script_counts_one_word_per_character(self) -> None: + """Japanese and Chinese write without spaces, so a run-based pattern + returns the whole sentence as one token, and two Latin product names + inside it carry that token past a half and the ratio to 1.0.""" + self.assertEqual( + reason( + "Runs Playwright in Docker with stable browser settings", + "安定したブラウザ設定でDockerでPlaywrightを実行", + "ja", + ), + "", + ) + self.assertEqual( + reason("Run Playwright in Docker", "在Docker中运行Playwright", "zh"), "" + ) + + def test_a_separatorless_script_answered_in_english_is_still_condemned( + self, + ) -> None: + self.assertEqual( + reason( + "Runs Playwright in Docker with stable settings", + "Runs Playwright in Docker with stable browser settings", + "ja", + ), + "untranslated", + ) + class TestForeignScript(unittest.TestCase): """A Latin-script language came back in a script it does not use. diff --git a/utils/i18n/damage.py b/utils/i18n/damage.py index 8a1e5de0c8..ffe195f31f 100644 --- a/utils/i18n/damage.py +++ b/utils/i18n/damage.py @@ -3,6 +3,14 @@ :func:`reason` is the single predicate both ends of the pipeline ask, and :func:`harms` is its boolean face; the rest of this module are the criteria it folds together. + +``WORDS`` takes one character as one word in Han, Kana, Thai, Lao, Khmer and +Myanmar, which write without separators. Its single-character branch has to +come first, or the run branch swallows a whole sentence together with the +Latin product names inside it and returns one token whose Latin share decides +the verdict alone. ``UNSPACED_RANGES`` holds codepoints rather than escapes +because a formatter rewrites an escape into the character it names, and the +literal blob that leaves behind cannot be read or reviewed. """ from __future__ import annotations @@ -28,7 +36,18 @@ COLLAPSE_RATIO = 0.15 MAX_SHARING_SOURCES = 2 ECHO_FLOOR = 30 STUTTER = re.compile(r"\b(\w+)(?:\s+\1\b){2,}", re.IGNORECASE | re.UNICODE) -WORDS = re.compile(r"[^\W\d_]{2,}", re.UNICODE) +UNSPACED_RANGES = ( + (0x3040, 0x30FF), + (0x3400, 0x4DBF), + (0x4E00, 0x9FFF), + (0xF900, 0xFAFF), + (0x0E00, 0x0E7F), + (0x0E80, 0x0EFF), + (0x1780, 0x17FF), + (0x1000, 0x109F), +) +UNSPACED = "".join(f"{chr(low)}-{chr(high)}" for low, high in UNSPACED_RANGES) +WORDS = re.compile(rf"[{UNSPACED}]|[^\W\d_]{{2,}}", re.UNICODE) LATIN_SHARE = 0.9 NON_LATIN_SCRIPTS = non_latin_languages(PROJECT_ROOT) @@ -94,6 +113,12 @@ def untranslated(translation: str, language: str) -> bool: four-word labels, which is where the wrong language hid for as long as a six-word floor stood here. + The ratio is only as good as the word count, which is why ``WORDS`` splits + a separator-less script per character: a Japanese sentence carrying two + Latin product names is one token to a run-based pattern, and one token + whose Latin share passes a half makes the ratio 1.0 on a sound + translation. + Args: translation: what came back for the entry. language: ISO 639-1 code of the catalog being filled.