mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-10 04:36:40 +00:00
fix(i18n): count a word per character where a script has no separators
untranslated() asks what share of a translation's words are Latin. WORDS matched runs of letters, and Han, Kana, Thai, Lao, Khmer and Myanmar write without separators, so a whole sentence came back as one token. Two Latin product names inside it carried that token's latin_share past a half, which made the ratio 1.0 and condemned a sound translation.
Measured on a Japanese rendering of "Runs Playwright in Docker with stable browser settings": one token, latin_share 0.516, ratio 1.00 against a threshold of 0.9. The six-word floor removed in 27d59f4bd4 had hidden this, because a separator-less script never reaches six words; dropping it as vacuous missed that it was the only guard those scripts had.
WORDS now takes one character of such a script as one word, with the single-character branch first so the run branch cannot swallow the sentence again. Ten real entries against both patterns: the old one errs on two, the new one on none, and every true positive survives, including Cyrillic welds like CI Деbugging and an English answer in a Japanese catalog.
The ranges are codepoints because a formatter rewrites an escape into the character it names, and the resulting literal blob cannot be reviewed.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
213f45031f
commit
2d1227d0f1
2 files changed
+54
-1
No files matched your search
@@ -67,6 +67,34 @@ class TestWrongScript(unittest.TestCase):
|
||||
def test_a_translation_without_a_word_is_kept(self) -> None:
|
||||
self.assertEqual(reason("Status", "100 %", "be"), "")
|
||||
|
||||
def test_a_separatorless_script_counts_one_word_per_character(self) -> None:
|
||||
"""Japanese and Chinese write without spaces, so a run-based pattern
|
||||
returns the whole sentence as one token, and two Latin product names
|
||||
inside it carry that token past a half and the ratio to 1.0."""
|
||||
self.assertEqual(
|
||||
reason(
|
||||
"Runs Playwright in Docker with stable browser settings",
|
||||
"安定したブラウザ設定でDockerでPlaywrightを実行",
|
||||
"ja",
|
||||
),
|
||||
"",
|
||||
)
|
||||
self.assertEqual(
|
||||
reason("Run Playwright in Docker", "在Docker中运行Playwright", "zh"), ""
|
||||
)
|
||||
|
||||
def test_a_separatorless_script_answered_in_english_is_still_condemned(
|
||||
self,
|
||||
) -> None:
|
||||
self.assertEqual(
|
||||
reason(
|
||||
"Runs Playwright in Docker with stable settings",
|
||||
"Runs Playwright in Docker with stable browser settings",
|
||||
"ja",
|
||||
),
|
||||
"untranslated",
|
||||
)
|
||||
|
||||
|
||||
class TestForeignScript(unittest.TestCase):
|
||||
"""A Latin-script language came back in a script it does not use.
|
||||
|
||||
Reference in new issue
Block a user