mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-09 04:06:42 +00:00
kevinveenbirkenbachandClaude Opus 5
c64c0628ae
27d59f4bd4 caught a non-Latin target answered in Latin letters. Its
mirror was missing, so the model answering a language it cannot write
with one it can passed every criterion:
ch 'AI Services' -> '人工智能服务' Chamorro, answered in Han
tr 'Sales' -> 'إدارة'
ae (76 entries) -> Arabic, Persian, Hebrew, Chinese
Chamorro writes in the Latin script and lives on Guam; the code 'ch'
reads as Chinese to a model that does not know the language, which is
how the whole catalog came back in Mandarin.
Measured over the 121 Latin-script catalogs before the rule was
written: 79 entries condemned, 76 of them in ae and 3 in tr. All three
Turkish ones were read individually and none is a false positive; two
are an Arabic fragment with a markdown rule and one is English plus
Bengali repeated four times.
Its own criterion rather than an arm of untranslated(): the catalogs
record the reason on the entry, and 'answered in another script' is a
different diagnosis from 'came back untranslated'. The two together
cover the script dimension for all 184 languages.
What it still cannot see: Spanish served as Corsican. Telling two
Latin-script languages apart needs a detector this repository does not
have, so that class stays open.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
98 lines
3.9 KiB
Python
98 lines
3.9 KiB
Python
"""Unit tests for the criteria that condemn an answer carrying its source or
|
|
coming back in the wrong script.
|
|
|
|
The shape is one the gateway produced across twenty languages: the English
|
|
source, a markdown rule, then a rendering. Every other criterion passed it,
|
|
because the rendering keeps the spans and the names and the separator is made
|
|
of characters the markup comparison does not count.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import unittest
|
|
from typing import ClassVar
|
|
|
|
from utils.i18n.damage import reason
|
|
|
|
|
|
class TestEchoAndContinue(unittest.TestCase):
|
|
"""A translation holding its whole source is the server failing, not translating."""
|
|
|
|
SOURCE: ClassVar[str] = (
|
|
"Corporate design injection for NGINX-served apps, unifying light and "
|
|
"dark theming from OKLCH design tokens"
|
|
)
|
|
RENDERING: ClassVar[str] = (
|
|
"Corporate-Design-Injection für NGINX-bediente Anwendungen, vereinheitlicht "
|
|
"helles und dunkles Theming aus OKLCH-Design-Tokens"
|
|
)
|
|
|
|
def test_an_answer_that_prepends_its_source_is_condemned(self) -> None:
|
|
answer = f"{self.SOURCE}\n---\n\n{self.RENDERING}"
|
|
self.assertEqual(reason(self.SOURCE, answer, "de"), "echo")
|
|
|
|
def test_an_answer_that_appends_its_source_is_condemned(self) -> None:
|
|
answer = f"{self.RENDERING}\n\n{self.SOURCE}"
|
|
self.assertEqual(reason(self.SOURCE, answer, "de"), "echo")
|
|
|
|
def test_a_rendering_that_drops_the_source_is_kept(self) -> None:
|
|
self.assertEqual(reason(self.SOURCE, self.RENDERING, "de"), "")
|
|
|
|
def test_a_short_source_may_live_inside_a_longer_rendering(self) -> None:
|
|
source = "Mail"
|
|
self.assertEqual(reason(source, "Mail-Konto des Benutzers", "de"), "")
|
|
|
|
|
|
class TestWrongScript(unittest.TestCase):
|
|
"""A language written in another script came back in Latin letters.
|
|
|
|
The catalogs are mostly one- to four-word labels, and a six-word floor
|
|
once stood in front of this criterion, so it never fired on them: a whole
|
|
run filled Belarusian with Polish and Abkhaz with Spanish while every
|
|
criterion passed.
|
|
"""
|
|
|
|
def test_a_one_word_answer_in_the_wrong_script_is_condemned(self) -> None:
|
|
self.assertEqual(reason("Applications", "Aplikacje", "be"), "untranslated")
|
|
|
|
def test_a_label_answered_in_another_language_is_condemned(self) -> None:
|
|
self.assertEqual(reason("Communication", "Comunicación", "ab"), "untranslated")
|
|
|
|
def test_an_answer_in_the_language_s_own_script_is_kept(self) -> None:
|
|
self.assertEqual(reason("Applications", "Праграмы", "be"), "")
|
|
|
|
def test_a_latin_script_target_is_not_judged_by_script(self) -> None:
|
|
self.assertEqual(reason("Applications", "Anwendungen", "de"), "")
|
|
|
|
def test_a_translation_without_a_word_is_kept(self) -> None:
|
|
self.assertEqual(reason("Status", "100 %", "be"), "")
|
|
|
|
|
|
class TestForeignScript(unittest.TestCase):
|
|
"""A Latin-script language came back in a script it does not use.
|
|
|
|
The model answers a language it cannot write with one it can. Chamorro
|
|
came back in Han because the ISO code ``ch`` reads as Chinese, Turkish in
|
|
Bengali, Avestan in Arabic, and none of it was caught while only the
|
|
non-Latin direction was checked.
|
|
"""
|
|
|
|
def test_han_in_a_latin_script_catalog_is_condemned(self) -> None:
|
|
self.assertEqual(reason("AI Services", "人工智能服务", "ch"), "foreign-script")
|
|
|
|
def test_arabic_in_a_latin_script_catalog_is_condemned(self) -> None:
|
|
self.assertEqual(reason("Sales", "إدارة", "tr"), "foreign-script")
|
|
|
|
def test_a_latin_script_answer_is_kept(self) -> None:
|
|
self.assertEqual(reason("Applications", "Anwendungen", "de"), "")
|
|
|
|
def test_a_non_latin_target_is_not_judged_by_this_rule(self) -> None:
|
|
self.assertEqual(reason("Applications", "Праграмы", "be"), "")
|
|
|
|
def test_a_translation_without_a_word_is_kept(self) -> None:
|
|
self.assertEqual(reason("Status", "100 %", "ch"), "")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|