mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-08 19:56:53 +00:00
kevinveenbirkenbachandClaude Opus 5
52d356544b
meta/languages.yml carried `libretranslate: true|false` per entry, which
answered two different questions with one field: which languages the
repository translates, and which engine renders them. The first is the
table's business; the second is not. The gateway offers every backend
that serves a pair, falls through when one refuses or mangles the
answer, drops an engine whose failures pile up, and settles a request
class on the engine that wins its comparisons - and `signature()` is
"{source}:{target}:{length bucket}:{markup|plain}", so that learning is
already per target language. A static column beside it is a second
opinion that goes stale the moment the router learns something.
So the field is gone, 184 lines, and the table now holds name, endonym
and direction. Every language it names carries a catalog in every
domain.
What one engine can do is declared with that engine instead:
services.libretranslate-engine.serves, 47 codes. Not a detour - both
readers run before any engine answers. The lane prefetches one model per
code, and the readiness probe is itself the thing that would have asked.
`translatable()` split along that seam. Catalog-scoped callers take
`domain_languages()`, engine-scoped ones take the new
`served_languages()`: the CI matrix, extraction and translation the
first, the model prefetch and the tuning sweep the second. Collapsing
the two is what this change is guarding against - `baked_languages()`
renders identically before and after, verified by diffing both outputs.
`translate()` also stops dying on a language whose catalog has not been
extracted yet. It reads a missing catalog as nothing pending rather than
raising, because 136 docs catalogs do not exist until the next
extraction and refusing those languages would quietly shrink the set the
repository promises to translate.
The oracle suite follows: tests/oracle/test_translation_score.py asks
utils/i18n/judge.py about every language instead of routing a laya
checkpoint per language. It cannot run yet - the tools lane carries no
LiteLLM - and its skip message says exactly that.
test_catalogs_complete_on_gpu now measures against the table rather than
one engine's coverage. It is live on a GPU host and will report the 136
missing catalogs as the gap they are, which is the point: narrowing it
to the engine would hide exactly the languages worth watching.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
96 lines
3.2 KiB
Python
96 lines
3.2 KiB
Python
"""The chat judge places a translation on a named level, or refuses to answer."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import unittest
|
|
import unittest.mock
|
|
|
|
from utils.i18n import judge
|
|
from utils.i18n.judge import LEVELS, MIDDLE, ChatJudge, parse_level, question
|
|
|
|
|
|
def _answer(content):
|
|
payload = {"choices": [{"message": {"content": content}}]}
|
|
return io.BytesIO(json.dumps(payload).encode())
|
|
|
|
|
|
class QuestionTestCase(unittest.TestCase):
|
|
def test_both_texts_reach_the_model(self) -> None:
|
|
asked = question("The backup runs at two.", "Das Backup läuft um zwei.")
|
|
|
|
self.assertIn("The backup runs at two.", asked)
|
|
self.assertIn("Das Backup läuft um zwei.", asked)
|
|
|
|
def test_every_level_is_offered_under_its_index(self) -> None:
|
|
asked = question("a", "b")
|
|
|
|
for index, level in enumerate(LEVELS):
|
|
with self.subTest(level=index):
|
|
self.assertIn(f"{index} {level}", asked)
|
|
|
|
def test_the_middle_level_is_the_one_the_suite_compares_against(self) -> None:
|
|
self.assertEqual(LEVELS[MIDDLE], LEVELS[1])
|
|
|
|
|
|
class ParseLevelTestCase(unittest.TestCase):
|
|
def test_a_bare_digit_is_the_level(self) -> None:
|
|
self.assertEqual(parse_level("2"), 2)
|
|
|
|
def test_a_digit_inside_a_sentence_still_counts(self) -> None:
|
|
self.assertEqual(parse_level("Level 1, it distorts a detail."), 1)
|
|
|
|
def test_an_answer_without_a_digit_is_refused(self) -> None:
|
|
with self.assertRaises(ValueError):
|
|
parse_level("I cannot judge this pair.")
|
|
|
|
def test_an_empty_answer_is_refused(self) -> None:
|
|
with self.assertRaises(ValueError):
|
|
parse_level("")
|
|
|
|
def test_a_level_outside_the_scale_is_refused(self) -> None:
|
|
"""Clamping would put a score on a non-answer: a model replying 7 has
|
|
not placed the pair on this scale at all."""
|
|
with self.assertRaises(ValueError):
|
|
parse_level("7")
|
|
|
|
|
|
class ScoreTestCase(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.judge = ChatJudge("http://gateway:4000/", "chat", api_key="k")
|
|
|
|
def _call(self, content):
|
|
with unittest.mock.patch.object(judge.urllib.request, "urlopen") as urlopen:
|
|
urlopen.return_value.__enter__.return_value = _answer(content)
|
|
result = self.judge.score("source", "translation")
|
|
self.request = urlopen.call_args[0][0]
|
|
self.sent = json.loads(self.request.data)
|
|
return result
|
|
|
|
def test_it_returns_the_level_the_model_named(self) -> None:
|
|
self.assertEqual(self._call("0"), 0)
|
|
|
|
def test_the_trailing_slash_of_the_base_url_is_not_doubled(self) -> None:
|
|
self._call("2")
|
|
|
|
self.assertEqual(self.request.full_url, "http://gateway:4000/chat/completions")
|
|
|
|
def test_the_key_travels_as_a_bearer_token(self) -> None:
|
|
self._call("2")
|
|
|
|
self.assertEqual(self.request.get_header("Authorization"), "Bearer k")
|
|
|
|
def test_the_judge_is_asked_without_sampling(self) -> None:
|
|
self._call("2")
|
|
|
|
self.assertEqual(self.sent["temperature"], 0)
|
|
|
|
def test_an_empty_completion_is_refused(self) -> None:
|
|
with self.assertRaises(ValueError):
|
|
self._call(" ")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|