mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-09 20:26:33 +00:00
kevinveenbirkenbachandClaude Opus 5
d2bc925671
untranslated() asked word.isascii() to decide whether a word is Latin. Pseudo-Faroese is Latin and not ASCII: afviksluður, byrðir, við, níu. On the gateway's own rendering for Avaric the measured Latin share came out at 0.36 against a threshold of 0.90, so the text passed into a Cyrillic catalog untouched. languages.py already held the right measure inside _is_non_latin, which reads the Unicode name of each letter. It becomes latin_share() and both callers share it. _is_non_latin keeps its behaviour: None for letterless text preserves the old False, where a bare 0.0 would have claimed non-Latin. Swept over every catalog: 582 damaged entries, all 582 invisible to the ascii rule, 569 of them in av, as and ba. make i18n-prune cleared exactly those and the damaged assertion went green again. The floor of 6 words stays. Lowering it to 2 condemns 4053 entries instead of 39, among them 'DNS Resolvers' and 'CI Runner' in core/ar, which are rightly Latin in an Arabic catalog. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
96 lines
3.4 KiB
Python
96 lines
3.4 KiB
Python
"""The language list of ``meta/languages.yml``.
|
|
|
|
The file is the one place that names the languages this repository supports,
|
|
and it carries nothing but their identity: name, endonym and writing
|
|
direction. Every language it names gets a catalog in every domain.
|
|
|
|
Which engine renders into a language is deliberately not written here. The
|
|
translation gateway offers each backend that serves the pair, falls through to
|
|
the next one when an engine refuses or mangles the answer, drops an engine
|
|
whose failures pile up, and settles a language on the engine that wins its
|
|
comparisons. A static table beside that would be a second opinion that goes
|
|
stale the moment the router learns something. What a single engine can do is
|
|
declared with that engine instead: ``services.libretranslate-engine.serves``
|
|
of ``roles/svc-ai-libretranslate-engine`` states the codes its models cover,
|
|
because the lane prefetches those models before any engine can be asked.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import unicodedata
|
|
from functools import lru_cache
|
|
from pathlib import Path
|
|
|
|
from utils.cache.yaml import load_yaml
|
|
|
|
SOURCE_LANGUAGE = "en"
|
|
LANGUAGES_FILE = Path("meta") / "languages.yml"
|
|
DOMAINS = ("core", "docs")
|
|
_LETTER = re.compile(r"[^\W\d_]", re.UNICODE)
|
|
|
|
|
|
def load_languages(root: Path) -> dict[str, dict]:
|
|
"""Return every language of ``meta/languages.yml`` keyed by ISO 639-1 code.
|
|
|
|
Args:
|
|
root: repository root.
|
|
"""
|
|
return load_yaml(root / LANGUAGES_FILE)
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def non_latin_languages(root: Path) -> frozenset[str]:
|
|
"""Return the codes whose endonym is not written in the Latin script.
|
|
|
|
The endonym is the evidence: a language writes its own name in its own
|
|
script, so the table already states which script each language expects
|
|
and a second hand-kept list beside it only drifts. It drifted: the set
|
|
this replaces named 41 languages and missed 21 the table answers for,
|
|
Assamese among them, which is how a catalogue filled with Latin text
|
|
passed every check.
|
|
|
|
Args:
|
|
root: repository root.
|
|
"""
|
|
return frozenset(
|
|
code
|
|
for code, entry in load_languages(root).items()
|
|
if _is_non_latin(str(entry.get("native", "")))
|
|
)
|
|
|
|
|
|
def latin_share(text: str) -> float | None:
|
|
"""Return the share of this text's letters written in the Latin script.
|
|
|
|
The script of a letter, not its codepoint range: ``ð`` and ``í`` are
|
|
Latin and not ASCII, which is the difference between reading
|
|
pseudo-Faroese as Latin text and reading it as something else.
|
|
|
|
Args:
|
|
text: any text; None when it carries no letter at all.
|
|
"""
|
|
letters = [char for char in text if _LETTER.match(char)]
|
|
if not letters:
|
|
return None
|
|
latin = sum(1 for char in letters if "LATIN" in unicodedata.name(char, ""))
|
|
return latin / len(letters)
|
|
|
|
|
|
def _is_non_latin(endonym: str) -> bool:
|
|
"""Whether fewer than half of a name's letters are Latin ones."""
|
|
share = latin_share(endonym)
|
|
return share is not None and share < 0.5
|
|
|
|
|
|
def domain_languages(languages: dict[str, dict], domain: str) -> list[str]:
|
|
"""Return the codes that carry a catalog of ``domain``.
|
|
|
|
Args:
|
|
languages: the mapping returned by ``load_languages``.
|
|
domain: a name of :data:`DOMAINS`.
|
|
"""
|
|
if domain not in DOMAINS:
|
|
raise ValueError(f"unknown domain: {domain}")
|
|
return sorted(code for code in languages if code != SOURCE_LANGUAGE)
|