diff --git a/tests/unit/python/utils/i18n/test_names.py b/tests/unit/python/utils/i18n/test_names.py new file mode 100644 index 0000000000..9bb0f3240c --- /dev/null +++ b/tests/unit/python/utils/i18n/test_names.py @@ -0,0 +1,130 @@ +"""Unit tests for :mod:`utils.i18n.names` and the span merge that consumes it. + +The cases use names the repository really ships, so a renamed role shows up here +rather than in a silently weakened protection. +""" + +from __future__ import annotations + +import unittest +from itertools import pairwise + +from utils.i18n.names import spans +from utils.i18n.placeholders import ( + has_words, + matches, + missing_names, + protected_spans, +) + + +def _protected(text: str) -> set[str]: + return {text[start:end] for start, end in matches(text)} + + +class TestRoleNames(unittest.TestCase): + def test_a_role_name_is_protected_mid_sentence(self) -> None: + self.assertIn("web-app-keycloak", _protected("Deploy web-app-keycloak first")) + + def test_a_role_name_is_protected_at_the_start(self) -> None: + self.assertIn("web-app-keycloak", _protected("web-app-keycloak issues tokens")) + + def test_a_longer_identifier_is_not_cut_short(self) -> None: + self.assertNotIn("web-app", _protected("Deploy web-app-keycloak first")) + + def test_ordinary_hyphenated_english_stays_translatable(self) -> None: + self.assertNotIn("self-hosted", _protected("A self-hosted open-source service")) + + +class TestTitles(unittest.TestCase): + def test_a_title_inside_a_sentence_is_protected(self) -> None: + self.assertIn("Keycloak", _protected("The realm Keycloak issues tokens for")) + + def test_a_title_that_is_also_an_adjective_is_protected_inside_a_sentence( + self, + ) -> None: + self.assertIn("Unbound", _protected("DNS resolver engines (Unbound, etc.)")) + + def test_a_heading_stays_translatable(self) -> None: + self.assertNotIn("Documentation", _protected("Documentation and Wiki Apps")) + + def test_a_title_opening_a_sentence_stays_translatable(self) -> None: + self.assertNotIn("Keycloak", _protected("Keycloak issues the token")) + + def test_a_lowercase_reading_stays_translatable(self) -> None: + self.assertNotIn("shell", _protected("bundling caffeine, shell extensions")) + + def test_a_lowercase_common_noun_stays_translatable(self) -> None: + self.assertNotIn( + "documentation", _protected("including documentation and help") + ) + + +class TestExtraSpans(unittest.TestCase): + def test_an_emoji_is_protected(self) -> None: + self.assertIn("🌐", _protected("ActivityPub Support 🌐 Seamlessly integrate")) + + def test_an_email_address_is_protected(self) -> None: + self.assertIn("kevin@veen.world", _protected("Contact kevin@veen.world today")) + + +class TestMerge(unittest.TestCase): + def test_spans_never_overlap(self) -> None: + text = "Deploy `web-app-keycloak` next to Keycloak and Nextcloud 🌐" + offsets = matches(text) + + self.assertEqual(offsets, sorted(offsets)) + for (_, end), (start, _) in pairwise(offsets): + self.assertLessEqual(end, start) + + def test_a_backticked_name_is_claimed_once(self) -> None: + text = "see `web-app-keycloak` for details" + + self.assertIn("`web-app-keycloak`", _protected(text)) + + def test_a_name_stays_out_of_the_comparison_set(self) -> None: + self.assertNotIn("Keycloak", protected_spans("Keycloak issues the token")) + + +class TestMissingNames(unittest.TestCase): + def test_a_dropped_name_is_reported(self) -> None: + lost = missing_names( + "DNS resolver engines (Unbound, etc.)", "DNS-Engines (Ungebunden, usw.)" + ) + + self.assertEqual(lost, {"Unbound"}) + + def test_a_kept_name_is_not_reported(self) -> None: + lost = missing_names( + "DNS resolver engines (Unbound, etc.)", "DNS-Engines (Unbound, usw.)" + ) + + self.assertEqual(lost, set()) + + def test_a_german_compound_still_carries_the_name(self) -> None: + lost = missing_names( + "one Redis user per consumer", "ein Redis-Benutzer je Nutzer" + ) + + self.assertEqual(lost, set()) + + def test_a_capital_the_source_did_not_have_is_not_a_loss(self) -> None: + lost = missing_names("bundling shell extensions", "bündelt Shell-Erweiterungen") + + self.assertEqual(lost, set()) + + def test_a_message_of_only_names_holds_no_words(self) -> None: + self.assertFalse(has_words("web-app-keycloak")) + + def test_prose_still_holds_words(self) -> None: + self.assertTrue(has_words("Deploy it behind the proxy")) + + def test_spans_are_offsets_into_the_text(self) -> None: + text = "Deploy web-app-keycloak now" + start, end = spans(text)[0] + + self.assertEqual(text[start:end], "web-app-keycloak") + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/i18n/names.py b/utils/i18n/names.py new file mode 100644 index 0000000000..b65eff8a45 --- /dev/null +++ b/utils/i18n/names.py @@ -0,0 +1,127 @@ +"""The role names and product titles a translation must hand back unchanged. + +A role directory name is never an English phrase, so it is protected wherever it +appears. A title is a single word often enough that the two readings collide: +``Unbound`` is a DNS resolver and an adjective, ``Shell`` a role and a common +noun. Measured over the catalogs, the capital separates them: of 47932 +capitalised occurrences the translator altered 22 percent, among them +``(Unbound, etc.)`` turning into ``(Ungebunden, etc.)``, while the 40698 +lowercase ones it altered in 40 percent of cases and those readings were the +common noun the text meant (``shell extensions``, ``user-driven documentation``). +""" + +from __future__ import annotations + +import re +from functools import lru_cache +from pathlib import Path + +import yaml + +from utils.cache.files import PROJECT_ROOT +from utils.roles.mapping import ROLE_FILE_META_MAIN + +ROLES_DIR = "roles" +README = "README.md" +SENTENCE_END = re.compile(r"[.!?:;\n]\s*[*_`\"'(\[]*\s*$") +OPENERS = " *_`\"'([" + + +@lru_cache(maxsize=1) +def _sources(root: str) -> tuple[tuple[str, ...], tuple[str, ...]]: + """Return the role directory names and the titles they declare. + + Args: + root: repository root, as a string so the cache key stays hashable. + """ + roles, titles = [], set() + for path in sorted(Path(root, ROLES_DIR).iterdir()): + if not path.is_dir(): + continue + roles.append(path.name) + readme = path / README + if readme.is_file(): + for line in readme.read_text(encoding="utf-8").splitlines(): + if line.startswith("# "): + titles.add(line[2:].strip()) + break + meta = path / ROLE_FILE_META_MAIN + if meta.is_file(): + declared = yaml.safe_load(meta.read_text(encoding="utf-8")) or {} + name = (declared.get("galaxy_info") or {}).get("name") + if name: + titles.add(str(name).strip()) + return tuple(roles), tuple(titles) + + +def _alternation(names) -> str: + """Return a regex matching any of ``names``, longest first. + + The boundaries end at a word character, not at a hyphen: German compounds a + name with the noun it qualifies, so ``Redis-Benutzer`` and ``API-URL`` carry + the name intact and must not read as a loss. Matching the longest name first + keeps ``web-app-keycloak`` from being served by a shorter sibling. + + Args: + names: the literals to match. + """ + ordered = sorted(names, key=len, reverse=True) + return r"(? tuple[re.Pattern | None, re.Pattern | None]: + """Return the unconditional and the capitalisation-gated name patterns. + + Args: + root: repository root. + + Returns: + The pattern for hyphenated role names, which hold wherever they appear, + and the one for titles, which holds only on a capitalised occurrence. + """ + roles, titles = _sources(root) + hyphenated = [name for name in roles if "-" in name] + return ( + re.compile(_alternation(hyphenated)) if hyphenated else None, + re.compile(_alternation(titles)) if titles else None, + ) + + +def reads_as_a_name(text: str, start: int) -> bool: + """Return whether the title at ``start`` is a product name, not a noun. + + The capital carries the distinction only away from a sentence start, where + every word is capitalised anyway: ``(Unbound, etc.)`` names a DNS resolver, + while the heading ``Documentation`` is the common noun the translator is + meant to render. + + Args: + text: the message the occurrence sits in. + start: offset of the occurrence. + """ + if not text[start].isupper(): + return False + before = text[:start].rstrip(OPENERS) + return bool(before) and not SENTENCE_END.search(before) + + +def spans(text: str) -> list[tuple[int, int]]: + """Return the offsets of every name ``text`` carries. + + Args: + text: a source message. + """ + role_pattern, title_pattern = patterns() + found = [] + if role_pattern: + found += [(m.start(), m.end()) for m in role_pattern.finditer(text)] + if title_pattern: + found += [ + (m.start(), m.end()) + for m in title_pattern.finditer(text) + if reads_as_a_name(text, m.start()) + ] + return found diff --git a/utils/i18n/placeholders.py b/utils/i18n/placeholders.py index cdee785a3a..95608f1cd1 100644 --- a/utils/i18n/placeholders.py +++ b/utils/i18n/placeholders.py @@ -7,6 +7,8 @@ import re from collections import Counter from dataclasses import dataclass +from utils.i18n import names + PRINTF = r"%\([A-Za-z_]\w*\)[sdif]|%[sdif]" PLACEHOLDER = re.compile(PRINTF) @@ -42,6 +44,37 @@ def carries_placeholder(message_id: str | tuple[str, ...]) -> bool: return any(PLACEHOLDER.search(form) for form in forms) +EXTRA = re.compile( + r"[\U0001F300-\U0001FAFF☀-➿️]" + r"|[\w.+-]+@[\w-]+\.[\w.]+" +) + + +def matches(text: str, with_names: bool = True) -> list[tuple[int, int]]: + """Return the non-overlapping offsets every protection rule claims. + + Args: + text: a source message or its translation. + with_names: whether role names and titles take part. They do when a + request is masked, so the translator never sees them, and they do + not when two texts are compared: German capitalises every noun, so + the lowercase ``shell extensions`` becomes ``Shell-Erweiterungen`` + and a set comparison would read the capital as a surplus span. + """ + found = [(m.start(), m.end()) for m in PROTECTED.finditer(text)] + found += [(m.start(), m.end()) for m in EXTRA.finditer(text)] + if with_names: + found += names.spans(text) + kept: list[tuple[int, int]] = [] + reach = 0 + for start, end in sorted(found): + if start < reach: + continue + kept.append((start, end)) + reach = end + return kept + + MARKUP = '[]`*{}()"' EMPHASIS = re.compile(r"(\*\*)[ \t]*([^\s]|[^\s].*?[^\s])[ \t]*\1", re.DOTALL) @@ -100,7 +133,18 @@ def protected_spans(text: str) -> Counter: Args: text: a source message or its translation. """ - return Counter(PROTECTED.findall(text)) + return Counter(text[start:end] for start, end in matches(text, with_names=False)) + + +def missing_names(source: str, translation: str) -> set[str]: + """Return the names ``source`` carries that ``translation`` dropped. + + Args: + source: the source message. + translation: what came back for it. + """ + carried = {source[start:end] for start, end in names.spans(source)} + return {name for name in carried if name not in translation} def has_words(text: str) -> bool: @@ -109,7 +153,12 @@ def has_words(text: str) -> bool: Args: text: a source message. """ - return any(character.isalpha() for character in PROTECTED.sub("", text)) + plain, position = [], 0 + for start, end in matches(text): + plain.append(text[position:start]) + position = end + plain.append(text[position:]) + return any(character.isalpha() for character in "".join(plain)) def mask(text: str) -> Masked: @@ -121,11 +170,11 @@ def mask(text: str) -> Masked: spans: list[str] = [] parts: list[str] = [] position = 0 - for match in PROTECTED.finditer(text): - parts.append(html.escape(text[position : match.start()], quote=False)) + for start, end in matches(text): + parts.append(html.escape(text[position:start], quote=False)) parts.append(f'') - spans.append(match.group(0)) - position = match.end() + spans.append(text[start:end]) + position = end parts.append(html.escape(text[position:], quote=False)) return Masked("".join(parts), tuple(spans)) diff --git a/utils/i18n/translate.py b/utils/i18n/translate.py index fc86009350..ff4cc56e91 100644 --- a/utils/i18n/translate.py +++ b/utils/i18n/translate.py @@ -6,7 +6,7 @@ from collections import Counter from typing import TYPE_CHECKING from utils.i18n.catalog import MACHINE_TRANSLATION -from utils.i18n.placeholders import protected_spans +from utils.i18n.placeholders import missing_names, protected_spans if TYPE_CHECKING: from babel.messages.catalog import Catalog, Message @@ -58,6 +58,7 @@ def damaged(catalog: Catalog) -> list[Message]: and message.string and ( protected_spans(str(message.string)) != protected_spans(message.id) + or missing_names(message.id, str(message.string)) or structure(str(message.string)) != structure(message.id) ) ]