diff --git a/README.md b/README.md index 8102533..fe29dca 100644 --- a/README.md +++ b/README.md @@ -171,10 +171,25 @@ make i18n This fills the interface strings of the languages that ship no catalogue as well. Existing entries are never overwritten, and a string the shipped catalogue already covers is never requested, so corrections you make by hand -survive later runs. Only prose (`description`, `text`, `warning`, `info`, `subtitel`) is -filled automatically; `name` and `title` are left to you, because a machine -cannot tell the menu label "Pictures" from the brand "Mastodon". Write those -into the content catalogue yourself when you want them translated. +survive later runs. + +`name`, `title`, `description`, `text`, `warning`, `info` and `subtitel` are +translated; `url`, `link_text`, `identifier` and icon classes never are. + +A machine cannot tell the menu label "Pictures" from the brand "Mastodon", so +list the brands in `app/i18n/keep.txt`, one per line — they are then stored as +themselves in every language and cost no request: + +``` +# Strings utils/i18n_sync.py stores as themselves instead of translating. +Mastodon +Nextcloud +freelancermap.de +``` + +Add one-off entries with `--keep Foo Bar`, or point somewhere else with +`--keep-file`. A protected string never replaces an entry you already wrote by +hand. --- diff --git a/app/i18n/keep.txt b/app/i18n/keep.txt new file mode 100644 index 0000000..de541db --- /dev/null +++ b/app/i18n/keep.txt @@ -0,0 +1,46 @@ +# Strings utils/i18n_sync.py stores as themselves instead of translating. +# One per line; blank lines and lines starting with # are ignored. + +Baserow +Big Blue Button +Bluesky +Buy me a Coffee +Discourse +Duolingo +Eversports +Facebook +Friendica +Garmin +Gitea +GitHub +GitHub Sponsors +Infinito.Nexus +Instagram +Keycloak +LDAP +LinkedIn +Mailu +Mastodon +Matomo +Matrix +Nextcloud +Open Project +Patreon +PayPal +Peertube +Pixelfed +Signal +Snipe IT +Spotify +Taiga +Telegram +Twitter +WhatsApp +XING +YouTube +Yourls +Zoom +freelancermap.de +malt +phpMyAdmin +upwork.com diff --git a/app/utils/i18n.py b/app/utils/i18n.py index a0f627a..65a19ad 100644 --- a/app/utils/i18n.py +++ b/app/utils/i18n.py @@ -23,9 +23,9 @@ CONTENT_DIR = I18N_DIR / "content" SOURCE_LANGUAGE = "en" -AUTOFILL_KEYS = frozenset({"description", "text", "warning", "info", "subtitel"}) - -TRANSLATABLE_KEYS = AUTOFILL_KEYS | frozenset({"name", "title"}) +TRANSLATABLE_KEYS = frozenset( + {"description", "info", "name", "subtitel", "text", "title", "warning"} +) UI_STRINGS = ( "Alternatives", diff --git a/tests/unit/test_i18n_sync.py b/tests/unit/test_i18n_sync.py index f4d9f8a..023fe5b 100644 --- a/tests/unit/test_i18n_sync.py +++ b/tests/unit/test_i18n_sync.py @@ -30,19 +30,26 @@ class TestCollectSources(unittest.TestCase): self.assertEqual( i18n_sync.collect_sources(config), { + "Agile Coach", "I lead transformations.", "Another card.", + "Apps", "Application menu", "A tagline", }, ) - def test_label_and_structural_keys_are_left_alone(self): + def test_labels_are_collected_too(self): + config = {"name": "Pictures", "title": "Agile Coach"} + + self.assertEqual(i18n_sync.collect_sources(config), {"Pictures", "Agile Coach"}) + + def test_structural_keys_are_left_alone(self): config = { - "name": "Mastodon", - "title": "Cybermaster", "url": "https://example.test", "link_text": "www.example.test", + "identifier": "@someone@example.test", + "icon": {"class": "fa-solid fa-users"}, } self.assertEqual(i18n_sync.collect_sources(config), set()) @@ -178,6 +185,80 @@ class TestSupportedLanguages(unittest.TestCase): i18n_sync.supported_languages("http://lt", self._session(payload)) +class TestReadKeep(unittest.TestCase): + def setUp(self): + self.directory = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.directory, True) + self.path = self.directory / "keep.txt" + + def test_a_missing_file_keeps_nothing(self): + self.assertEqual(i18n_sync.read_keep(self.path), []) + + def test_comments_and_blank_lines_are_ignored(self): + self.path.write_text( + "# brands\n\nMastodon\n Bluesky \n\n# more\nNextcloud\n", + encoding="utf-8", + ) + + self.assertEqual( + i18n_sync.read_keep(self.path), ["Mastodon", "Bluesky", "Nextcloud"] + ) + + def test_the_shipped_list_names_the_brands_of_the_sample_configuration(self): + shipped = i18n_sync.read_keep(i18n_sync.DEFAULT_KEEP_PATH) + + self.assertGreaterEqual(set(shipped), {"Mastodon", "Nextcloud", "Matrix"}) + self.assertNotIn("Pictures", shipped) + self.assertNotIn("Imprint", shipped) + + +class TestCommandLine(unittest.TestCase): + def setUp(self): + self.directory = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.directory, True) + self.keep_file = self.directory / "keep.txt" + self.config = self.directory / "config.yaml" + self.config.write_text("cards: []\n", encoding="utf-8") + + def _main(self, *extra): + with patch.object(i18n_sync, "sync") as sync: + i18n_sync.main( + [ + "--url", + "http://lt/", + "--config", + str(self.config), + "--keep-file", + str(self.keep_file), + *extra, + ] + ) + return sync.call_args + + def test_the_keep_file_reaches_the_sync(self): + self.keep_file.write_text("# brands\nMastodon\nNextcloud\n", encoding="utf-8") + + keep = self._main().args[5] + + self.assertEqual(list(keep), ["Mastodon", "Nextcloud"]) + + def test_the_command_line_adds_to_the_keep_file(self): + self.keep_file.write_text("Mastodon\n", encoding="utf-8") + + keep = self._main("--keep", "Taiga").args[5] + + self.assertEqual(list(keep), ["Mastodon", "Taiga"]) + + def test_a_trailing_slash_is_stripped_from_the_url(self): + self.assertEqual(self._main().args[0], "http://lt") + + def test_a_backend_failure_is_reported_as_an_exit_code(self): + with patch.object(i18n_sync, "sync", side_effect=i18n_sync.BackendError("no")): + self.assertEqual( + i18n_sync.main(["--url", "http://lt", "--config", str(self.config)]), 1 + ) + + class TestLoadExisting(unittest.TestCase): def setUp(self): self.directory = Path(tempfile.mkdtemp()) @@ -256,7 +337,7 @@ class TestSync(unittest.TestCase): i18n.CONTENT_DIR = self.directory self.path = self.directory / "de.yaml" - def _run(self, sources, offered=("de",), translated="UEBERSETZT"): + def _run(self, sources, offered=("de",), translated="UEBERSETZT", keep=()): listing = Mock(raise_for_status=Mock()) listing.json.return_value = [{"code": code} for code in offered] answer = Mock(ok=True) @@ -264,7 +345,7 @@ class TestSync(unittest.TestCase): session = Mock(get=Mock(return_value=listing), post=Mock(return_value=answer)) with patch.object(i18n_sync.requests, "Session", return_value=session): - i18n_sync.sync("http://lt", "", set(sources), ["de"], self.directory) + i18n_sync.sync("http://lt", "", set(sources), ["de"], self.directory, keep) return session def test_missing_entries_are_written(self): @@ -299,6 +380,22 @@ class TestSync(unittest.TestCase): self.assertFalse(self.path.exists()) session.post.assert_not_called() + def test_a_protected_string_is_stored_as_itself(self): + session = self._run(["Mastodon", "Hello"], keep=["Mastodon"]) + + catalog = yaml.safe_load(self.path.read_text(encoding="utf-8")) + self.assertEqual(catalog["Mastodon"], "Mastodon") + self.assertEqual(session.post.call_count, 1) + + def test_a_protected_string_never_overwrites_an_existing_entry(self): + self.path.write_text("Mastodon: HAND-EDITED\n", encoding="utf-8") + + self._run(["Mastodon", "Hello"], keep=["Mastodon"]) + + catalog = yaml.safe_load(self.path.read_text(encoding="utf-8")) + self.assertEqual(catalog["Mastodon"], "HAND-EDITED") + self.assertEqual(catalog["Hello"], "UEBERSETZT") + def test_an_empty_translation_is_not_stored(self): self._run(["Hello"], translated="") diff --git a/utils/i18n_sync.py b/utils/i18n_sync.py index 1f8712d..62dda58 100644 --- a/utils/i18n_sync.py +++ b/utils/i18n_sync.py @@ -20,6 +20,7 @@ sys.path.insert(0, str(REPO_ROOT)) from app.utils import i18n # noqa: E402 DEFAULT_CONFIG_PATH = REPO_ROOT / "app" / "config.yaml" +DEFAULT_KEEP_PATH = i18n.I18N_DIR / "keep.txt" REQUEST_TIMEOUT = 30 @@ -38,11 +39,27 @@ def collect_sources(node, key=None, found=None): elif isinstance(node, list): for item in node: collect_sources(item, key, found) - elif isinstance(node, str) and key in i18n.AUTOFILL_KEYS and node.strip(): + elif isinstance(node, str) and key in i18n.TRANSLATABLE_KEYS and node.strip(): found.add(node) return found +def read_keep(path): + """Return the strings listed in ``path``, ignoring blanks and comments. + + Args: + path: a text file with one string per line, or a path that does not exist. + """ + if not path.is_file(): + return [] + lines = path.read_text(encoding="utf-8").splitlines() + return [ + stripped + for stripped in (line.strip() for line in lines) + if stripped and not stripped.startswith("#") + ] + + class BackendError(Exception): """The translation backend answered in a way the run cannot continue from.""" @@ -144,7 +161,7 @@ def translate(session, url, api_key, text, target): return translated -def sync(url, api_key, sources, targets, directory): +def sync(url, api_key, sources, targets, directory, keep=()): """Fill and write the catalogue of every language in ``targets``. Args: @@ -153,6 +170,7 @@ def sync(url, api_key, sources, targets, directory): sources: English strings to translate. targets: language codes to fill. directory: catalogue directory to write into. + keep: strings to store as themselves instead of translating. """ directory.mkdir(parents=True, exist_ok=True) session = requests.Session() @@ -168,6 +186,9 @@ def sync(url, api_key, sources, targets, directory): if catalog is None: continue + protected = {word: word for word in keep if word not in catalog} + catalog.update(protected) + shipped = ( {} if directory == i18n.UI_DIR @@ -190,7 +211,7 @@ def sync(url, api_key, sources, targets, directory): catalog[source] = translated added += 1 - if not added: + if not added and not protected: print(f" ! {target}: nothing translated, leaving the file untouched") continue @@ -227,6 +248,24 @@ def main(argv=None): "ui: the interface strings that ship with the project." ), ) + parser.add_argument( + "--keep", + nargs="+", + default=[], + metavar="STRING", + help=( + "Store these as themselves instead of translating, " + "in addition to --keep-file." + ), + ) + parser.add_argument( + "--keep-file", + type=Path, + default=DEFAULT_KEEP_PATH, + help=( + f"One string per line, stored untranslated (default: {DEFAULT_KEEP_PATH})." + ), + ) parser.add_argument( "--languages", nargs="+", @@ -245,8 +284,18 @@ def main(argv=None): sources = collect_sources(config) | set(i18n.UI_STRINGS) print(f"{len(sources)} string(s) in {args.config} and the interface") + keep = read_keep(args.keep_file) + args.keep + print(f"{len(keep)} string(s) kept untranslated") + try: - sync(args.url.rstrip("/"), args.api_key, sources, args.languages, directory) + sync( + args.url.rstrip("/"), + args.api_key, + sources, + args.languages, + directory, + keep, + ) except BackendError as error: print(f"ERROR: {error}", file=sys.stderr) return 1