diff --git a/plugins/lookup/current_play_redirect_domains.py b/plugins/lookup/current_play_redirect_domains.py index e4317a0297..73d46715d7 100644 --- a/plugins/lookup/current_play_redirect_domains.py +++ b/plugins/lookup/current_play_redirect_domains.py @@ -125,9 +125,10 @@ class LookupModule(LookupBase): DOMAIN_PRIMARY even when no app claims it. On onion nodes (``svc-net-tor`` in ``group_names`` with a provisioned ``services.tor.node``) the bare node-onion apex is scheduled as an - additional source targeting the same ``DOMAIN_HOMEPAGE``, which is - already family-correct there (onion-injected domain lookup); both - entries require ``web-opt-rdr-domains`` in + additional source targeting the same ``DOMAIN_HOMEPAGE`` when that + is an onion host; a clearnet ``DOMAIN_HOMEPAGE`` (no dashboard + deployed) gets no onion apex entry, so an onion vhost never + redirects to clearnet. Both entries require ``web-opt-rdr-domains`` in ``lookup('deployment').deployed`` so isolated CI variants do not schedule a redirect whose target is not served. 3. per-app alias-to-canonical mappings from @@ -209,7 +210,11 @@ class LookupModule(LookupBase): primary_redirects.append( {"source": domain_primary, "target": domain_homepage} ) - if node_onion and node_onion != domain_homepage: + if ( + node_onion + and node_onion != domain_homepage + and domain_homepage.endswith(".onion") + ): primary_redirects.append( {"source": node_onion, "target": domain_homepage} ) diff --git a/roles/web-app-bridgy-fed/meta/services.yml b/roles/web-app-bridgy-fed/meta/services.yml index 3a8634227f..56a6175d16 100644 --- a/roles/web-app-bridgy-fed/meta/services.yml +++ b/roles/web-app-bridgy-fed/meta/services.yml @@ -49,7 +49,8 @@ bridgy-fed: swarm: enabled: true image: python - version: 3.15-slim + # nocheck: docker-version upstream's requirements.pinned.txt pins cbor2 6.1.4, which ships wheels up to cp314 and needs Rust >= 1.88 to build from source. TODO: return to 3.15-slim once the pinned upstream ref carries cbor2 >= 6.1.5 (cp315 wheels) + version: 3.14-slim name: web-app-bridgy-fed repository: https://github.com/snarfed/bridgy-fed.git # nocheck: repository-version -- upstream publishes no tags, so the pin is a commit diff --git a/roles/web-app-n8n/meta/services.yml b/roles/web-app-n8n/meta/services.yml index 5eb22f175a..8797f41581 100644 --- a/roles/web-app-n8n/meta/services.yml +++ b/roles/web-app-n8n/meta/services.yml @@ -73,7 +73,7 @@ n8n: backup: no_stop_required: false image: docker.n8n.io/n8nio/n8n - version: "2.44.1" + version: "2.44.2" update: type: registry_tags image: docker.n8n.io/n8nio/n8n diff --git a/tests/unit/python/plugins/lookup/test_current_play_redirect_domains.py b/tests/unit/python/plugins/lookup/test_current_play_redirect_domains.py index 76e92629e9..4ca9efbe04 100644 --- a/tests/unit/python/plugins/lookup/test_current_play_redirect_domains.py +++ b/tests/unit/python/plugins/lookup/test_current_play_redirect_domains.py @@ -334,6 +334,21 @@ class CurrentPlayRedirectDomainsLookupTests(unittest.TestCase): [{"source": "infinito.test", "target": "abcdefonion.onion"}], ) + def test_onion_apex_absent_when_homepage_is_clearnet(self): + lm = self._make_lookup() + variables = self._onion_variables( + DOMAIN_HOMEPAGE="infinito.nexus", + group_names=["svc-net-tor"], + ) + with self._patch_lookups( + deployed=["web-opt-rdr-domains"], current_play_apps={} + ): + result = lm.run(terms=[], variables=variables)[0] + self.assertEqual( + result, + [{"source": "infinito.test", "target": "infinito.nexus"}], + ) + def test_onion_apex_node_value_templated(self): class _ResolvingTemplar: def __init__(self, mapping): diff --git a/tests/unit/python/utils/install/lint/test_mermaid_cli.py b/tests/unit/python/utils/install/lint/test_mermaid_cli.py index 600879cf61..5b66791177 100644 --- a/tests/unit/python/utils/install/lint/test_mermaid_cli.py +++ b/tests/unit/python/utils/install/lint/test_mermaid_cli.py @@ -2,20 +2,97 @@ from __future__ import annotations +import contextlib import os import subprocess import unittest import unittest.mock as mock from pathlib import Path from tempfile import TemporaryDirectory +from typing import TYPE_CHECKING from utils.install.lint import mermaid_cli +if TYPE_CHECKING: + from collections.abc import Iterator + +REVISION = "155.0.8059.39" + def _done(returncode: int) -> subprocess.CompletedProcess[str]: return subprocess.CompletedProcess([], returncode, stdout="out\n", stderr="err\n") +def _write_core(node_modules: Path, revision: str) -> None: + revisions = node_modules / "puppeteer-core" / "lib" / "puppeteer" + revisions.mkdir(parents=True) + (revisions / "revisions.js").write_text( + f"export const PUPPETEER_REVISIONS = Object.freeze({{\n" + f" chrome: '155.0.8059.39',\n" + f" 'chrome-headless-shell': '{revision}',\n" + f"}});\n", + encoding="utf-8", + ) + + +def _fake_mmdc( + root: Path, *, revision: str | None = REVISION, nested: str | None = None +) -> Path: + """Lay out a global mermaid-cli install the way npm does. + + Args: + root: npm prefix to build the install under. + revision: chrome-headless-shell revision of the puppeteer-core beside + puppeteer, or None to install mmdc without a puppeteer next to it. + nested: revision of a puppeteer-core nested under puppeteer, as npm + lays it out when mermaid-cli pins another puppeteer-core. + + Returns: + The ``bin/mmdc`` symlink. + """ + package = root / "lib" / "node_modules" / "@mermaid-js" / "mermaid-cli" + (package / "src").mkdir(parents=True) + (package / "src" / "cli.js").write_text("", encoding="utf-8") + if revision is not None: + (package / "node_modules" / "puppeteer").mkdir(parents=True) + _write_core(package / "node_modules", revision) + if nested is not None: + _write_core(package / "node_modules" / "puppeteer" / "node_modules", nested) + (root / "bin").mkdir() + mmdc = root / "bin" / "mmdc" + mmdc.symlink_to(package / "src" / "cli.js") + return mmdc + + +def _install_browser(cache: Path, revision: str) -> None: + build = cache / "chrome-headless-shell" / f"linux-{revision}" / "linux64" + build.mkdir(parents=True) + (build / "chrome-headless-shell").write_text("", encoding="utf-8") + + +@contextlib.contextmanager +def _environment( + *, revision: str | None = REVISION, nested: str | None = None +) -> Iterator[tuple[Path, Path]]: + """Patch mmdc's location and the puppeteer cache onto temporary dirs. + + Args: + revision: passed to :func:`_fake_mmdc`. + nested: passed to :func:`_fake_mmdc`. + + Yields: + The npm prefix and the puppeteer cache directory. + """ + with TemporaryDirectory() as prefix, TemporaryDirectory() as cache: + mmdc = _fake_mmdc(Path(prefix), revision=revision, nested=nested) + with ( + mock.patch.dict(os.environ, {"PUPPETEER_CACHE_DIR": cache}), + mock.patch.object(mermaid_cli, "which", return_value=True), + mock.patch.object(mermaid_cli.shutil, "which", return_value=str(mmdc)), + ): + yield Path(prefix), Path(cache) + + class TestBrowserProvisioning(unittest.TestCase): def provision(self, *, euid: int, results: list[int]) -> tuple[list, list[str]]: """Run ``ensure()`` with mmdc present and a stubbed puppeteer. @@ -29,9 +106,7 @@ class TestBrowserProvisioning(unittest.TestCase): """ warnings: list[str] = [] with ( - TemporaryDirectory() as cache, - mock.patch.dict(os.environ, {"PUPPETEER_CACHE_DIR": cache}), - mock.patch.object(mermaid_cli, "which", return_value=True), + _environment(), mock.patch.object(mermaid_cli.os, "geteuid", return_value=euid), mock.patch.object(mermaid_cli, "warn", side_effect=warnings.append), mock.patch.object( @@ -51,6 +126,14 @@ class TestBrowserProvisioning(unittest.TestCase): ) self.assertEqual(warnings, []) + def test_the_browser_comes_from_the_puppeteer_next_to_mmdc(self) -> None: + calls, _ = self.provision(euid=0, results=[0]) + self.assertTrue( + calls[0] + .args[0][0] + .endswith("@mermaid-js/mermaid-cli/node_modules/.bin/puppeteer") + ) + def test_a_user_installs_the_browser_without_libraries(self) -> None: calls, _ = self.provision(euid=1000, results=[0]) self.assertNotIn("--install-deps", calls[0].args[0]) @@ -60,23 +143,21 @@ class TestBrowserProvisioning(unittest.TestCase): calls, _ = self.provision(euid=0, results=[0]) self.assertNotIn("PUPPETEER_SKIP_DOWNLOAD", calls[0].kwargs["env"]) - def test_a_failed_install_clears_the_browser_cache_and_retries_once(self) -> None: - with TemporaryDirectory() as cache: - folder = Path(cache) / "chrome-headless-shell" / "linux-1" - folder.mkdir(parents=True) - seen: list[bool] = [] + def test_a_failed_install_clears_its_revision_and_retries_once(self) -> None: + seen: list[tuple[bool, bool]] = [] + with _environment() as (_, cache): + partial = cache / "chrome-headless-shell" / f"linux-{REVISION}" + partial.mkdir(parents=True) + other = cache / "chrome-headless-shell" / "linux-155.0.8059.79" + other.mkdir(parents=True) def run(*_args, **_kwargs): - seen.append(folder.exists()) + seen.append((partial.exists(), other.exists())) return _done(1 if len(seen) == 1 else 0) - with ( - mock.patch.dict(os.environ, {"PUPPETEER_CACHE_DIR": cache}), - mock.patch.object(mermaid_cli, "which", return_value=True), - mock.patch.object(mermaid_cli.subprocess, "run", side_effect=run), - ): + with mock.patch.object(mermaid_cli.subprocess, "run", side_effect=run): mermaid_cli.ensure() - self.assertEqual(seen, [True, False]) + self.assertEqual(seen, [(True, True), (False, True)]) def test_a_second_failure_warns_with_puppeteers_output(self) -> None: calls, warnings = self.provision(euid=0, results=[1, 1]) @@ -84,28 +165,39 @@ class TestBrowserProvisioning(unittest.TestCase): self.assertEqual(len(warnings), 1) self.assertIn("out\nerr\n", warnings[0]) - def test_an_installed_browser_is_not_provisioned_again(self) -> None: - with TemporaryDirectory() as cache: - build = Path(cache) / "chrome-headless-shell" / "linux-1" / "linux64" - build.mkdir(parents=True) - (build / "chrome-headless-shell").write_text("", encoding="utf-8") - with ( - mock.patch.dict(os.environ, {"PUPPETEER_CACHE_DIR": cache}), - mock.patch.object(mermaid_cli, "which", return_value=True), - mock.patch.object(mermaid_cli.subprocess, "run") as run, - ): + def test_the_pinned_browser_is_not_provisioned_again(self) -> None: + with _environment() as (_, cache): + _install_browser(cache, REVISION) + with mock.patch.object(mermaid_cli.subprocess, "run") as run: mermaid_cli.ensure() run.assert_not_called() + def test_a_browser_of_another_revision_is_replaced(self) -> None: + with _environment() as (_, cache): + _install_browser(cache, "155.0.8059.79") + with mock.patch.object( + mermaid_cli.subprocess, "run", return_value=_done(0) + ) as run: + mermaid_cli.ensure() + run.assert_called_once() + + def test_mmdc_without_its_own_puppeteer_warns(self) -> None: + with ( + _environment(revision=None), + mock.patch.object(mermaid_cli, "warn") as warn, + mock.patch.object(mermaid_cli.subprocess, "run") as run, + ): + mermaid_cli.ensure() + run.assert_not_called() + warn.assert_called_once() + def test_a_timeout_warns_instead_of_raising(self) -> None: with ( - TemporaryDirectory() as cache, - mock.patch.dict(os.environ, {"PUPPETEER_CACHE_DIR": cache}), - mock.patch.object(mermaid_cli, "which", return_value=True), + _environment(), mock.patch.object( mermaid_cli.subprocess, "run", - side_effect=subprocess.TimeoutExpired("npx", 900), + side_effect=subprocess.TimeoutExpired("puppeteer", 900), ), mock.patch.object(mermaid_cli, "warn") as warn, ): @@ -114,14 +206,22 @@ class TestBrowserProvisioning(unittest.TestCase): class TestBrowserInstalled(unittest.TestCase): - def test_the_executable_in_the_puppeteer_cache_counts(self) -> None: - with TemporaryDirectory() as cache: - build = Path(cache) / "chrome-headless-shell" / "linux-1" / "linux64" - build.mkdir(parents=True) - with mock.patch.dict(os.environ, {"PUPPETEER_CACHE_DIR": cache}): - self.assertFalse(mermaid_cli.browser_installed()) - (build / "chrome-headless-shell").write_text("", encoding="utf-8") - self.assertTrue(mermaid_cli.browser_installed()) + def test_only_the_pinned_revision_counts(self) -> None: + with _environment() as (_, cache): + _install_browser(cache, "155.0.8059.79") + self.assertFalse(mermaid_cli.browser_installed()) + _install_browser(cache, REVISION) + self.assertTrue(mermaid_cli.browser_installed()) + + def test_the_revision_is_read_from_mmdcs_puppeteer_core(self) -> None: + with _environment(): + self.assertEqual(mermaid_cli.browser_revision(), REVISION) + + def test_a_puppeteer_core_nested_under_puppeteer_wins(self) -> None: + with _environment(revision="155.0.8059.79", nested=REVISION) as (_, cache): + self.assertEqual(mermaid_cli.browser_revision(), REVISION) + _install_browser(cache, REVISION) + self.assertTrue(mermaid_cli.browser_installed()) if __name__ == "__main__": diff --git a/utils/install/lint/mermaid_cli.py b/utils/install/lint/mermaid_cli.py index e60a96371a..f1782e0d3a 100644 --- a/utils/install/lint/mermaid_cli.py +++ b/utils/install/lint/mermaid_cli.py @@ -1,7 +1,8 @@ """Install @mermaid-js/mermaid-cli (mmdc) and the chrome-headless-shell it renders with. mmdc comes from npm with ``PUPPETEER_SKIP_DOWNLOAD`` set; the browser is -provisioned through puppeteer afterwards, as root with its shared libraries +provisioned afterwards through the puppeteer that npm installed next to mmdc, +at the revision its puppeteer-core pins, as root with its shared libraries (``--install-deps``). ``unzip`` is ensured first because puppeteer extracts the browser with it. A failed install warns instead of aborting ``install-lint``, so sandboxes without npm or browser access still get the rest of the lint @@ -12,6 +13,7 @@ from __future__ import annotations import contextlib import os +import re import shutil import subprocess from pathlib import Path @@ -36,12 +38,59 @@ def browser_cache() -> Path: return Path(base) / BROWSER +def _mmdc_puppeteer() -> Path | None: + mmdc = shutil.which("mmdc") + if mmdc is None: + return None + for parent in Path(mmdc).resolve().parents: + puppeteer = parent / "node_modules" / "puppeteer" + if puppeteer.is_dir(): + return puppeteer + return None + + +def _puppeteer_core(puppeteer: Path) -> Path | None: + for base in (puppeteer, *puppeteer.parents): + if base.name == "node_modules": + continue + core = base / "node_modules" / "puppeteer-core" + if core.is_dir(): + return core + return None + + +def browser_revision() -> str | None: + puppeteer = _mmdc_puppeteer() + if puppeteer is None: + return None + core = _puppeteer_core(puppeteer) + if core is None: + return None + revisions = core / "lib" / "puppeteer" / "revisions.js" + try: + text = revisions.read_text(encoding="utf-8") # nocheck: cache-read npm prefix + except OSError: + return None + match = re.search(rf"'{BROWSER}':\s*'([^']+)'", text) + return match.group(1) if match else None + + def browser_installed() -> bool: - return any(path.is_file() for path in browser_cache().glob(f"*/*/{BROWSER}")) + revision = browser_revision() + if revision is None: + return False + return any( + path.is_file() for path in browser_cache().glob(f"*-{revision}/*/{BROWSER}") + ) -def _provision_browser() -> subprocess.CompletedProcess[str]: - argv = ["npx", "--yes", "puppeteer", "browsers", "install", BROWSER] +def _provision_browser(puppeteer: Path) -> subprocess.CompletedProcess[str]: + argv = [ + str(puppeteer.parent / ".bin" / "puppeteer"), + "browsers", + "install", + BROWSER, + ] if os.geteuid() == 0: argv.append("--install-deps") env = {k: v for k, v in os.environ.items() if k != "PUPPETEER_SKIP_DOWNLOAD"} @@ -58,12 +107,19 @@ def _provision_browser() -> subprocess.CompletedProcess[str]: def _ensure_browser() -> None: if browser_installed(): return - log(f"Provisioning {BROWSER} for mmdc via puppeteer.") + puppeteer = _mmdc_puppeteer() + if puppeteer is None: + warn(f"{BROWSER} provisioning skipped: no puppeteer next to mmdc") + return + log(f"Provisioning {BROWSER} for mmdc via its own puppeteer.") try: - done = _provision_browser() + done = _provision_browser(puppeteer) if done.returncode != 0: - shutil.rmtree(browser_cache(), ignore_errors=True) - done = _provision_browser() + revision = browser_revision() + if revision is not None: + for partial in browser_cache().glob(f"*-{revision}"): + shutil.rmtree(partial, ignore_errors=True) + done = _provision_browser(puppeteer) except (OSError, subprocess.TimeoutExpired) as exc: warn(f"{BROWSER} provisioning skipped (optional lint tool): {exc}") return