Merge branch 'main' into feature/designer

plugins/lookup/current_play_redirect_domains.py

@@ -125,9 +125,10 @@ class LookupModule(LookupBase):
          DOMAIN_PRIMARY even when no app claims it. On onion nodes
          (``svc-net-tor`` in ``group_names`` with a provisioned
          ``services.tor.node``) the bare node-onion apex is scheduled as an
-         additional source targeting the same ``DOMAIN_HOMEPAGE``, which is
-         already family-correct there (onion-injected domain lookup); both
-         entries require ``web-opt-rdr-domains`` in
+         additional source targeting the same ``DOMAIN_HOMEPAGE`` when that
+         is an onion host; a clearnet ``DOMAIN_HOMEPAGE`` (no dashboard
+         deployed) gets no onion apex entry, so an onion vhost never
+         redirects to clearnet. Both entries require ``web-opt-rdr-domains`` in
          ``lookup('deployment').deployed`` so isolated CI variants do not
          schedule a redirect whose target is not served.
       3. per-app alias-to-canonical mappings from
@@ -209,7 +210,11 @@ class LookupModule(LookupBase):
                     primary_redirects.append(
                         {"source": domain_primary, "target": domain_homepage}
                     )
-                if node_onion and node_onion != domain_homepage:
+                if (
+                    node_onion
+                    and node_onion != domain_homepage
+                    and domain_homepage.endswith(".onion")
+                ):
                     primary_redirects.append(
                         {"source": node_onion, "target": domain_homepage}
                     )

roles/web-app-bridgy-fed/meta/services.yml

@@ -49,7 +49,8 @@ bridgy-fed:
     swarm:
       enabled: true
   image: python
-  version: 3.15-slim
+  # nocheck: docker-version  upstream's requirements.pinned.txt pins cbor2 6.1.4, which ships wheels up to cp314 and needs Rust >= 1.88 to build from source. TODO: return to 3.15-slim once the pinned upstream ref carries cbor2 >= 6.1.5 (cp315 wheels)
+  version: 3.14-slim
   name: web-app-bridgy-fed
   repository: https://github.com/snarfed/bridgy-fed.git
   # nocheck: repository-version -- upstream publishes no tags, so the pin is a commit

roles/web-app-n8n/meta/services.yml

@@ -73,7 +73,7 @@ n8n:
   backup:
     no_stop_required: false
   image: docker.n8n.io/n8nio/n8n
-  version: "2.44.1"
+  version: "2.44.2"
   update:
     type: registry_tags
     image: docker.n8n.io/n8nio/n8n

tests/unit/python/plugins/lookup/test_current_play_redirect_domains.py

@@ -334,6 +334,21 @@ class CurrentPlayRedirectDomainsLookupTests(unittest.TestCase):
             [{"source": "infinito.test", "target": "abcdefonion.onion"}],
         )
 
+    def test_onion_apex_absent_when_homepage_is_clearnet(self):
+        lm = self._make_lookup()
+        variables = self._onion_variables(
+            DOMAIN_HOMEPAGE="infinito.nexus",
+            group_names=["svc-net-tor"],
+        )
+        with self._patch_lookups(
+            deployed=["web-opt-rdr-domains"], current_play_apps={}
+        ):
+            result = lm.run(terms=[], variables=variables)[0]
+        self.assertEqual(
+            result,
+            [{"source": "infinito.test", "target": "infinito.nexus"}],
+        )
+
     def test_onion_apex_node_value_templated(self):
         class _ResolvingTemplar:
             def __init__(self, mapping):

tests/unit/python/utils/install/lint/test_mermaid_cli.py

@@ -2,20 +2,97 @@
 
 from __future__ import annotations
 
+import contextlib
 import os
 import subprocess
 import unittest
 import unittest.mock as mock
 from pathlib import Path
 from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING
 
 from utils.install.lint import mermaid_cli
 
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+REVISION = "155.0.8059.39"
+
 
 def _done(returncode: int) -> subprocess.CompletedProcess[str]:
     return subprocess.CompletedProcess([], returncode, stdout="out\n", stderr="err\n")
 
 
+def _write_core(node_modules: Path, revision: str) -> None:
+    revisions = node_modules / "puppeteer-core" / "lib" / "puppeteer"
+    revisions.mkdir(parents=True)
+    (revisions / "revisions.js").write_text(
+        f"export const PUPPETEER_REVISIONS = Object.freeze({{\n"
+        f"    chrome: '155.0.8059.39',\n"
+        f"    'chrome-headless-shell': '{revision}',\n"
+        f"}});\n",
+        encoding="utf-8",
+    )
+
+
+def _fake_mmdc(
+    root: Path, *, revision: str | None = REVISION, nested: str | None = None
+) -> Path:
+    """Lay out a global mermaid-cli install the way npm does.
+
+    Args:
+        root: npm prefix to build the install under.
+        revision: chrome-headless-shell revision of the puppeteer-core beside
+            puppeteer, or None to install mmdc without a puppeteer next to it.
+        nested: revision of a puppeteer-core nested under puppeteer, as npm
+            lays it out when mermaid-cli pins another puppeteer-core.
+
+    Returns:
+        The ``bin/mmdc`` symlink.
+    """
+    package = root / "lib" / "node_modules" / "@mermaid-js" / "mermaid-cli"
+    (package / "src").mkdir(parents=True)
+    (package / "src" / "cli.js").write_text("", encoding="utf-8")
+    if revision is not None:
+        (package / "node_modules" / "puppeteer").mkdir(parents=True)
+        _write_core(package / "node_modules", revision)
+    if nested is not None:
+        _write_core(package / "node_modules" / "puppeteer" / "node_modules", nested)
+    (root / "bin").mkdir()
+    mmdc = root / "bin" / "mmdc"
+    mmdc.symlink_to(package / "src" / "cli.js")
+    return mmdc
+
+
+def _install_browser(cache: Path, revision: str) -> None:
+    build = cache / "chrome-headless-shell" / f"linux-{revision}" / "linux64"
+    build.mkdir(parents=True)
+    (build / "chrome-headless-shell").write_text("", encoding="utf-8")
+
+
+@contextlib.contextmanager
+def _environment(
+    *, revision: str | None = REVISION, nested: str | None = None
+) -> Iterator[tuple[Path, Path]]:
+    """Patch mmdc's location and the puppeteer cache onto temporary dirs.
+
+    Args:
+        revision: passed to :func:`_fake_mmdc`.
+        nested: passed to :func:`_fake_mmdc`.
+
+    Yields:
+        The npm prefix and the puppeteer cache directory.
+    """
+    with TemporaryDirectory() as prefix, TemporaryDirectory() as cache:
+        mmdc = _fake_mmdc(Path(prefix), revision=revision, nested=nested)
+        with (
+            mock.patch.dict(os.environ, {"PUPPETEER_CACHE_DIR": cache}),
+            mock.patch.object(mermaid_cli, "which", return_value=True),
+            mock.patch.object(mermaid_cli.shutil, "which", return_value=str(mmdc)),
+        ):
+            yield Path(prefix), Path(cache)
+
+
 class TestBrowserProvisioning(unittest.TestCase):
     def provision(self, *, euid: int, results: list[int]) -> tuple[list, list[str]]:
         """Run ``ensure()`` with mmdc present and a stubbed puppeteer.
@@ -29,9 +106,7 @@ class TestBrowserProvisioning(unittest.TestCase):
         """
         warnings: list[str] = []
         with (
-            TemporaryDirectory() as cache,
-            mock.patch.dict(os.environ, {"PUPPETEER_CACHE_DIR": cache}),
-            mock.patch.object(mermaid_cli, "which", return_value=True),
+            _environment(),
             mock.patch.object(mermaid_cli.os, "geteuid", return_value=euid),
             mock.patch.object(mermaid_cli, "warn", side_effect=warnings.append),
             mock.patch.object(
@@ -51,6 +126,14 @@ class TestBrowserProvisioning(unittest.TestCase):
         )
         self.assertEqual(warnings, [])
 
+    def test_the_browser_comes_from_the_puppeteer_next_to_mmdc(self) -> None:
+        calls, _ = self.provision(euid=0, results=[0])
+        self.assertTrue(
+            calls[0]
+            .args[0][0]
+            .endswith("@mermaid-js/mermaid-cli/node_modules/.bin/puppeteer")
+        )
+
     def test_a_user_installs_the_browser_without_libraries(self) -> None:
         calls, _ = self.provision(euid=1000, results=[0])
         self.assertNotIn("--install-deps", calls[0].args[0])
@@ -60,23 +143,21 @@ class TestBrowserProvisioning(unittest.TestCase):
             calls, _ = self.provision(euid=0, results=[0])
         self.assertNotIn("PUPPETEER_SKIP_DOWNLOAD", calls[0].kwargs["env"])
 
-    def test_a_failed_install_clears_the_browser_cache_and_retries_once(self) -> None:
-        with TemporaryDirectory() as cache:
-            folder = Path(cache) / "chrome-headless-shell" / "linux-1"
-            folder.mkdir(parents=True)
-            seen: list[bool] = []
+    def test_a_failed_install_clears_its_revision_and_retries_once(self) -> None:
+        seen: list[tuple[bool, bool]] = []
+        with _environment() as (_, cache):
+            partial = cache / "chrome-headless-shell" / f"linux-{REVISION}"
+            partial.mkdir(parents=True)
+            other = cache / "chrome-headless-shell" / "linux-155.0.8059.79"
+            other.mkdir(parents=True)
 
             def run(*_args, **_kwargs):
-                seen.append(folder.exists())
+                seen.append((partial.exists(), other.exists()))
                 return _done(1 if len(seen) == 1 else 0)
 
-            with (
-                mock.patch.dict(os.environ, {"PUPPETEER_CACHE_DIR": cache}),
-                mock.patch.object(mermaid_cli, "which", return_value=True),
-                mock.patch.object(mermaid_cli.subprocess, "run", side_effect=run),
-            ):
+            with mock.patch.object(mermaid_cli.subprocess, "run", side_effect=run):
                 mermaid_cli.ensure()
-        self.assertEqual(seen, [True, False])
+        self.assertEqual(seen, [(True, True), (False, True)])
 
     def test_a_second_failure_warns_with_puppeteers_output(self) -> None:
         calls, warnings = self.provision(euid=0, results=[1, 1])
@@ -84,28 +165,39 @@ class TestBrowserProvisioning(unittest.TestCase):
         self.assertEqual(len(warnings), 1)
         self.assertIn("out\nerr\n", warnings[0])
 
-    def test_an_installed_browser_is_not_provisioned_again(self) -> None:
-        with TemporaryDirectory() as cache:
-            build = Path(cache) / "chrome-headless-shell" / "linux-1" / "linux64"
-            build.mkdir(parents=True)
-            (build / "chrome-headless-shell").write_text("", encoding="utf-8")
-            with (
-                mock.patch.dict(os.environ, {"PUPPETEER_CACHE_DIR": cache}),
-                mock.patch.object(mermaid_cli, "which", return_value=True),
-                mock.patch.object(mermaid_cli.subprocess, "run") as run,
-            ):
+    def test_the_pinned_browser_is_not_provisioned_again(self) -> None:
+        with _environment() as (_, cache):
+            _install_browser(cache, REVISION)
+            with mock.patch.object(mermaid_cli.subprocess, "run") as run:
                 mermaid_cli.ensure()
         run.assert_not_called()
 
+    def test_a_browser_of_another_revision_is_replaced(self) -> None:
+        with _environment() as (_, cache):
+            _install_browser(cache, "155.0.8059.79")
+            with mock.patch.object(
+                mermaid_cli.subprocess, "run", return_value=_done(0)
+            ) as run:
+                mermaid_cli.ensure()
+        run.assert_called_once()
+
+    def test_mmdc_without_its_own_puppeteer_warns(self) -> None:
+        with (
+            _environment(revision=None),
+            mock.patch.object(mermaid_cli, "warn") as warn,
+            mock.patch.object(mermaid_cli.subprocess, "run") as run,
+        ):
+            mermaid_cli.ensure()
+        run.assert_not_called()
+        warn.assert_called_once()
+
     def test_a_timeout_warns_instead_of_raising(self) -> None:
         with (
-            TemporaryDirectory() as cache,
-            mock.patch.dict(os.environ, {"PUPPETEER_CACHE_DIR": cache}),
-            mock.patch.object(mermaid_cli, "which", return_value=True),
+            _environment(),
             mock.patch.object(
                 mermaid_cli.subprocess,
                 "run",
-                side_effect=subprocess.TimeoutExpired("npx", 900),
+                side_effect=subprocess.TimeoutExpired("puppeteer", 900),
             ),
             mock.patch.object(mermaid_cli, "warn") as warn,
         ):
@@ -114,14 +206,22 @@ class TestBrowserProvisioning(unittest.TestCase):
 
 
 class TestBrowserInstalled(unittest.TestCase):
-    def test_the_executable_in_the_puppeteer_cache_counts(self) -> None:
-        with TemporaryDirectory() as cache:
-            build = Path(cache) / "chrome-headless-shell" / "linux-1" / "linux64"
-            build.mkdir(parents=True)
-            with mock.patch.dict(os.environ, {"PUPPETEER_CACHE_DIR": cache}):
-                self.assertFalse(mermaid_cli.browser_installed())
-                (build / "chrome-headless-shell").write_text("", encoding="utf-8")
-                self.assertTrue(mermaid_cli.browser_installed())
+    def test_only_the_pinned_revision_counts(self) -> None:
+        with _environment() as (_, cache):
+            _install_browser(cache, "155.0.8059.79")
+            self.assertFalse(mermaid_cli.browser_installed())
+            _install_browser(cache, REVISION)
+            self.assertTrue(mermaid_cli.browser_installed())
+
+    def test_the_revision_is_read_from_mmdcs_puppeteer_core(self) -> None:
+        with _environment():
+            self.assertEqual(mermaid_cli.browser_revision(), REVISION)
+
+    def test_a_puppeteer_core_nested_under_puppeteer_wins(self) -> None:
+        with _environment(revision="155.0.8059.79", nested=REVISION) as (_, cache):
+            self.assertEqual(mermaid_cli.browser_revision(), REVISION)
+            _install_browser(cache, REVISION)
+            self.assertTrue(mermaid_cli.browser_installed())
 
 
 if __name__ == "__main__":

utils/install/lint/mermaid_cli.py

@@ -1,7 +1,8 @@
 """Install @mermaid-js/mermaid-cli (mmdc) and the chrome-headless-shell it renders with.
 
 mmdc comes from npm with ``PUPPETEER_SKIP_DOWNLOAD`` set; the browser is
-provisioned through puppeteer afterwards, as root with its shared libraries
+provisioned afterwards through the puppeteer that npm installed next to mmdc,
+at the revision its puppeteer-core pins, as root with its shared libraries
 (``--install-deps``). ``unzip`` is ensured first because puppeteer extracts the
 browser with it. A failed install warns instead of aborting ``install-lint``, so
 sandboxes without npm or browser access still get the rest of the lint
@@ -12,6 +13,7 @@ from __future__ import annotations
 
 import contextlib
 import os
+import re
 import shutil
 import subprocess
 from pathlib import Path
@@ -36,12 +38,59 @@ def browser_cache() -> Path:
     return Path(base) / BROWSER
 
 
+def _mmdc_puppeteer() -> Path | None:
+    mmdc = shutil.which("mmdc")
+    if mmdc is None:
+        return None
+    for parent in Path(mmdc).resolve().parents:
+        puppeteer = parent / "node_modules" / "puppeteer"
+        if puppeteer.is_dir():
+            return puppeteer
+    return None
+
+
+def _puppeteer_core(puppeteer: Path) -> Path | None:
+    for base in (puppeteer, *puppeteer.parents):
+        if base.name == "node_modules":
+            continue
+        core = base / "node_modules" / "puppeteer-core"
+        if core.is_dir():
+            return core
+    return None
+
+
+def browser_revision() -> str | None:
+    puppeteer = _mmdc_puppeteer()
+    if puppeteer is None:
+        return None
+    core = _puppeteer_core(puppeteer)
+    if core is None:
+        return None
+    revisions = core / "lib" / "puppeteer" / "revisions.js"
+    try:
+        text = revisions.read_text(encoding="utf-8")  # nocheck: cache-read  npm prefix
+    except OSError:
+        return None
+    match = re.search(rf"'{BROWSER}':\s*'([^']+)'", text)
+    return match.group(1) if match else None
+
+
 def browser_installed() -> bool:
-    return any(path.is_file() for path in browser_cache().glob(f"*/*/{BROWSER}"))
+    revision = browser_revision()
+    if revision is None:
+        return False
+    return any(
+        path.is_file() for path in browser_cache().glob(f"*-{revision}/*/{BROWSER}")
+    )
 
 
-def _provision_browser() -> subprocess.CompletedProcess[str]:
-    argv = ["npx", "--yes", "puppeteer", "browsers", "install", BROWSER]
+def _provision_browser(puppeteer: Path) -> subprocess.CompletedProcess[str]:
+    argv = [
+        str(puppeteer.parent / ".bin" / "puppeteer"),
+        "browsers",
+        "install",
+        BROWSER,
+    ]
     if os.geteuid() == 0:
         argv.append("--install-deps")
     env = {k: v for k, v in os.environ.items() if k != "PUPPETEER_SKIP_DOWNLOAD"}
@@ -58,12 +107,19 @@ def _provision_browser() -> subprocess.CompletedProcess[str]:
 def _ensure_browser() -> None:
     if browser_installed():
         return
-    log(f"Provisioning {BROWSER} for mmdc via puppeteer.")
+    puppeteer = _mmdc_puppeteer()
+    if puppeteer is None:
+        warn(f"{BROWSER} provisioning skipped: no puppeteer next to mmdc")
+        return
+    log(f"Provisioning {BROWSER} for mmdc via its own puppeteer.")
     try:
-        done = _provision_browser()
+        done = _provision_browser(puppeteer)
         if done.returncode != 0:
-            shutil.rmtree(browser_cache(), ignore_errors=True)
-            done = _provision_browser()
+            revision = browser_revision()
+            if revision is not None:
+                for partial in browser_cache().glob(f"*-{revision}"):
+                    shutil.rmtree(partial, ignore_errors=True)
+            done = _provision_browser(puppeteer)
     except (OSError, subprocess.TimeoutExpired) as exc:
         warn(f"{BROWSER} provisioning skipped (optional lint tool): {exc}")
         return