fix(hlth-csp): give onion vhosts a navigation budget Tor can meet

csp-checker navigates every URL with a 20000 ms budget until
domcontentloaded. Over the Tor SOCKS proxy that expires before a page of
module scripts finishes: run 34614940508 lost Mastodon#1 and OpenProject#1
to it, 26 of 27 attempts logging "Unable to reach http://<onion>/
(Navigation timeout of 20000 ms exceeded)" while sys-service-terminator
retried the unit for 35 minutes. No policy was ever read; the checker never
got that far.

csp-checker 2.3.0 takes --timeout, so script.py takes --onion-timeout and
hands it to the onion batch alone, before the -- separator that would make
it a URL. Clearnet keeps the image default. The budget is
HEALTH_CSP_TIMEOUT_MS (20000, the image default) times the tor flavor's
multiplier, 100000 ms on the public network.

The unit's TimeoutStartSec had to follow. It was 30 + 25 s per domain --
130 s for the two onion vhosts Mastodon checks, while a 100 s budget needs
up to 200 s, so systemd would have killed the check mid-run. The filter now
takes onion_per_domain_seconds and counts .onion domains at that rate;
without the argument it computes exactly what it did before, so
sys-ctl-hlth-webserver is unchanged. The Mastodon case goes from 130 s to
450 s, a clearnet-only play stays at 130 s.

Tests pin that --timeout is appended only when set and stays ahead of the
separator, that main budgets the onion batch alone, that the role passes
both --onion-timeout and the onion rate for the start budget (both guards
fail against the previous tasks file), and the filter's split rate.

Not covered: the filter still caps at 3600 s, which binds beyond 34 onion
domains in one round.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

plugins/filter/timeout_start_sec_for_domains.py

@@ -12,6 +12,7 @@ class FilterModule:
         domains_dict,
         include_www=True,
         per_domain_seconds=25,
+        onion_per_domain_seconds=None,
         overhead_seconds=30,
         min_seconds=120,
         max_seconds=3600,
@@ -22,6 +23,9 @@ class FilterModule:
                 (values can be str | list[str] | dict[str,str]) or an already
                 flattened list of domains, or a single domain string.
             include_www (bool): If true, add 'www.<domain>' for non-www entries.
+            per_domain_seconds (int): Budget a clearnet domain contributes.
+            onion_per_domain_seconds (int | None): Budget a '.onion' domain
+                contributes; None gives it per_domain_seconds.
             ...
         """
         try:
@@ -59,9 +63,21 @@ class FilterModule:
                 flat.extend(www_variants)
 
             unique_domains = sorted(set(flat))
-            count = len(unique_domains)
+            onion_count = sum(
+                1 for d in unique_domains if str(d).lower().endswith(".onion")
+            )
+            clearnet_count = len(unique_domains) - onion_count
+            onion_seconds = (
+                per_domain_seconds
+                if onion_per_domain_seconds is None
+                else int(onion_per_domain_seconds)
+            )
 
-            raw = overhead_seconds + per_domain_seconds * count
+            raw = (
+                overhead_seconds
+                + per_domain_seconds * clearnet_count
+                + onion_seconds * onion_count
+            )
             return max(min_seconds, min(max_seconds, int(raw)))
 
         except AnsibleFilterError:

roles/sys-ctl-hlth-csp/defaults/main.yml

@@ -10,3 +10,9 @@ HEALTH_CSP_IMAGE: "{{ lookup('config', 'sys-ctl-hlth-csp', 'services.csp-checker
 
 # If true, pull the image on each run (safe but slower)
 HEALTH_CSP_ALWAYS_PULL: false
+
+# Navigation budget in milliseconds a clearnet vhost gets; mirrors the checker's own default
+HEALTH_CSP_TIMEOUT_MS: 20000
+
+# Seconds added per vhost on top of its navigation budget when sizing TimeoutStartSec
+HEALTH_CSP_DOMAIN_RESERVE_SECONDS: 5

roles/sys-ctl-hlth-csp/files/python/script.py

@@ -130,6 +130,7 @@ def build_docker_cmd(
     ignore_network_blocks_from: list[str],
     use_host_network: bool = True,
     proxy: str = "",
+    timeout_ms: int = 0,
 ) -> list[str]:
     cmd = ["container", "run", "--rm"]
 
@@ -148,6 +149,9 @@ def build_docker_cmd(
     if proxy:
         cmd.extend(["--proxy", proxy])
 
+    if timeout_ms:
+        cmd.extend(["--timeout", str(timeout_ms)])
+
     if ignore_network_blocks_from:
         cmd.append("--ignore-network-blocks-from")
         cmd.extend(ignore_network_blocks_from)
@@ -165,6 +169,7 @@ def run_checker(
     always_pull: bool,
     use_host_network: bool = True,
     proxy: str = "",
+    timeout_ms: int = 0,
 ) -> int:
     """
     Runs the CSP checker container and returns its exit code.
@@ -180,6 +185,7 @@ def run_checker(
         ignore_network_blocks_from=ignore_network_blocks_from,
         use_host_network=use_host_network,
         proxy=proxy,
+        timeout_ms=timeout_ms,
     )
 
     try:
@@ -239,6 +245,16 @@ def main() -> None:
             "otherwise treat as unreachable."
         ),
     )
+    parser.add_argument(
+        "--onion-timeout",
+        type=int,
+        default=0,
+        help=(
+            "Navigation budget in milliseconds for the .onion batch, which "
+            "reaches its vhosts over Tor and needs longer than the checker's "
+            "own default. 0 leaves that default in place."
+        ),
+    )
     parser.add_argument(
         "--tor-proxy",
         default="",
@@ -287,9 +303,9 @@ def main() -> None:
         sys.exit(0)
 
     rc = 0
-    for batch_domains, batch_proxy in (
-        (clearnet_domains, ""),
-        (onion_domains, args.tor_proxy),
+    for batch_domains, batch_proxy, batch_timeout in (
+        (clearnet_domains, "", 0),
+        (onion_domains, args.tor_proxy, args.onion_timeout),
     ):
         if not batch_domains:
             continue
@@ -305,6 +321,7 @@ def main() -> None:
             always_pull=bool(args.always_pull),
             use_host_network=not bool(args.no_host_network),
             proxy=batch_proxy,
+            timeout_ms=batch_timeout,
         )
         rc = rc or batch_rc
     sys.exit(rc)

roles/sys-ctl-hlth-csp/meta/services.yml

@@ -8,4 +8,4 @@ csp:
       categories: [web]
 csp-checker:
   image: ghcr.io/kevinveenbirkenbach/csp-checker
-  version: 2.2.1
+  version: 2.3.0

roles/sys-ctl-hlth-csp/tasks/00_core.yml

@@ -36,7 +36,8 @@
     system_service_on_calendar: "{{ SYS_SCHEDULE_HEALTH_CSP_CRAWLER }}"
     system_service_timer_enabled: true
     system_service_tpl_on_failure: "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
-    system_service_tpl_timeout_start_sec: "{{ CURRENT_PLAY_DOMAINS_ALL | timeout_start_sec_for_domains }}"
+    system_service_tpl_timeout_start_sec: "{{ CURRENT_PLAY_DOMAINS_ALL | timeout_start_sec_for_domains(onion_per_domain_seconds=HEALTH_CSP_ONION_DOMAIN_SECONDS
+      | int) }}"
     system_service_tpl_environment: "{{ CA_TRUST_SYSTEMD_ENV_SELFSIGNED }}"
     system_service_tpl_exec_start: >-
       {{
@@ -60,7 +61,8 @@
             ''
           ),
           (lookup('config', 'svc-net-tor', 'services.tor.node', '') | length > 0) | ternary(
-            '--tor-proxy=socks5://' ~ lookup('tor_socks'),
+            '--tor-proxy=socks5://' ~ lookup('tor_socks')
+            ~ ' --onion-timeout=' ~ HEALTH_CSP_ONION_TIMEOUT_MS,
             ''
           )
         ]

roles/sys-ctl-hlth-csp/vars/main.yml

@@ -1,5 +1,8 @@
 ---
 system_service_id: sys-ctl-hlth-csp
+HEALTH_CSP_ONION_TIMEOUT_MS: "{{ (HEALTH_CSP_TIMEOUT_MS | int) * (lookup('config', 'svc-net-tor', 'services.tor.timeout_multipliers')[lookup('config', 'svc-net-tor',
+  'services.tor.flavor')] | int) }}"
+HEALTH_CSP_ONION_DOMAIN_SECONDS: "{{ ((HEALTH_CSP_ONION_TIMEOUT_MS | int) // 1000) + (HEALTH_CSP_DOMAIN_RESERVE_SECONDS | int) }}"
 HEALTH_CSP_SKIP_DOMAINS: >-
   {{ (
        lookup('csp_skip_domains', group_names=lookup('deployment').deployed)

tests/unit/python/plugins/filter/test_timeout_start_sec_for_domains.py

@@ -25,6 +25,41 @@ class TestTimeoutStartSecForDomains(unittest.TestCase):
         )
         self.assertEqual(result, 180)
 
+    def test_an_onion_domain_costs_its_own_rate(self):
+        result = _f()(
+            ["example.com", "abc.onion"],
+            include_www=False,
+            per_domain_seconds=25,
+            onion_per_domain_seconds=105,
+            overhead_seconds=30,
+            min_seconds=0,
+            max_seconds=3600,
+        )
+        self.assertEqual(result, 160)
+
+    def test_without_an_onion_rate_every_domain_costs_the_same(self):
+        result = _f()(
+            ["example.com", "abc.onion"],
+            include_www=False,
+            per_domain_seconds=25,
+            overhead_seconds=30,
+            min_seconds=0,
+            max_seconds=3600,
+        )
+        self.assertEqual(result, 80)
+
+    def test_the_onion_rate_covers_the_www_variant_too(self):
+        result = _f()(
+            ["abc.onion"],
+            include_www=True,
+            per_domain_seconds=25,
+            onion_per_domain_seconds=105,
+            overhead_seconds=30,
+            min_seconds=0,
+            max_seconds=3600,
+        )
+        self.assertEqual(result, 240)
+
     def test_no_www_min_clamp_applies(self):
         domains = {
             "canonical": ["example.com", "foo.bar"],

tests/unit/python/roles/sys-ctl-hlth-csp/__init__.py

@@ -1 +1,3 @@
+from pathlib import Path
 
+PROJECT_ROOT: Path = Path(__file__).resolve().parents[5]

tests/unit/python/roles/sys-ctl-hlth-csp/files/test_script.py

@@ -171,6 +171,86 @@ class TestBuildDockerCmdProxy(unittest.TestCase):
         self.assertNotIn("--proxy", cmd)
 
 
+class TestBuildDockerCmdTimeout(unittest.TestCase):
+    def test_timeout_arg_appended(self) -> None:
+        cmd = script.build_docker_cmd(
+            image="img",
+            urls=["http://a.onion/"],
+            short_mode=True,
+            ignore_network_blocks_from=[],
+            timeout_ms=100000,
+        )
+        idx = cmd.index("--timeout")
+        self.assertEqual(cmd[idx + 1], "100000")
+
+    def test_no_timeout_arg_without_one(self) -> None:
+        cmd = script.build_docker_cmd(
+            image="img",
+            urls=["https://a.example/"],
+            short_mode=True,
+            ignore_network_blocks_from=[],
+        )
+        self.assertNotIn("--timeout", cmd)
+
+    def test_the_timeout_precedes_the_url_separator(self) -> None:
+        cmd = script.build_docker_cmd(
+            image="img",
+            urls=["http://a.onion/"],
+            short_mode=True,
+            ignore_network_blocks_from=["cdn.example"],
+            timeout_ms=100000,
+        )
+        self.assertLess(
+            cmd.index("--timeout"),
+            cmd.index("--"),
+            "an argument after the separator reaches the checker as a URL",
+        )
+
+
+class TestMainTimesOutOnionsOnly(unittest.TestCase):
+    @patch("script.run_checker")
+    @patch("script.build_urls_from_nginx_confs")
+    @patch("script.extract_domains_from_filenames")
+    def test_the_onion_batch_alone_carries_the_budget(
+        self,
+        mock_extract: MagicMock,
+        mock_build_urls: MagicMock,
+        mock_run_checker: MagicMock,
+    ) -> None:
+        mock_extract.return_value = ["auth.infinito.test", "auth.abc123.onion"]
+        mock_build_urls.side_effect = lambda _dir, domains: [
+            f"http://{d}/" for d in domains
+        ]
+        mock_run_checker.return_value = 0
+
+        with (
+            patch.object(
+                script.sys,
+                "argv",
+                [
+                    "script.py",
+                    "--nginx-config-dir",
+                    "/etc/nginx",
+                    "--image",
+                    "img:tag",
+                    "--tor-proxy",
+                    "socks5://127.0.0.1:9050",
+                    "--onion-timeout",
+                    "100000",
+                ],
+            ),
+            self.assertRaises(SystemExit),
+        ):
+            script.main()
+
+        budgets = {
+            call.kwargs["urls"][0]: call.kwargs["timeout_ms"]
+            for call in mock_run_checker.call_args_list
+        }
+        self.assertEqual(budgets["http://auth.abc123.onion/"], 100000)
+        self.assertEqual(budgets["http://auth.infinito.test/"], 0)
+
+
 class TestDetectSchemeFromConf(unittest.TestCase):
     def test_detects_https_via_443(self) -> None:
         with patch("pathlib.Path.read_text", return_value="listen 443 ssl;"):

tests/unit/python/roles/sys-ctl-hlth-csp/test_onion_timeout_is_passed.py

@@ -0,0 +1,36 @@
+import re
+import unittest
+
+from utils.cache.files import read_text
+
+from . import PROJECT_ROOT
+
+TASKS = PROJECT_ROOT / "roles" / "sys-ctl-hlth-csp" / "tasks" / "00_core.yml"
+_TOR_BRANCH = re.compile(r"--tor-proxy=.{0,200}?--onion-timeout=", re.DOTALL)
+_START_BUDGET = re.compile(
+    r"timeout_start_sec_for_domains\(\s*onion_per_domain_seconds=", re.DOTALL
+)
+
+
+class TestOnionTimeoutIsPassed(unittest.TestCase):
+    def test_the_tor_branch_also_carries_the_onion_budget(self) -> None:
+        self.assertRegex(
+            read_text(str(TASKS)),
+            _TOR_BRANCH,
+            "the service reaches onion vhosts over Tor, where the checker's own "
+            "20 s navigation budget expires before a heavy page finishes; pass "
+            "--onion-timeout wherever --tor-proxy is passed",
+        )
+
+    def test_the_unit_start_budget_knows_the_onion_rate(self) -> None:
+        self.assertRegex(
+            read_text(str(TASKS)),
+            _START_BUDGET,
+            "TimeoutStartSec defaults to 25 s per domain, which an onion vhost "
+            "with a multiplied navigation budget outlasts; systemd then kills "
+            "the check mid-run instead of letting it report",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()