Files
computer-playbook/tests/unit/python/roles/sys-ctl-hlth-csp/files/test_script.py
T
kevinveenbirkenbachandClaude Opus 5 6e98ecf66f fix(hlth-csp): give onion vhosts a navigation budget Tor can meet
csp-checker navigates every URL with a 20000 ms budget until
domcontentloaded. Over the Tor SOCKS proxy that expires before a page of
module scripts finishes: run 34614940508 lost Mastodon#1 and OpenProject#1
to it, 26 of 27 attempts logging "Unable to reach http://<onion>/
(Navigation timeout of 20000 ms exceeded)" while sys-service-terminator
retried the unit for 35 minutes. No policy was ever read; the checker never
got that far.

csp-checker 2.3.0 takes --timeout, so script.py takes --onion-timeout and
hands it to the onion batch alone, before the -- separator that would make
it a URL. Clearnet keeps the image default. The budget is
HEALTH_CSP_TIMEOUT_MS (20000, the image default) times the tor flavor's
multiplier, 100000 ms on the public network.

The unit's TimeoutStartSec had to follow. It was 30 + 25 s per domain --
130 s for the two onion vhosts Mastodon checks, while a 100 s budget needs
up to 200 s, so systemd would have killed the check mid-run. The filter now
takes onion_per_domain_seconds and counts .onion domains at that rate;
without the argument it computes exactly what it did before, so
sys-ctl-hlth-webserver is unchanged. The Mastodon case goes from 130 s to
450 s, a clearnet-only play stays at 130 s.

Tests pin that --timeout is appended only when set and stays ahead of the
separator, that main budgets the onion batch alone, that the role passes
both --onion-timeout and the onion rate for the start budget (both guards
fail against the previous tasks file), and the filter's split rate.

Not covered: the filter still caps at 3600 s, which binds beyond 34 onion
domains in one round.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-12 00:18:00 +02:00

604 lines
20 KiB
Python

from __future__ import annotations
import sys
import unittest
from pathlib import Path
from unittest.mock import MagicMock, patch
from . import PROJECT_ROOT
ROLE_FILES = PROJECT_ROOT / "roles/sys-ctl-hlth-csp/files/python"
sys.path.insert(0, str(ROLE_FILES))
import script # noqa: E402
class TestExtractDomainsFromFilenames(unittest.TestCase):
@patch("script.os.listdir")
def test_extract_domains_filters_valid_conf_domains(
self, mock_listdir: MagicMock
) -> None:
mock_listdir.return_value = [
"example.com.conf",
"api.example.com.conf",
"not-a-domain.txt",
"no-tld.conf",
".hidden.conf",
"a..b.com.conf",
"example.com.conf.bak",
"localhost.conf",
"sub.domain.co.uk.conf",
]
domains = script.extract_domains_from_filenames(
"/etc/nginx/conf.d/http/servers/"
)
self.assertIsInstance(domains, list)
self.assertIn("example.com", domains)
self.assertIn("api.example.com", domains)
self.assertIn("sub.domain.co.uk", domains)
self.assertNotIn("not-a-domain", domains)
self.assertNotIn("no-tld", domains)
self.assertNotIn(".hidden", domains)
self.assertNotIn("a..b.com", domains)
self.assertNotIn("example.com.conf", domains)
self.assertNotIn("localhost", domains)
@patch("script.os.listdir", side_effect=FileNotFoundError)
def test_extract_domains_returns_none_when_directory_missing(
self, _mock_listdir: MagicMock
) -> None:
domains = script.extract_domains_from_filenames("/missing")
self.assertIsNone(domains)
class TestSplitOnionDomains(unittest.TestCase):
def test_splits_families(self) -> None:
domains = [
"auth.abc123.onion",
"app.infinito.test",
"matomo.abc123.onion",
]
clearnet, onion = script.split_onion_domains(domains)
self.assertEqual(clearnet, ["app.infinito.test"])
self.assertEqual(onion, ["auth.abc123.onion", "matomo.abc123.onion"])
def test_clearnet_only(self) -> None:
domains = ["a.example", "b.example"]
self.assertEqual(script.split_onion_domains(domains), (domains, []))
def test_empty_list(self) -> None:
self.assertEqual(script.split_onion_domains([]), ([], []))
class TestIsSkippedDomain(unittest.TestCase):
def test_explicit_clearnet_domain_is_skipped(self) -> None:
skip_set = {"mirror.infinito.test"}
labels = {d.split(".", 1)[0] for d in skip_set}
self.assertTrue(
script.is_skipped_domain("mirror.infinito.test", skip_set, labels)
)
def test_onion_sibling_of_skipped_clearnet_is_skipped(self) -> None:
skip_set = {"mirror.infinito.test"}
labels = {d.split(".", 1)[0] for d in skip_set}
self.assertTrue(
script.is_skipped_domain("mirror.abc123.onion", skip_set, labels)
)
def test_unrelated_onion_is_not_skipped(self) -> None:
skip_set = {"mirror.infinito.test"}
labels = {d.split(".", 1)[0] for d in skip_set}
self.assertFalse(
script.is_skipped_domain("auth.abc123.onion", skip_set, labels)
)
def test_unrelated_clearnet_is_not_skipped(self) -> None:
skip_set = {"mirror.infinito.test"}
labels = {d.split(".", 1)[0] for d in skip_set}
self.assertFalse(
script.is_skipped_domain("auth.infinito.test", skip_set, labels)
)
class TestMainSkipsOnionSiblings(unittest.TestCase):
@patch("script.run_checker")
@patch("script.build_urls_from_nginx_confs")
@patch("script.extract_domains_from_filenames")
def test_onion_sibling_of_skip_domain_is_excluded(
self,
mock_extract: MagicMock,
mock_build_urls: MagicMock,
mock_run_checker: MagicMock,
) -> None:
mock_extract.return_value = [
"mirror.infinito.test",
"mirror.abc123.onion",
"auth.abc123.onion",
]
mock_build_urls.side_effect = lambda _dir, domains: [
f"http://{d}/" for d in domains
]
mock_run_checker.return_value = 0
with (
patch.object(
script.sys,
"argv",
[
"script.py",
"--nginx-config-dir",
"/etc/nginx",
"--image",
"img:tag",
"--skip-domain",
"mirror.infinito.test",
"--tor-proxy",
"socks5://127.0.0.1:9050",
],
),
self.assertRaises(SystemExit),
):
script.main()
probed = [d for call in mock_build_urls.call_args_list for d in call.args[1]]
self.assertNotIn("mirror.infinito.test", probed)
self.assertNotIn("mirror.abc123.onion", probed)
self.assertIn("auth.abc123.onion", probed)
class TestBuildDockerCmdProxy(unittest.TestCase):
def test_proxy_arg_appended(self) -> None:
cmd = script.build_docker_cmd(
image="img",
urls=["http://a.onion/"],
short_mode=True,
ignore_network_blocks_from=[],
proxy="socks5://127.0.0.1:9050",
)
idx = cmd.index("--proxy")
self.assertEqual(cmd[idx + 1], "socks5://127.0.0.1:9050")
def test_no_proxy_arg_without_proxy(self) -> None:
cmd = script.build_docker_cmd(
image="img",
urls=["https://a.example/"],
short_mode=True,
ignore_network_blocks_from=[],
)
self.assertNotIn("--proxy", cmd)
class TestBuildDockerCmdTimeout(unittest.TestCase):
def test_timeout_arg_appended(self) -> None:
cmd = script.build_docker_cmd(
image="img",
urls=["http://a.onion/"],
short_mode=True,
ignore_network_blocks_from=[],
timeout_ms=100000,
)
idx = cmd.index("--timeout")
self.assertEqual(cmd[idx + 1], "100000")
def test_no_timeout_arg_without_one(self) -> None:
cmd = script.build_docker_cmd(
image="img",
urls=["https://a.example/"],
short_mode=True,
ignore_network_blocks_from=[],
)
self.assertNotIn("--timeout", cmd)
def test_the_timeout_precedes_the_url_separator(self) -> None:
cmd = script.build_docker_cmd(
image="img",
urls=["http://a.onion/"],
short_mode=True,
ignore_network_blocks_from=["cdn.example"],
timeout_ms=100000,
)
self.assertLess(
cmd.index("--timeout"),
cmd.index("--"),
"an argument after the separator reaches the checker as a URL",
)
class TestMainTimesOutOnionsOnly(unittest.TestCase):
@patch("script.run_checker")
@patch("script.build_urls_from_nginx_confs")
@patch("script.extract_domains_from_filenames")
def test_the_onion_batch_alone_carries_the_budget(
self,
mock_extract: MagicMock,
mock_build_urls: MagicMock,
mock_run_checker: MagicMock,
) -> None:
mock_extract.return_value = ["auth.infinito.test", "auth.abc123.onion"]
mock_build_urls.side_effect = lambda _dir, domains: [
f"http://{d}/" for d in domains
]
mock_run_checker.return_value = 0
with (
patch.object(
script.sys,
"argv",
[
"script.py",
"--nginx-config-dir",
"/etc/nginx",
"--image",
"img:tag",
"--tor-proxy",
"socks5://127.0.0.1:9050",
"--onion-timeout",
"100000",
],
),
self.assertRaises(SystemExit),
):
script.main()
budgets = {
call.kwargs["urls"][0]: call.kwargs["timeout_ms"]
for call in mock_run_checker.call_args_list
}
self.assertEqual(budgets["http://auth.abc123.onion/"], 100000)
self.assertEqual(budgets["http://auth.infinito.test/"], 0)
class TestDetectSchemeFromConf(unittest.TestCase):
def test_detects_https_via_443(self) -> None:
with patch("pathlib.Path.read_text", return_value="listen 443 ssl;"):
self.assertEqual(script.detect_scheme_from_conf(Path("x.conf")), "https")
def test_detects_https_via_ssl_flag(self) -> None:
with patch("pathlib.Path.read_text", return_value="listen 8443 ssl;"):
self.assertEqual(script.detect_scheme_from_conf(Path("x.conf")), "https")
def test_detects_http_via_80(self) -> None:
with patch("pathlib.Path.read_text", return_value="listen 80;"):
self.assertEqual(script.detect_scheme_from_conf(Path("x.conf")), "http")
def test_detects_none_when_no_listen_lines(self) -> None:
with patch("pathlib.Path.read_text", return_value="server_name example.com;"):
self.assertIsNone(script.detect_scheme_from_conf(Path("x.conf")))
def test_ignores_comments_and_blank_lines(self) -> None:
conf = """
# listen 443 ssl;
# listen 80;
server_name example.com;
"""
with patch("pathlib.Path.read_text", return_value=conf):
self.assertIsNone(script.detect_scheme_from_conf(Path("x.conf")))
def test_returns_none_when_file_missing(self) -> None:
with patch("pathlib.Path.read_text", side_effect=FileNotFoundError):
self.assertIsNone(script.detect_scheme_from_conf(Path("missing.conf")))
class TestBuildUrlsFromNginxConfs(unittest.TestCase):
@patch("script.detect_scheme_from_conf")
def test_build_urls_https_preferred(self, mock_detect: MagicMock) -> None:
mock_detect.return_value = "https"
urls = script.build_urls_from_nginx_confs("/etc/nginx", ["example.com"])
self.assertEqual(urls, ["https://example.com/"])
@patch("script.detect_scheme_from_conf")
def test_build_urls_http_when_http_detected(self, mock_detect: MagicMock) -> None:
mock_detect.return_value = "http"
urls = script.build_urls_from_nginx_confs("/etc/nginx", ["example.com"])
self.assertEqual(urls, ["http://example.com/"])
@patch("script.detect_scheme_from_conf")
def test_build_urls_falls_back_to_http_and_warns(
self, mock_detect: MagicMock
) -> None:
mock_detect.return_value = None
with patch("sys.stderr") as _stderr:
urls = script.build_urls_from_nginx_confs("/etc/nginx", ["example.com"])
self.assertEqual(urls, ["http://example.com/"])
class TestBuildDockerCmd(unittest.TestCase):
def test_build_docker_cmd_defaults_to_host_network_and_root_user(self) -> None:
cmd = script.build_docker_cmd(
image="ghcr.io/kevinveenbirkenbach/csp-checker:stable",
urls=["http://example.com/"],
short_mode=False,
ignore_network_blocks_from=[],
)
self.assertEqual(cmd[0:3], ["container", "run", "--rm"])
self.assertIn("--user", cmd)
uidx = cmd.index("--user")
self.assertEqual(cmd[uidx + 1], "0:0")
self.assertIn("--network", cmd)
nidx = cmd.index("--network")
self.assertEqual(cmd[nidx + 1], "host")
self.assertIn("ghcr.io/kevinveenbirkenbach/csp-checker:stable", cmd)
self.assertEqual(cmd[-1], "http://example.com/")
def test_build_docker_cmd_can_disable_host_network(self) -> None:
cmd = script.build_docker_cmd(
image="img:tag",
urls=["http://example.com/"],
short_mode=False,
ignore_network_blocks_from=[],
use_host_network=False,
)
self.assertEqual(cmd[0:3], ["container", "run", "--rm"])
self.assertIn("--user", cmd)
uidx = cmd.index("--user")
self.assertEqual(cmd[uidx + 1], "0:0")
self.assertNotIn("--network", cmd)
self.assertNotIn("host", cmd)
def test_build_docker_cmd_short_mode(self) -> None:
cmd = script.build_docker_cmd(
image="img:tag",
urls=["http://example.com/"],
short_mode=True,
ignore_network_blocks_from=[],
)
self.assertIn("--short", cmd)
def test_build_docker_cmd_ignore_list_adds_separator_and_urls(self) -> None:
cmd = script.build_docker_cmd(
image="img:tag",
urls=["http://a.example/", "https://b.example/"],
short_mode=False,
ignore_network_blocks_from=["pxscdn.com", "cdn.example.org"],
)
self.assertIn("--ignore-network-blocks-from", cmd)
idx = cmd.index("--ignore-network-blocks-from")
self.assertEqual(cmd[idx + 1], "pxscdn.com")
self.assertEqual(cmd[idx + 2], "cdn.example.org")
self.assertEqual(cmd[idx + 3], "--")
self.assertEqual(cmd[idx + 4 :], ["http://a.example/", "https://b.example/"])
class TestRunChecker(unittest.TestCase):
@patch("script.subprocess.run")
def test_run_checker_pulls_image_when_always_pull_true(
self, mock_run: MagicMock
) -> None:
mock_run.side_effect = [
MagicMock(returncode=0),
MagicMock(returncode=3),
]
rc = script.run_checker(
image="img:tag",
urls=["http://example.com/"],
short_mode=True,
ignore_network_blocks_from=[],
always_pull=True,
use_host_network=True,
)
self.assertEqual(rc, 3)
self.assertGreaterEqual(mock_run.call_count, 2)
pull_call = mock_run.call_args_list[0]
self.assertEqual(pull_call.kwargs.get("check"), False)
self.assertEqual(pull_call.args[0], ["container", "pull", "img:tag"])
run_call = mock_run.call_args_list[1]
self.assertEqual(run_call.kwargs.get("check"), False)
self.assertEqual(run_call.args[0][0:3], ["container", "run", "--rm"])
self.assertIn("--user", run_call.args[0])
uidx = run_call.args[0].index("--user")
self.assertEqual(run_call.args[0][uidx + 1], "0:0")
@patch("script.subprocess.run")
def test_run_checker_returns_127_if_run_missing(self, mock_run: MagicMock) -> None:
mock_run.side_effect = FileNotFoundError
rc = script.run_checker(
image="img:tag",
urls=["http://example.com/"],
short_mode=False,
ignore_network_blocks_from=[],
always_pull=False,
use_host_network=True,
)
self.assertEqual(rc, 127)
@patch("script.subprocess.run")
def test_run_checker_returns_1_on_unexpected_exception(
self, mock_run: MagicMock
) -> None:
mock_run.side_effect = RuntimeError("boom")
rc = script.run_checker(
image="img:tag",
urls=["http://example.com/"],
short_mode=False,
ignore_network_blocks_from=[],
always_pull=False,
use_host_network=True,
)
self.assertEqual(rc, 1)
class TestMain(unittest.TestCase):
@patch("script.run_checker")
@patch("script.extract_domains_from_filenames")
def test_main_exits_1_when_extract_domains_returns_none(
self,
mock_extract: MagicMock,
mock_run_checker: MagicMock,
) -> None:
mock_extract.return_value = None
with (
patch.object(
script.sys,
"argv",
["script.py", "--nginx-config-dir", "/missing", "--image", "img:tag"],
),
self.assertRaises(SystemExit) as cm,
):
script.main()
self.assertEqual(cm.exception.code, 1)
mock_run_checker.assert_not_called()
@patch("script.run_checker")
@patch("script.extract_domains_from_filenames")
def test_main_exits_0_when_no_domains_found(
self,
mock_extract: MagicMock,
mock_run_checker: MagicMock,
) -> None:
mock_extract.return_value = []
with (
patch.object(
script.sys,
"argv",
["script.py", "--nginx-config-dir", "/etc/nginx", "--image", "img:tag"],
),
self.assertRaises(SystemExit) as cm,
):
script.main()
self.assertEqual(cm.exception.code, 0)
mock_run_checker.assert_not_called()
@patch("script.run_checker")
@patch("script.build_urls_from_nginx_confs")
@patch("script.extract_domains_from_filenames")
def test_main_passes_defaults_and_exits_with_run_checker_rc(
self,
mock_extract: MagicMock,
mock_build_urls: MagicMock,
mock_run_checker: MagicMock,
) -> None:
mock_extract.return_value = ["example.com", "api.example.com"]
mock_build_urls.return_value = [
"http://example.com/",
"https://api.example.com/",
]
mock_run_checker.return_value = 5
with (
patch.object(
script.sys,
"argv",
[
"script.py",
"--nginx-config-dir",
"/etc/nginx",
"--image",
"img:tag",
"--short",
"--ignore-network-blocks-from",
"pxscdn.com",
"cdn.example.org",
],
),
self.assertRaises(SystemExit) as cm,
):
script.main()
self.assertEqual(cm.exception.code, 5)
mock_run_checker.assert_called_once()
kwargs = mock_run_checker.call_args.kwargs
self.assertEqual(kwargs["image"], "img:tag")
self.assertEqual(
kwargs["urls"], ["http://example.com/", "https://api.example.com/"]
)
self.assertTrue(kwargs["short_mode"])
self.assertEqual(
kwargs["ignore_network_blocks_from"], ["pxscdn.com", "cdn.example.org"]
)
self.assertFalse(kwargs["always_pull"])
self.assertTrue(kwargs["use_host_network"])
@patch("script.run_checker")
@patch("script.build_urls_from_nginx_confs")
@patch("script.extract_domains_from_filenames")
def test_main_no_host_network_flag_disables_host_network(
self,
mock_extract: MagicMock,
mock_build_urls: MagicMock,
mock_run_checker: MagicMock,
) -> None:
mock_extract.return_value = ["example.com"]
mock_build_urls.return_value = ["http://example.com/"]
mock_run_checker.return_value = 0
with (
patch.object(
script.sys,
"argv",
[
"script.py",
"--nginx-config-dir",
"/etc/nginx",
"--image",
"img:tag",
"--no-host-network",
],
),
self.assertRaises(SystemExit) as cm,
):
script.main()
self.assertEqual(cm.exception.code, 0)
kwargs = mock_run_checker.call_args.kwargs
self.assertFalse(kwargs["use_host_network"])
@patch("script.run_checker")
@patch("script.build_urls_from_nginx_confs")
@patch("script.extract_domains_from_filenames")
def test_main_exits_0_when_no_urls_built(
self,
mock_extract: MagicMock,
mock_build_urls: MagicMock,
mock_run_checker: MagicMock,
) -> None:
mock_extract.return_value = ["example.com"]
mock_build_urls.return_value = []
with (
patch.object(
script.sys,
"argv",
["script.py", "--nginx-config-dir", "/etc/nginx", "--image", "img:tag"],
),
self.assertRaises(SystemExit) as cm,
):
script.main()
self.assertEqual(cm.exception.code, 0)
mock_run_checker.assert_not_called()
if __name__ == "__main__":
unittest.main()