kevinveenbirkenbach/computer-playbook
1
0 Fork 0
mirror of https://github.com/kevinveenbirkenbach/computer-playbook.git synced 2026-11-08 19:56:53 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
4c4b00a0e974600a07cdc072c8dd152a8575d9c5
computer-playbook/tests/unit/python/roles/sys-ctl-hlth-csp/files/test_script.py
T
kevinveenbirkenbachandClaude Opus 5 8bb500a833 fix(sys-ctl-hlth-csp): give the clearnet batch the budget the role advertises 
Moodle#2 of run 39186571513 failed the health check 26 times with

  academy.main.infinito.test: Unable to reach https://academy.main.infinito.test/
  (Navigation timeout of 20000 ms exceeded)

while every other vhost on the stack printed 'No CSP or network blocks
detected' and nginx answered 200 or 303 to every request. Nothing was
blocked; the page is slow. Moodle #0, #1 and #3 pass, and only the arm64
swarm row fails.

The role already claimed to own that budget. HEALTH_CSP_TIMEOUT_MS sat in
defaults with a comment calling it the clearnet navigation budget, but
script.py ran the clearnet batch as (clearnet_domains, '', 0), so clearnet
silently inherited the checker's own 20000 and the knob did nothing.
Only the onion batch ever received a real --timeout.

--clearnet-timeout closes that, defaulting to 0 so a call without the flag
behaves exactly as before. The value lives in its own variable rather than
raising the shared base, which would have multiplied through
HEALTH_CSP_ONION_TIMEOUT_MS and taken the public onion budget from 100s to
225s, toward the 3600s TimeoutStartSec cap. Onion budgets are unchanged.

45000 is derived, not picked: the slowest measured navigation on that
target reached domcontentloaded after 26s, with single requests stalling
up to 5.8s, so 26 + 3*5.8 = 43.4s, rounded. That is 1.7x the worst
navigation observed there.

TimeoutStartSec is deliberately NOT derived from the new budget. The
per-domain rate sizes the whole unit and nearly every domain answers in
well under a second; using the worst case as the scheduling rate would
have doubled the clearnet term and pushed large stacks toward the cap for
no measured reason. Scheduling is unchanged.

Known cost: above 20s the check no longer notices that an app got slow.
That coverage was incidental, not declared, and a page-load regression
needs its own assertion rather than a CSP probe.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-10-20 18:13:19 +02:00

644 lines
21 KiB
Python
Raw Blame History

from __future__ import annotations
import sys
import unittest
from pathlib import Path
from unittest.mock import MagicMock, patch
from . import PROJECT_ROOT
ROLE_FILES = PROJECT_ROOT / "roles/sys-ctl-hlth-csp/files/python"
sys.path.insert(0, str(ROLE_FILES))
import script # noqa: E402
class TestExtractDomainsFromFilenames(unittest.TestCase):
@patch("script.os.listdir")
def test_extract_domains_filters_valid_conf_domains(
self, mock_listdir: MagicMock
) -> None:
mock_listdir.return_value = [
"example.com.conf",
"api.example.com.conf",
"not-a-domain.txt",
"no-tld.conf",
".hidden.conf",
"a..b.com.conf",
"example.com.conf.bak",
"localhost.conf",
"sub.domain.co.uk.conf",
]
domains = script.extract_domains_from_filenames(
"/etc/nginx/conf.d/http/servers/"
)
self.assertIsInstance(domains, list)
self.assertIn("example.com", domains)
self.assertIn("api.example.com", domains)
self.assertIn("sub.domain.co.uk", domains)
self.assertNotIn("not-a-domain", domains)
self.assertNotIn("no-tld", domains)
self.assertNotIn(".hidden", domains)
self.assertNotIn("a..b.com", domains)
self.assertNotIn("example.com.conf", domains)
self.assertNotIn("localhost", domains)
@patch("script.os.listdir", side_effect=FileNotFoundError)
def test_extract_domains_returns_none_when_directory_missing(
self, _mock_listdir: MagicMock
) -> None:
domains = script.extract_domains_from_filenames("/missing")
self.assertIsNone(domains)
class TestSplitOnionDomains(unittest.TestCase):
def test_splits_families(self) -> None:
domains = [
"auth.abc123.onion",
"app.infinito.test",
"matomo.abc123.onion",
]
clearnet, onion = script.split_onion_domains(domains)
self.assertEqual(clearnet, ["app.infinito.test"])
self.assertEqual(onion, ["auth.abc123.onion", "matomo.abc123.onion"])
def test_clearnet_only(self) -> None:
domains = ["a.example", "b.example"]
self.assertEqual(script.split_onion_domains(domains), (domains, []))
def test_empty_list(self) -> None:
self.assertEqual(script.split_onion_domains([]), ([], []))
class TestIsSkippedDomain(unittest.TestCase):
def test_explicit_clearnet_domain_is_skipped(self) -> None:
skip_set = {"mirror.infinito.test"}
labels = {d.split(".", 1)[0] for d in skip_set}
self.assertTrue(
script.is_skipped_domain("mirror.infinito.test", skip_set, labels)
)
def test_onion_sibling_of_skipped_clearnet_is_skipped(self) -> None:
skip_set = {"mirror.infinito.test"}
labels = {d.split(".", 1)[0] for d in skip_set}
self.assertTrue(
script.is_skipped_domain("mirror.abc123.onion", skip_set, labels)
)
def test_unrelated_onion_is_not_skipped(self) -> None:
skip_set = {"mirror.infinito.test"}
labels = {d.split(".", 1)[0] for d in skip_set}
self.assertFalse(
script.is_skipped_domain("auth.abc123.onion", skip_set, labels)
)
def test_unrelated_clearnet_is_not_skipped(self) -> None:
skip_set = {"mirror.infinito.test"}
labels = {d.split(".", 1)[0] for d in skip_set}
self.assertFalse(
script.is_skipped_domain("auth.infinito.test", skip_set, labels)
)
class TestMainSkipsOnionSiblings(unittest.TestCase):
@patch("script.run_checker")
@patch("script.build_urls_from_nginx_confs")
@patch("script.extract_domains_from_filenames")
def test_onion_sibling_of_skip_domain_is_excluded(
self,
mock_extract: MagicMock,
mock_build_urls: MagicMock,
mock_run_checker: MagicMock,
) -> None:
mock_extract.return_value = [
"mirror.infinito.test",
"mirror.abc123.onion",
"auth.abc123.onion",
]
mock_build_urls.side_effect = lambda _dir, domains: [
f"http://{d}/" for d in domains
]
mock_run_checker.return_value = 0
with (
patch.object(
script.sys,
"argv",
[
"script.py",
"--nginx-config-dir",
"/etc/nginx",
"--image",
"img:tag",
"--skip-domain",
"mirror.infinito.test",
"--tor-proxy",
"socks5://127.0.0.1:9050",
],
),
self.assertRaises(SystemExit),
):
script.main()
probed = [d for call in mock_build_urls.call_args_list for d in call.args[1]]
self.assertNotIn("mirror.infinito.test", probed)
self.assertNotIn("mirror.abc123.onion", probed)
self.assertIn("auth.abc123.onion", probed)
class TestBuildDockerCmdProxy(unittest.TestCase):
def test_proxy_arg_appended(self) -> None:
cmd = script.build_docker_cmd(
image="img",
urls=["http://a.onion/"],
short_mode=True,
ignore_network_blocks_from=[],
proxy="socks5://127.0.0.1:9050",
)
idx = cmd.index("--proxy")
self.assertEqual(cmd[idx + 1], "socks5://127.0.0.1:9050")
def test_no_proxy_arg_without_proxy(self) -> None:
cmd = script.build_docker_cmd(
image="img",
urls=["https://a.example/"],
short_mode=True,
ignore_network_blocks_from=[],
)
self.assertNotIn("--proxy", cmd)
class TestBuildDockerCmdTimeout(unittest.TestCase):
def test_timeout_arg_appended(self) -> None:
cmd = script.build_docker_cmd(
image="img",
urls=["http://a.onion/"],
short_mode=True,
ignore_network_blocks_from=[],
timeout_ms=100000,
)
idx = cmd.index("--timeout")
self.assertEqual(cmd[idx + 1], "100000")
def test_no_timeout_arg_without_one(self) -> None:
cmd = script.build_docker_cmd(
image="img",
urls=["https://a.example/"],
short_mode=True,
ignore_network_blocks_from=[],
)
self.assertNotIn("--timeout", cmd)
def test_the_timeout_precedes_the_url_separator(self) -> None:
cmd = script.build_docker_cmd(
image="img",
urls=["http://a.onion/"],
short_mode=True,
ignore_network_blocks_from=["cdn.example"],
timeout_ms=100000,
)
self.assertLess(
cmd.index("--timeout"),
cmd.index("--"),
"an argument after the separator reaches the checker as a URL",
)
class TestMainTimesOutOnionsOnly(unittest.TestCase):
@patch("script.run_checker")
@patch("script.build_urls_from_nginx_confs")
@patch("script.extract_domains_from_filenames")
def test_the_onion_batch_alone_carries_the_budget(
self,
mock_extract: MagicMock,
mock_build_urls: MagicMock,
mock_run_checker: MagicMock,
) -> None:
mock_extract.return_value = ["auth.infinito.test", "auth.abc123.onion"]
mock_build_urls.side_effect = lambda _dir, domains: [
f"http://{d}/" for d in domains
]
mock_run_checker.return_value = 0
with (
patch.object(
script.sys,
"argv",
[
"script.py",
"--nginx-config-dir",
"/etc/nginx",
"--image",
"img:tag",
"--tor-proxy",
"socks5://127.0.0.1:9050",
"--onion-timeout",
"100000",
],
),
self.assertRaises(SystemExit),
):
script.main()
budgets = {
call.kwargs["urls"][0]: call.kwargs["timeout_ms"]
for call in mock_run_checker.call_args_list
}
self.assertEqual(budgets["http://auth.abc123.onion/"], 100000)
self.assertEqual(budgets["http://auth.infinito.test/"], 0)
@patch("script.run_checker")
@patch("script.build_urls_from_nginx_confs")
@patch("script.extract_domains_from_filenames")
def test_the_clearnet_batch_carries_its_own_budget(
self,
mock_extract: MagicMock,
mock_build_urls: MagicMock,
mock_run_checker: MagicMock,
) -> None:
mock_extract.return_value = ["auth.infinito.test"]
mock_build_urls.side_effect = lambda _dir, domains: [
f"http://{d}/" for d in domains
]
mock_run_checker.return_value = 0
with (
patch.object(
script.sys,
"argv",
[
"script.py",
"--nginx-config-dir",
"/etc/nginx",
"--image",
"img:tag",
"--clearnet-timeout",
"45000",
],
),
self.assertRaises(SystemExit),
):
script.main()
self.assertEqual(
mock_run_checker.call_args.kwargs["timeout_ms"],
45000,
"without it the clearnet batch keeps the checker's 20 s default, "
"which a heavy app outlasts and the probe reports as unreachable",
)
class TestDetectSchemeFromConf(unittest.TestCase):
def test_detects_https_via_443(self) -> None:
with patch("pathlib.Path.read_text", return_value="listen 443 ssl;"):
self.assertEqual(script.detect_scheme_from_conf(Path("x.conf")), "https")
def test_detects_https_via_ssl_flag(self) -> None:
with patch("pathlib.Path.read_text", return_value="listen 8443 ssl;"):
self.assertEqual(script.detect_scheme_from_conf(Path("x.conf")), "https")
def test_detects_http_via_80(self) -> None:
with patch("pathlib.Path.read_text", return_value="listen 80;"):
self.assertEqual(script.detect_scheme_from_conf(Path("x.conf")), "http")
def test_detects_none_when_no_listen_lines(self) -> None:
with patch("pathlib.Path.read_text", return_value="server_name example.com;"):
self.assertIsNone(script.detect_scheme_from_conf(Path("x.conf")))
def test_ignores_comments_and_blank_lines(self) -> None:
conf = """
# listen 443 ssl;
# listen 80;
server_name example.com;
"""
with patch("pathlib.Path.read_text", return_value=conf):
self.assertIsNone(script.detect_scheme_from_conf(Path("x.conf")))
def test_returns_none_when_file_missing(self) -> None:
with patch("pathlib.Path.read_text", side_effect=FileNotFoundError):
self.assertIsNone(script.detect_scheme_from_conf(Path("missing.conf")))
class TestBuildUrlsFromNginxConfs(unittest.TestCase):
@patch("script.detect_scheme_from_conf")
def test_build_urls_https_preferred(self, mock_detect: MagicMock) -> None:
mock_detect.return_value = "https"
urls = script.build_urls_from_nginx_confs("/etc/nginx", ["example.com"])
self.assertEqual(urls, ["https://example.com/"])
@patch("script.detect_scheme_from_conf")
def test_build_urls_http_when_http_detected(self, mock_detect: MagicMock) -> None:
mock_detect.return_value = "http"
urls = script.build_urls_from_nginx_confs("/etc/nginx", ["example.com"])
self.assertEqual(urls, ["http://example.com/"])
@patch("script.detect_scheme_from_conf")
def test_build_urls_falls_back_to_http_and_warns(
self, mock_detect: MagicMock
) -> None:
mock_detect.return_value = None
with patch("sys.stderr") as _stderr:
urls = script.build_urls_from_nginx_confs("/etc/nginx", ["example.com"])
self.assertEqual(urls, ["http://example.com/"])
class TestBuildDockerCmd(unittest.TestCase):
def test_build_docker_cmd_defaults_to_host_network_and_root_user(self) -> None:
cmd = script.build_docker_cmd(
image="ghcr.io/kevinveenbirkenbach/csp-checker:stable",
urls=["http://example.com/"],
short_mode=False,
ignore_network_blocks_from=[],
)
self.assertEqual(cmd[0:3], ["container", "run", "--rm"])
self.assertIn("--user", cmd)
uidx = cmd.index("--user")
self.assertEqual(cmd[uidx + 1], "0:0")
self.assertIn("--network", cmd)
nidx = cmd.index("--network")
self.assertEqual(cmd[nidx + 1], "host")
self.assertIn("ghcr.io/kevinveenbirkenbach/csp-checker:stable", cmd)
self.assertEqual(cmd[-1], "http://example.com/")
def test_build_docker_cmd_can_disable_host_network(self) -> None:
cmd = script.build_docker_cmd(
image="img:tag",
urls=["http://example.com/"],
short_mode=False,
ignore_network_blocks_from=[],
use_host_network=False,
)
self.assertEqual(cmd[0:3], ["container", "run", "--rm"])
self.assertIn("--user", cmd)
uidx = cmd.index("--user")
self.assertEqual(cmd[uidx + 1], "0:0")
self.assertNotIn("--network", cmd)
self.assertNotIn("host", cmd)
def test_build_docker_cmd_short_mode(self) -> None:
cmd = script.build_docker_cmd(
image="img:tag",
urls=["http://example.com/"],
short_mode=True,
ignore_network_blocks_from=[],
)
self.assertIn("--short", cmd)
def test_build_docker_cmd_ignore_list_adds_separator_and_urls(self) -> None:
cmd = script.build_docker_cmd(
image="img:tag",
urls=["http://a.example/", "https://b.example/"],
short_mode=False,
ignore_network_blocks_from=["pxscdn.com", "cdn.example.org"],
)
self.assertIn("--ignore-network-blocks-from", cmd)
idx = cmd.index("--ignore-network-blocks-from")
self.assertEqual(cmd[idx + 1], "pxscdn.com")
self.assertEqual(cmd[idx + 2], "cdn.example.org")
self.assertEqual(cmd[idx + 3], "--")
self.assertEqual(cmd[idx + 4 :], ["http://a.example/", "https://b.example/"])
class TestRunChecker(unittest.TestCase):
@patch("script.subprocess.run")
def test_run_checker_pulls_image_when_always_pull_true(
self, mock_run: MagicMock
) -> None:
mock_run.side_effect = [
MagicMock(returncode=0),
MagicMock(returncode=3),
]
rc = script.run_checker(
image="img:tag",
urls=["http://example.com/"],
short_mode=True,
ignore_network_blocks_from=[],
always_pull=True,
use_host_network=True,
)
self.assertEqual(rc, 3)
self.assertGreaterEqual(mock_run.call_count, 2)
pull_call = mock_run.call_args_list[0]
self.assertEqual(pull_call.kwargs.get("check"), False)
self.assertEqual(pull_call.args[0], ["container", "pull", "img:tag"])
run_call = mock_run.call_args_list[1]
self.assertEqual(run_call.kwargs.get("check"), False)
self.assertEqual(run_call.args[0][0:3], ["container", "run", "--rm"])
self.assertIn("--user", run_call.args[0])
uidx = run_call.args[0].index("--user")
self.assertEqual(run_call.args[0][uidx + 1], "0:0")
@patch("script.subprocess.run")
def test_run_checker_returns_127_if_run_missing(self, mock_run: MagicMock) -> None:
mock_run.side_effect = FileNotFoundError
rc = script.run_checker(
image="img:tag",
urls=["http://example.com/"],
short_mode=False,
ignore_network_blocks_from=[],
always_pull=False,
use_host_network=True,
)
self.assertEqual(rc, 127)
@patch("script.subprocess.run")
def test_run_checker_returns_1_on_unexpected_exception(
self, mock_run: MagicMock
) -> None:
mock_run.side_effect = RuntimeError("boom")
rc = script.run_checker(
image="img:tag",
urls=["http://example.com/"],
short_mode=False,
ignore_network_blocks_from=[],
always_pull=False,
use_host_network=True,
)
self.assertEqual(rc, 1)
class TestMain(unittest.TestCase):
@patch("script.run_checker")
@patch("script.extract_domains_from_filenames")
def test_main_exits_1_when_extract_domains_returns_none(
self,
mock_extract: MagicMock,
mock_run_checker: MagicMock,
) -> None:
mock_extract.return_value = None
with (
patch.object(
script.sys,
"argv",
["script.py", "--nginx-config-dir", "/missing", "--image", "img:tag"],
),
self.assertRaises(SystemExit) as cm,
):
script.main()
self.assertEqual(cm.exception.code, 1)
mock_run_checker.assert_not_called()
@patch("script.run_checker")
@patch("script.extract_domains_from_filenames")
def test_main_exits_0_when_no_domains_found(
self,
mock_extract: MagicMock,
mock_run_checker: MagicMock,
) -> None:
mock_extract.return_value = []
with (
patch.object(
script.sys,
"argv",
["script.py", "--nginx-config-dir", "/etc/nginx", "--image", "img:tag"],
),
self.assertRaises(SystemExit) as cm,
):
script.main()
self.assertEqual(cm.exception.code, 0)
mock_run_checker.assert_not_called()
@patch("script.run_checker")
@patch("script.build_urls_from_nginx_confs")
@patch("script.extract_domains_from_filenames")
def test_main_passes_defaults_and_exits_with_run_checker_rc(
self,
mock_extract: MagicMock,
mock_build_urls: MagicMock,
mock_run_checker: MagicMock,
) -> None:
mock_extract.return_value = ["example.com", "api.example.com"]
mock_build_urls.return_value = [
"http://example.com/",
"https://api.example.com/",
]
mock_run_checker.return_value = 5
with (
patch.object(
script.sys,
"argv",
[
"script.py",
"--nginx-config-dir",
"/etc/nginx",
"--image",
"img:tag",
"--short",
"--ignore-network-blocks-from",
"pxscdn.com",
"cdn.example.org",
],
),
self.assertRaises(SystemExit) as cm,
):
script.main()
self.assertEqual(cm.exception.code, 5)
mock_run_checker.assert_called_once()
kwargs = mock_run_checker.call_args.kwargs
self.assertEqual(kwargs["image"], "img:tag")
self.assertEqual(
kwargs["urls"], ["http://example.com/", "https://api.example.com/"]
)
self.assertTrue(kwargs["short_mode"])
self.assertEqual(
kwargs["ignore_network_blocks_from"], ["pxscdn.com", "cdn.example.org"]
)
self.assertFalse(kwargs["always_pull"])
self.assertTrue(kwargs["use_host_network"])
@patch("script.run_checker")
@patch("script.build_urls_from_nginx_confs")
@patch("script.extract_domains_from_filenames")
def test_main_no_host_network_flag_disables_host_network(
self,
mock_extract: MagicMock,
mock_build_urls: MagicMock,
mock_run_checker: MagicMock,
) -> None:
mock_extract.return_value = ["example.com"]
mock_build_urls.return_value = ["http://example.com/"]
mock_run_checker.return_value = 0
with (
patch.object(
script.sys,
"argv",
[
"script.py",
"--nginx-config-dir",
"/etc/nginx",
"--image",
"img:tag",
"--no-host-network",
],
),
self.assertRaises(SystemExit) as cm,
):
script.main()
self.assertEqual(cm.exception.code, 0)
kwargs = mock_run_checker.call_args.kwargs
self.assertFalse(kwargs["use_host_network"])
@patch("script.run_checker")
@patch("script.build_urls_from_nginx_confs")
@patch("script.extract_domains_from_filenames")
def test_main_exits_0_when_no_urls_built(
self,
mock_extract: MagicMock,
mock_build_urls: MagicMock,
mock_run_checker: MagicMock,
) -> None:
mock_extract.return_value = ["example.com"]
mock_build_urls.return_value = []
with (
patch.object(
script.sys,
"argv",
["script.py", "--nginx-config-dir", "/etc/nginx", "--image", "img:tag"],
),
self.assertRaises(SystemExit) as cm,
):
script.main()
self.assertEqual(cm.exception.code, 0)
mock_run_checker.assert_not_called()
if __name__ == "__main__":
unittest.main()
Reference in new issue View Git Blame Copy Permalink