From 2d0016d27be459e2f82862cd830e633d0b48493f Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 28 Oct 2026 22:39:00 +0100 Subject: [PATCH] fix(svc-net-tor): let a port probe attempt outlast Tor's SocksTimeout The onion port probe gave every SOCKS connect 60s, half of Tor's SocksTimeout (120s by default; torrc does not set it), although its own docstring argued that abandoning an attempt before SocksTimeout is destructive. Every attempt authenticates with fresh credentials, so Tor builds a whole new rendezvous for it and keeps working on it until SocksTimeout. The probe walked away first and called the port dead. The other raw onion connects in the repository, the svc-db-* CLI tests and test-e2e-playwright's wait_onion_reachable.sh, wait 120s per attempt. Run 40128751852 shows how often an attempt needs more than that. Of 178 svc-net-tor CLI runs in its 89 compose onion jobs, three lost their first attempt to the 60s timeout and connected on the second within seconds (Shopware#1, Hugo#1, n8n#1; the CLI test took 77-79s against the usual 1-16s), and Hugo#1's other run took 40s on a single attempt. When the stall outlasts the 70s until the second attempt, the probe fails while the HTTP probe reaches the same onion port seconds earlier: OpenBao#1 (run 39166850273, job 118050491920, the original failure of #1048) and Socialhome#1 (run 40088473634, job 121019758461). In the Socialhome job tor.log holds no line for either attempt, because Tor never reached its own verdict. ONION_PORT_TIMEOUT is now 130: SocksTimeout plus room for Tor's once-per-second expiry sweep, so an attempt ends with Tor's own answer (reply 0x06, and a "Giving up" notice in tor.log, which the rescue artifact captures) instead of the probe's. The second attempt now starts at 140s and ends at 270s, so the probe rides out every stall the old budget did, and any stall shorter than about 260s instead of about 120s. A green run still costs one connect per port, and a refused or unmapped port still fails at once. A unit test pins the window above Tor's SocksTimeout and fails as soon as torrc sets its own. That doubles what a dead port costs, and meta/tests.yml caps the whole CLI test. The largest compose onion deploy in run 40128751852 probes 19 domains and 5 ports; with everything dead that is 19 x 350 + 5 x 260 + 10 = 7960s, beyond the 7200s that 069df39027 set for the same reason. 9000 keeps the full failure output for up to 22 domains. Closes #1214 Co-Authored-By: Claude Opus 5.5 --- roles/svc-net-tor/files/test/onion_ports.py | 15 ++++---- roles/svc-net-tor/meta/tests.yml | 2 +- roles/svc-net-tor/templates/test.env.j2 | 2 +- .../svc-net-tor/test_onion_port_window.py | 38 +++++++++++++++++++ 4 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 tests/unit/python/roles/svc-net-tor/test_onion_port_window.py diff --git a/roles/svc-net-tor/files/test/onion_ports.py b/roles/svc-net-tor/files/test/onion_ports.py index adb6a81c66..7b06449a63 100755 --- a/roles/svc-net-tor/files/test/onion_ports.py +++ b/roles/svc-net-tor/files/test/onion_ports.py @@ -25,12 +25,13 @@ Env (rendered into test.env from templates/test.env.j2): ONION_HOST the node onion host ONION_PORTS space-separated forwarded ports; empty means there is nothing to check - ONION_PORT_TIMEOUT seconds per connect attempt. 60 for the same reason the - domain probe uses it: Tor's SocksTimeout default is 120s - and abandoning earlier is destructive, because the next - attempt opens a fresh SOCKS connection and throws away - the descriptor fetches and rendezvous retries the last - one was still working through + ONION_PORT_TIMEOUT seconds per connect attempt, above Tor's SocksTimeout + (120s by default; torrc does not set it). Every attempt + builds a fresh rendezvous, which Tor keeps working on + until SocksTimeout and then ends itself with reply 0x06 + and a "Giving up" notice in its log. An attempt abandoned + earlier throws away a rendezvous Tor may still complete + and leaves Tor's log without a trace of the failure ONION_PORT_RETRIES passes over the failing ports. Below the domain probe's RETRIES because this phase multiplies by the port count, and meta/tests.yml caps the whole CLI test @@ -148,7 +149,7 @@ def main() -> int: retries = int(_required("ONION_PORT_RETRIES", "passes over the failing ports")) sleep_seconds = float(_required("SLEEP_SECONDS", "wait between passes")) timeout = float( - _required("ONION_PORT_TIMEOUT", "seconds per connect, flavor-dependent") + _required("ONION_PORT_TIMEOUT", "seconds per connect, above Tor's SocksTimeout") ) ports = [int(entry) for entry in os.environ.get("ONION_PORTS", "").split()] diff --git a/roles/svc-net-tor/meta/tests.yml b/roles/svc-net-tor/meta/tests.yml index 272c1e148d..036f2dec6c 100644 --- a/roles/svc-net-tor/meta/tests.yml +++ b/roles/svc-net-tor/meta/tests.yml @@ -1,3 +1,3 @@ --- cli: - timeout: 7200 + timeout: 9000 diff --git a/roles/svc-net-tor/templates/test.env.j2 b/roles/svc-net-tor/templates/test.env.j2 index 51288356b2..0788bce017 100644 --- a/roles/svc-net-tor/templates/test.env.j2 +++ b/roles/svc-net-tor/templates/test.env.j2 @@ -2,7 +2,7 @@ NGINX_SERVERS_DIR={{ lookup('nginx', 'directories.configuration.servers') }} ONION_HOST={{ lookup('config', 'svc-net-tor', 'services.tor.node') }} ONION_PORTS={{ lookup('tor_ports') | map(attribute='onion_port') | unique | sort | join(' ') | dotenv_quote }} ONION_PORT_RETRIES=2 -ONION_PORT_TIMEOUT=60 +ONION_PORT_TIMEOUT=130 RETRIES=5 SLEEP_SECONDS=10 TOR_CONTAINER={{ lookup('container_address', application_id, 'tor') }} diff --git a/tests/unit/python/roles/svc-net-tor/test_onion_port_window.py b/tests/unit/python/roles/svc-net-tor/test_onion_port_window.py new file mode 100644 index 0000000000..0072ac161e --- /dev/null +++ b/tests/unit/python/roles/svc-net-tor/test_onion_port_window.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import re +import unittest + +from utils.cache.files import read_text + +from . import PROJECT_ROOT + +ROLE = PROJECT_ROOT / "roles" / "svc-net-tor" +TEST_ENV = ROLE / "templates" / "test.env.j2" +TORRC = ROLE / "templates" / "torrc.j2" +TOR_DEFAULT_SOCKS_TIMEOUT = 120 + + +class TestOnionPortWindow(unittest.TestCase): + def test_an_attempt_outlasts_tors_socks_timeout(self) -> None: + self.assertNotRegex( + read_text(str(TORRC)), + r"(?m)^SocksTimeout\b", + "torrc now sets SocksTimeout; ONION_PORT_TIMEOUT has to be derived " + "from it instead of from Tor's default", + ) + declared = re.search( + r"(?m)^ONION_PORT_TIMEOUT=(\d+)$", read_text(str(TEST_ENV)) + ) + self.assertIsNotNone(declared, "test.env.j2 declares no ONION_PORT_TIMEOUT") + self.assertGreater( + int(declared.group(1)), + TOR_DEFAULT_SOCKS_TIMEOUT, + "a port probe attempt that ends before Tor's SocksTimeout walks away " + "from a rendezvous Tor is still building, and Tor never reaches its " + "own verdict or logs it", + ) + + +if __name__ == "__main__": + unittest.main()