mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-08 19:56:53 +00:00
Two defects where a suite-budget kill is handled.
`SUITE_RESERVE_MS` exists so a waiter that runs out reports which predicate it
was waiting on, instead of letting the global timeout fire first and count the
in-flight test and everything behind it as "did not run". a490b28af6 set it to
60 s, but the cap it feeds is measured from the waiter's own start while the
global timeout runs from the suite's, so the reserve must also absorb every
test that ran before the waiter plus the post-failure trace and report write.
Measured prefixes in run 36651124458 were 3.5 s, 32.4 s and 86.2 s; at 86.2 s
the cap landed 26 s past the global timeout and OpenProject#1 kept only a
screenshot, no error-context. 180 s leaves roughly double the largest prefix
seen.
No waiter changes category: the newly cut band, between the old cap and the
new one, is empty on both budgets. The already-capped ones lose 120 s of about
5.28 M, and the only one that reached its cap was failing at the old value too.
The retry at orchestrate.yml exists for a Tor circuit flake. A run killed by
the global timeout fails the same way every time, so it burned a second full
budget for an identical result, up to 80% of some jobs' wall clock. Such a role
is now recorded and excluded from the retry loop.
The exclusion is only safe because the stash keeps it. That task reset
`_playwright_failures` to an empty list and only a retried role re-adds itself,
so dropping a role from the retry while leaving that reset would have removed
it from the final fail task and turned a red job green. The intersect keeps
exactly the budget timeouts in, and a test renders the task's own expressions to
hold that: the regression it guards against is a wrong set operation, not wrong
wording. Reverting the reset to `[]` makes it report `[] != ['web-app-docs']`.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
157 lines
5.2 KiB
Python
157 lines
5.2 KiB
Python
"""Unit tests for roles/test-e2e-playwright/files/timeouts.js.
|
|
|
|
The helper is dependency-free plain CommonJS. We copy it into a temp dir
|
|
and call it from a short Node script with controlled env, then assert the
|
|
computed timeout / onion detection.
|
|
"""
|
|
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import textwrap
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from . import PROJECT_ROOT
|
|
|
|
HELPER_PATH = str(
|
|
PROJECT_ROOT / "roles" / "test-e2e-playwright" / "files" / "timeouts.js"
|
|
)
|
|
|
|
|
|
def _have_node():
|
|
return shutil.which("node") is not None
|
|
|
|
|
|
@unittest.skipUnless(_have_node(), "node is not available in PATH")
|
|
class TestTimeoutsHelper(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmpdir = tempfile.mkdtemp(prefix="timeouts-helper-")
|
|
shutil.copy(HELPER_PATH, str(Path(self.tmpdir) / "timeouts.js"))
|
|
|
|
def tearDown(self):
|
|
shutil.rmtree(self.tmpdir, ignore_errors=True)
|
|
|
|
def _run(self, expr, env):
|
|
script = textwrap.dedent(
|
|
f"""
|
|
const t = require("./timeouts");
|
|
process.stdout.write("RESULT:" + JSON.stringify({expr}) + "\\n");
|
|
"""
|
|
)
|
|
script_path = str(Path(self.tmpdir) / "run.js")
|
|
with Path(script_path).open("w") as f:
|
|
f.write(script)
|
|
proc = subprocess.run(
|
|
["node", "run.js"],
|
|
capture_output=True,
|
|
text=True,
|
|
env={**os.environ, **env},
|
|
cwd=self.tmpdir,
|
|
timeout=10,
|
|
check=False,
|
|
)
|
|
self.assertEqual(
|
|
proc.returncode, 0, msg=f"stderr={proc.stderr}\nstdout={proc.stdout}"
|
|
)
|
|
return proc.stdout
|
|
|
|
def test_clearnet_default_factor_is_identity(self):
|
|
out = self._run(
|
|
"t.resolveTimeout(60000)",
|
|
{"CANONICAL_DOMAIN": '"x.infinito.test"'},
|
|
)
|
|
self.assertIn("RESULT:60000", out)
|
|
|
|
def test_onion_applies_default_multiplier(self):
|
|
out = self._run(
|
|
"t.resolveTimeout(60000)",
|
|
{"CANONICAL_DOMAIN": '"x.abc123.onion"'},
|
|
)
|
|
self.assertIn("RESULT:300000", out)
|
|
|
|
def test_global_factor_scales_clearnet(self):
|
|
out = self._run(
|
|
"t.resolveTimeout(60000)",
|
|
{
|
|
"CANONICAL_DOMAIN": "x.infinito.test",
|
|
"PLAYWRIGHT_TIMEOUT_FACTOR": "2",
|
|
},
|
|
)
|
|
self.assertIn("RESULT:120000", out)
|
|
|
|
def test_factor_and_onion_multiplier_compose(self):
|
|
out = self._run(
|
|
"t.resolveTimeout(60000)",
|
|
{
|
|
"CANONICAL_DOMAIN": '"x.abc123.onion"',
|
|
"PLAYWRIGHT_TIMEOUT_FACTOR": "2",
|
|
"PLAYWRIGHT_ONION_TIMEOUT_MULTIPLIER": "3",
|
|
},
|
|
)
|
|
self.assertIn("RESULT:360000", out)
|
|
|
|
def test_invalid_factor_falls_back_to_one(self):
|
|
out = self._run(
|
|
"t.resolveTimeout(60000)",
|
|
{
|
|
"CANONICAL_DOMAIN": "x.infinito.test",
|
|
"PLAYWRIGHT_TIMEOUT_FACTOR": "0",
|
|
},
|
|
)
|
|
self.assertIn("RESULT:60000", out)
|
|
|
|
def test_a_waiter_may_not_outlive_the_suite_that_contains_it(self):
|
|
"""The onion run that starved web-app-docs: base 2_700_000 at
|
|
multiplier 2 asked for 5_400_000 against a 5_340_000 suite budget, so
|
|
the global timeout fired first and every remaining spec was reported
|
|
"did not run" without naming the waiter."""
|
|
out = self._run(
|
|
"t.resolveTimeout(2700000) + ':' + t.SUITE_RESERVE_MS",
|
|
{
|
|
"CANONICAL_DOMAIN": '"docs.abc123.onion"',
|
|
"PLAYWRIGHT_TIMEOUT_FACTOR": "1",
|
|
"PLAYWRIGHT_ONION_TIMEOUT_MULTIPLIER": "2",
|
|
"INFINITO_PLAYWRIGHT_GLOBAL_TIMEOUT_MS": "5340000", # nocheck: the playwright role passes this into the sidecar container, it is not a .env key
|
|
},
|
|
)
|
|
capped, reserve = out.split("RESULT:")[1].strip().strip('"').split(":")
|
|
self.assertEqual(int(capped), 5340000 - int(reserve))
|
|
|
|
def test_a_timeout_inside_the_budget_is_left_alone(self):
|
|
out = self._run(
|
|
"t.resolveTimeout(30000)",
|
|
{
|
|
"CANONICAL_DOMAIN": '"docs.abc123.onion"',
|
|
"PLAYWRIGHT_ONION_TIMEOUT_MULTIPLIER": "2",
|
|
"INFINITO_PLAYWRIGHT_GLOBAL_TIMEOUT_MS": "5340000", # nocheck: the playwright role passes this into the sidecar container, it is not a .env key
|
|
},
|
|
)
|
|
self.assertIn("RESULT:60000", out)
|
|
|
|
def test_without_a_declared_budget_nothing_is_capped(self):
|
|
out = self._run(
|
|
"t.resolveTimeout(5400000)",
|
|
{"CANONICAL_DOMAIN": '"x.infinito.test"'},
|
|
)
|
|
self.assertIn("RESULT:5400000", out)
|
|
|
|
def test_is_onion_target_detects_quoted_and_bare(self):
|
|
self.assertIn(
|
|
"RESULT:true",
|
|
self._run("t.isOnionTarget()", {"CANONICAL_DOMAIN": '"x.abc.onion"'}),
|
|
)
|
|
self.assertIn(
|
|
"RESULT:true",
|
|
self._run("t.isOnionTarget()", {"CANONICAL_DOMAIN": "x.abc.onion"}),
|
|
)
|
|
self.assertIn(
|
|
"RESULT:false",
|
|
self._run("t.isOnionTarget()", {"CANONICAL_DOMAIN": '"x.infinito.test"'}),
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|