diff --git a/cli/administration/deploy/ci/runs.py b/cli/administration/deploy/ci/runs.py index aed5d89194..2341e5577e 100644 --- a/cli/administration/deploy/ci/runs.py +++ b/cli/administration/deploy/ci/runs.py @@ -24,9 +24,11 @@ import re import subprocess import sys +from utils.cache import PROJECT_ROOT +from utils.cache.yaml import load_yaml_any from utils.distros import distro_names from utils.github import run_name -from utils.github.variant import axes +from utils.github.variant import axes, selection from utils.roles.display import display_names from utils.symbol_glossary import to_emoji @@ -38,6 +40,38 @@ MISSING = "➖" MODES = ("docker", "swarm", "host") +ENTRY_WORKFLOW = ".github/workflows/entry-manual-steer.yml" + +SELECTION_INPUTS = ("priority",) +"""The one input a retrigger decides itself: it IS the retrigger. Everything +else the source run was dispatched with is carried over verbatim -- including +``whitelist``, so a retrigger of a scoped run stays inside that scope instead +of quietly widening to the whole repository.""" + + +def dispatch_inputs() -> tuple[str, ...]: + """Every ``workflow_dispatch`` input the manual entry declares. + + Read from the workflow rather than listed here, so an input added to the + form is carried over without anyone remembering to teach this module about + it -- the failure mode otherwise is silent: the retrigger runs with that + input on its default and nothing says so. + """ + data = load_yaml_any(str(PROJECT_ROOT / ENTRY_WORKFLOW), default_if_missing={}) or {} + triggers = data.get("on") if isinstance(data.get("on"), dict) else data.get(True) + dispatch = triggers.get("workflow_dispatch") if isinstance(triggers, dict) else None + inputs = dispatch.get("inputs") if isinstance(dispatch, dict) else None + return tuple(inputs or ()) + + +def carried_inputs() -> tuple[str, ...]: + """The inputs a retrigger reproduces: every dispatch input except the + selection it computes itself.""" + return tuple( + name for name in dispatch_inputs() if name not in SELECTION_INPUTS + ) + + CONFIG_INPUTS = ( "distros", "mode", @@ -180,6 +214,46 @@ def failed_roles( return sorted(app for app, modes in statuses.items() if fails(modes)) +def failed_selections(jobs: list[dict], *, strict: bool = False) -> list[str]: + """The selection tokens that reproduce exactly what did not pass. + + A role aggregated to its id loses what actually broke: the retrigger then + redeploys every variant of it, in whatever mode and onion state the sweep + rotation happens to pick, and the combination that failed may not be among + them. Each failed job therefore contributes its own + ``role#variant@mode+tor`` token (:mod:`utils.github.variant.selection`), so + the priority line replays that job and nothing else. + + Every mode is read; there is no scope to narrow to. A run that failed in + swarm and in compose comes back as two tokens for the same role. + + Args: + jobs: the source run's jobs. + strict: only hard failures (❌) count; cancelled, timed out and still + running are left out. + + Returns: + sorted, deduplicated tokens. + """ + tokens = set() + for app, _mode, job in _iter_deploy_jobs(jobs): + state = _effective(job) + if state == "success" or (strict and state != "failure"): + continue + label = axes.parse_label(str(job.get("name", ""))) + tokens.add( + selection.describe( + selection.Pin( + app, + tuple(int(part) for part in label.variant.split(",") if part), + label.mode, + label.tor, + ) + ) + ) + return sorted(tokens) + + def run_id_from_url(url: str) -> str: match = re.search(r"/runs/(\d+)", url) if not match: @@ -405,22 +479,30 @@ def config_from_title(title: str) -> dict[str, str]: def config_from_run( title: str, jobs: list[dict], logged: dict[str, str] | None = None ) -> dict[str, str]: - """The source run's configuration, with the distros the title does not - record recovered from its jobs (:func:`distros_from_jobs`). + """Every input the source run was dispatched with, except the selection. + + The job log is the source: it records all inputs verbatim, including the + ones the title renders as a glyph and the ones it renders not at all. The + title fills the gaps when a log is unreadable, and the distros -- which a + randomly picked run records nowhere -- come from the job names. + + An input the source left on its default resolves to empty and is dropped, + so the retrigger leaves it on that same default rather than pinning + today's default into a run that never asked for it. Args: title: the source run's display title. jobs: the source run's jobs. logged: inputs read verbatim from a called job's log - (:func:`inputs_from_jobs`); :data:`LOG_INPUTS` are taken from here - instead of from the title. + (:func:`inputs_from_jobs`). """ - config = config_from_title(title) + recorded = run_name.values_from_title(title) + config = { + name: (logged or {}).get(name) or recorded.get(name, "") + for name in carried_inputs() + } if not config.get("distros"): - config = {**config, "distros": distros_from_jobs(jobs)} - config.update( - {name: (logged or {}).get(name, "") for name in LOG_INPUTS}, - ) + config["distros"] = distros_from_jobs(jobs) return {name: value for name, value in config.items() if value} diff --git a/cli/administration/deploy/ci/trigger/__main__.py b/cli/administration/deploy/ci/trigger/__main__.py index 4fb62830ac..937706700d 100644 --- a/cli/administration/deploy/ci/trigger/__main__.py +++ b/cli/administration/deploy/ci/trigger/__main__.py @@ -2,6 +2,12 @@ entry-manual-steer.yml reads the "__ALL__" whitelist sentinel as "force a full deploy across all roles". + +A retrigger differs from its source run in the selection and in nothing else: +every other dispatch input is carried over (:func:`runs.carried_inputs`, read +off the workflow itself), and the priority line names the exact selections that +failed -- role, variant, deploy mode and onion state -- rather than the roles +they belong to. """ from __future__ import annotations @@ -39,16 +45,16 @@ def main(argv: list[str] | None = None) -> int: group.add_argument( "--failed", nargs="?", - const="total", + const=True, default=None, - choices=("total", "swarm", "compose", "docker", "host"), - metavar="{total,swarm,compose,host}", + metavar="(ignored)", help=( - "Re-trigger roles that were not green in the source run as the " - "priority line, together with the priority roles that run never " - "deployed at all; the remaining roles follow once they succeed. " - "Optional scope: 'total' (default; failed in any mode), " - "'swarm', 'compose', or 'host'." + "Re-trigger what was not green in the source run as the priority " + "line, together with the priority entries that run never deployed " + "at all; the remaining roles follow once they succeed. Every mode " + "is read, and each failed job comes back as the exact selection " + "that failed -- variant, deploy mode and onion state included. A " + "leftover scope argument is accepted and ignored." ), ) group.add_argument( @@ -106,14 +112,13 @@ def main(argv: list[str] | None = None) -> int: p.error("--apps was empty") whitelist = apps elif args.failed is not None: - scope = "docker" if args.failed == "compose" else args.failed statuses = runs.parse_role_statuses(source["jobs"]) - failed = runs.failed_roles(statuses, scope, strict=args.strict) + failed = runs.failed_selections(source["jobs"], strict=args.strict) untriggered = runs.untriggered_priority( runs.dispatched_priority(source, repo), statuses ) if not failed and not untriggered: - print(f"Nothing failed ({args.failed}) in that run; not triggering.") + print("Nothing failed in that run; not triggering.") return 0 if untriggered: print(f"Priority roles that never deployed: {' '.join(untriggered)}") @@ -128,6 +133,9 @@ def main(argv: list[str] | None = None) -> int: source["jobs"], runs.inputs_from_jobs(source["jobs"], repo), ) + carried_whitelist = config.pop("whitelist", "") + if not whitelist and carried_whitelist: + whitelist = carried_whitelist if priority: label = f"priority {priority}, then the remaining roles" diff --git a/tests/integration/meta/workflows/test_ci_retrigger_inputs.py b/tests/integration/meta/workflows/test_ci_retrigger_inputs.py new file mode 100644 index 0000000000..1d0ffa1e67 --- /dev/null +++ b/tests/integration/meta/workflows/test_ci_retrigger_inputs.py @@ -0,0 +1,144 @@ +"""A retrigger reproduces its source run in every input but the selection. + +``infinito administration deploy ci trigger --failed`` (the ``i8cifail`` +family) re-dispatches ``entry-manual-steer.yml``. Two things have to hold, and +neither is visible in a green run: + +* **Every dispatch input travels.** An input the source run set and the + retrigger drops silently falls back to the workflow default, so the rerun + deploys under a configuration nobody chose -- a swarm-only run coming back as + a full rotation, a tor-enforced run coming back on the clearnet. The carried + set is therefore derived from the workflow itself, and this test walks the + same declaration: adding an input to the form without teaching the trigger + fails here rather than in a run three hours later. +* **The priority line names what actually failed.** Aggregated to a role id, a + retrigger redeploys whichever variant/mode/onion combination the rotation + picks next, which need not be the one that broke. Each failed job comes back + as its own selection token instead. +""" + +from __future__ import annotations + +import io +import unittest +import unittest.mock as mock +from contextlib import redirect_stdout + +from cli.administration.deploy.ci import runs +from cli.administration.deploy.ci.trigger import __main__ as trigger +from tests.utils.ci.job_names import deploy_job_name + +_REPO = "o/r" +_BRANCH = "feature/x" +_RUN_URL = "https://github.com/o/r/actions/runs/55" # nocheck: url + +_VALUES = { + "distros": "arch debian", + "priority": "web-app-seed", + "whitelist": "web-app-x web-app-y", + "lifecycles": "stable", + "mode": "swarm", + "filesystem": "btrfs", + "tor": "enforced", + "resume_from_chunk": "2", + "chunk_gate": "false", + "sweep": "7", + "workspace": "true", + "workspace_track": "compose", + "instructions": "false", +} +"""A value for every dispatch input, distinct from that input's default so a +dropped one shows up as a missing key rather than as a coincidence.""" + +_JOBS = [ + { + "name": deploy_job_name("swarm", "web-app-x", "0", tor=True), + "status": "completed", + "conclusion": "failure", + }, + { + "name": deploy_job_name("docker", "web-app-y", "1"), + "status": "completed", + "conclusion": "success", + }, +] + + +def _dispatch(argv: list[str], logged: dict[str, str]) -> tuple[str, str, dict]: + """Run the trigger against a source run whose log records *logged*.""" + seen: list[tuple[str, str, dict]] = [] + source = {"jobs": _JOBS, "displayTitle": ""} + with ( + mock.patch.object(runs, "current_branch", return_value=_BRANCH), + mock.patch.object(runs, "resolve_repo", return_value=_REPO), + mock.patch.object(runs, "fetch_run", return_value=source), + mock.patch.object(runs, "inputs_from_jobs", return_value=logged), + mock.patch.object( + runs, + "dispatch_workflow", + side_effect=lambda wf, ref, wl="", priority="", config=None, repo=None: ( + seen.append((wl, priority, config or {})) + ), + ), + redirect_stdout(io.StringIO()), + ): + assert trigger.main(argv) == 0 + assert seen, "the trigger dispatched nothing" + return seen[0] + + +class TestCarriedInputs(unittest.TestCase): + def test_the_carried_set_is_the_workflow_minus_the_priority_line(self) -> None: + declared = set(runs.dispatch_inputs()) + self.assertTrue(declared, "entry-manual-steer.yml declares no inputs") + self.assertEqual(set(runs.carried_inputs()), declared - {"priority"}) + + def test_the_test_fixture_covers_every_declared_input(self) -> None: + self.assertEqual(set(_VALUES), set(runs.dispatch_inputs())) + + def test_every_carried_input_reaches_the_dispatch_unchanged(self) -> None: + whitelist, _priority, config = _dispatch( + ["--failed", "--run", _RUN_URL], _VALUES + ) + sent = {**config, "whitelist": whitelist} + for name in runs.carried_inputs(): + with self.subTest(input=name): + self.assertEqual(sent.get(name), _VALUES[name]) + + def test_the_priority_line_is_not_carried_but_recomputed(self) -> None: + _whitelist, priority, config = _dispatch( + ["--failed", "--run", _RUN_URL], _VALUES + ) + self.assertNotIn("priority", config) + self.assertNotEqual(priority, _VALUES["priority"]) + + def test_an_input_the_source_left_on_its_default_is_not_invented(self) -> None: + _whitelist, _priority, config = _dispatch( + ["--failed", "--run", _RUN_URL], {"tor": "exclusive"} + ) + self.assertEqual(config.get("tor"), "exclusive") + self.assertNotIn("mode", config) + + +class TestRecomputedSelection(unittest.TestCase): + def test_the_priority_line_replays_the_exact_failed_selection(self) -> None: + _whitelist, priority, _config = _dispatch( + ["--failed", "--run", _RUN_URL], {"priority": ""} + ) + self.assertEqual(priority, "web-app-x#0@swarm+tor") + + def test_a_green_selection_of_the_same_run_is_left_alone(self) -> None: + _whitelist, priority, _config = _dispatch( + ["--failed", "--run", _RUN_URL], {"priority": ""} + ) + self.assertNotIn("web-app-y", priority) + + def test_the_scope_of_the_source_run_survives_the_retrigger(self) -> None: + whitelist, _priority, _config = _dispatch( + ["--failed", "--run", _RUN_URL], _VALUES + ) + self.assertEqual(whitelist, _VALUES["whitelist"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/cli/administration/deploy/ci/test_trigger.py b/tests/unit/cli/administration/deploy/ci/test_trigger.py index c58f29fea1..9581e30666 100644 --- a/tests/unit/cli/administration/deploy/ci/test_trigger.py +++ b/tests/unit/cli/administration/deploy/ci/test_trigger.py @@ -27,6 +27,10 @@ _JOBS = [ _job("swarm", "web-app-y", "success"), ] +_FAILED_TOKENS = "web-app-x#0,1@swarm+clearnet web-app-y#0,1@compose+clearnet" +"""What _JOBS failed at, exactly: x in swarm, y in compose, both on variant +shard 0,1 and both on the clearnet.""" + _RUN_URL = "https://github.com/o/r/actions/runs/55" # nocheck: url _SOURCE_CONFIG = { "distros": "arch centos", @@ -82,15 +86,24 @@ class TestTriggerMain(unittest.TestCase): rc, calls = self._run(["--failed"], run={"_jobs": _JOBS}) self.assertEqual(rc, 0) self.assertEqual(calls[0][2], "") - self.assertEqual(calls[0][3], "web-app-x web-app-y") + self.assertEqual(calls[0][3], _FAILED_TOKENS) - def test_failed_swarm_scope(self) -> None: - _rc, calls = self._run(["--failed", "swarm"], run={"_jobs": _JOBS}) - self.assertEqual(calls[0][3], "web-app-x") + def test_a_leftover_scope_argument_no_longer_narrows_the_modes(self) -> None: + for scope in ("swarm", "compose", "total"): + with self.subTest(scope=scope): + _rc, calls = self._run(["--failed", scope], run={"_jobs": _JOBS}) + self.assertEqual(calls[0][3], _FAILED_TOKENS) - def test_failed_compose_scope(self) -> None: - _rc, calls = self._run(["--failed", "compose"], run={"_jobs": _JOBS}) - self.assertEqual(calls[0][3], "web-app-y") + def test_every_failed_mode_of_one_role_comes_back_separately(self) -> None: + both = [ + _job("docker", "web-app-x", "failure"), + _job("swarm", "web-app-x", "failure"), + ] + _rc, calls = self._run(["--failed"], run={"_jobs": both}) + self.assertEqual( + calls[0][3], + "web-app-x#0,1@compose+clearnet web-app-x#0,1@swarm+clearnet", + ) def test_never_deployed_priority_roles_join_the_retrigger(self) -> None: source = {"jobs": _JOBS, "displayTitle": render(_SOURCE_CONFIG)} @@ -115,7 +128,7 @@ class TestTriggerMain(unittest.TestCase): ): rc = trigger.main(["--failed", "--run", _RUN_URL]) self.assertEqual(rc, 0) - self.assertEqual(calls[0], "web-app-never web-app-x web-app-y") + self.assertEqual(calls[0], "web-app-never web-app-x#0,1@swarm+clearnet web-app-y#0,1@compose+clearnet") def test_an_unreadable_job_log_aborts_instead_of_dropping_the_priority( self, @@ -190,7 +203,7 @@ class TestTriggerMain(unittest.TestCase): ): rc = trigger.main(["--failed", "--run", _RUN_URL]) self.assertEqual(rc, 0) - self.assertEqual(calls[0], ("web-app-x web-app-y", _SOURCE_CONFIG)) + self.assertEqual(calls[0], (_FAILED_TOKENS, _SOURCE_CONFIG)) fetch.assert_called_once() find_last.assert_not_called() @@ -212,7 +225,7 @@ class TestTriggerMain(unittest.TestCase): ): rc = trigger.main(["--failed", "--run", "55"]) self.assertEqual(rc, 0) - self.assertEqual(calls[0], ("web-app-x web-app-y", _SOURCE_CONFIG)) + self.assertEqual(calls[0], (_FAILED_TOKENS, _SOURCE_CONFIG)) fetch.assert_called_once_with("55", repo="o/r") find_last.assert_not_called()