fix(ci): let a cancel reap jobs guarded by always()

Closing a PR left "Cancel: PR Runs on Close" (run 36385268627) on
pending for hours. The canceller joins the PR orchestrator's
concurrency group, so GitHub cancelled the orchestrator run within a
second. But a cancel re-evaluates the job-level if: of every live job
and spares each one whose condition still holds. The deploy chunks
started with always(), which always holds: test-workspace died at
06:11:59, while Chunk 0 kept deploying and its ~60 queued matrix jobs
kept waiting for runners. The group stays occupied until the occupant
is terminal, so the canceller parked behind the very run it had
already cancelled.

The chunks, test-runner-smoke, the fork-prereqs gate and the release
job now guard with !cancelled(). Outside a cancel it matches always():
they still run after a failed or skipped need. On a cancel it turns
false and the job is cancelled with the run.

done and report-main-failures keep always(). They exist to report a
cancelled run, and a done skipped on cancel would count as a passing
required check.

The new lint walks every workflow whose concurrency cancels in
progress, plus everything it calls transitively, and rejects a
job-level always() outside that allowlist. It found the release job,
which a line-based grep had missed. It also fails when an allowlisted
job stops needing the exemption, and when its scope no longer reaches
the deploy workflows, so it cannot go silently empty. Against the
pre-fix tree it reports all six offenders.

Behaviour change: manually cancelling a release run now stops the
retag too. cron-release-highest re-dispatches the release.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

.github/workflows/README.md

@@ -287,6 +287,11 @@ belong to the entry's group, and the entry's `cancel-in-progress` reaches them.
 one, so a run whose only live job is theirs stays outside the entry's group and
 waits for the fallback.
 
+A job that runs after a failed `needs` guards its `if:` with `!cancelled()`, not
+`always()`, so a cancel stops it. Only `done` and `report-main-failures` keep
+`always()`: they report the cancelled run. Enforced by
+[test_workflow_cancellable_jobs_no_always.py](../../tests/lint/repository/test_workflow_cancellable_jobs_no_always.py).
+
 ## Scheduled and standalone
 
 ```mermaid

.github/workflows/call-orchestrator.yml

@@ -161,7 +161,7 @@ jobs:
   test-deploy-chunk-0:
     name: 🧱 Chunk 0
     if: >-
-      ${{ always()
+      ${{ !cancelled()
           && inputs.resume_from_chunk <= 0
           && needs.lint.result == 'success'
           && needs.test.result == 'success'
@@ -190,7 +190,7 @@ jobs:
   test-deploy-chunk-1:
     name: 🧱 Chunk 1
     if: >-
-      ${{ always()
+      ${{ !cancelled()
           && inputs.resume_from_chunk <= 1
           && needs.lint.result == 'success'
           && needs.test.result == 'success'
@@ -222,7 +222,7 @@ jobs:
   test-deploy-chunk-2:
     name: 🧱 Chunk 2
     if: >-
-      ${{ always()
+      ${{ !cancelled()
           && inputs.resume_from_chunk <= 2
           && needs.lint.result == 'success'
           && needs.test.result == 'success'
@@ -256,7 +256,7 @@ jobs:
   test-deploy-chunk-3:
     name: 🧱 Chunk 3
     if: >-
-      ${{ always()
+      ${{ !cancelled()
           && inputs.resume_from_chunk <= 3
           && needs.lint.result == 'success'
           && needs.test.result == 'success'
@@ -292,7 +292,7 @@ jobs:
   test-runner-smoke:
     needs:
       - test-deploy-chunk-0
-    if: ${{ always() && vars.CI_SELF_HOSTED_RUNNER_COUNT != '0' && vars.CI_SELF_HOSTED_RUNNER_COUNT != '' && (needs.test-deploy-chunk-0.result == 'success' || needs.test-deploy-chunk-0.result == 'failure') }}
+    if: ${{ !cancelled() && vars.CI_SELF_HOSTED_RUNNER_COUNT != '0' && vars.CI_SELF_HOSTED_RUNNER_COUNT != '' && (needs.test-deploy-chunk-0.result == 'success' || needs.test-deploy-chunk-0.result == 'failure') }}
     uses: ./.github/workflows/call-test-runner-smoke.yml
     with:
       ref: ${{ github.sha }}

.github/workflows/call-release-version.yml

@@ -117,7 +117,7 @@ jobs:
       - check-ci-images
       - build-ci-images-if-missing
     if: |
-      always() &&
+      !cancelled() &&
       needs.check-ci-images.result == 'success' &&
       (
         needs.build-ci-images-if-missing.result == 'success' ||

.github/workflows/call-validation.yml

@@ -193,7 +193,7 @@ jobs:
   fork-prereqs-ready:
     name: 🚦 Gate Dependencies
     needs: wait-fork-prereq-run
-    if: ${{ always() && inputs.fork_prereqs }}
+    if: ${{ !cancelled() && inputs.fork_prereqs }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7.0.0

tests/lint/repository/test_workflow_cancellable_jobs_no_always.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import re
+import unittest
+
+from utils.cache.yaml import load_yaml_any
+
+from . import PROJECT_ROOT
+
+_WORKFLOWS_DIR = PROJECT_ROOT / ".github" / "workflows"
+_LOCAL_PREFIX = "./.github/workflows/"
+_ALWAYS = re.compile(r"\balways\(\s*\)")
+
+_REPORTERS = frozenset(
+    {
+        ("call-orchestrator.yml", "done"),
+        ("call-orchestrator.yml", "report-main-failures"),
+    }
+)
+
+
+def _load(path):
+    return load_yaml_any(str(path), default_if_missing={}) or {}
+
+
+def _jobs(document):
+    return {
+        job_id: job
+        for job_id, job in (document.get("jobs") or {}).items()
+        if isinstance(job, dict)
+    }
+
+
+def _cancels_in_progress(document) -> bool:
+    concurrency = document.get("concurrency")
+    return (
+        isinstance(concurrency, dict)
+        and concurrency.get("cancel-in-progress", False) is not False
+    )
+
+
+def _cancellable(documents) -> set[str]:
+    pending = [name for name, doc in documents.items() if _cancels_in_progress(doc)]
+    seen: set[str] = set()
+    while pending:
+        name = pending.pop()
+        if name in seen or name not in documents:
+            continue
+        seen.add(name)
+        pending += [
+            job["uses"][len(_LOCAL_PREFIX) :]
+            for job in _jobs(documents[name]).values()
+            if str(job.get("uses", "")).startswith(_LOCAL_PREFIX)
+        ]
+    return seen
+
+
+def _uses_always(job) -> bool:
+    return bool(_ALWAYS.search(str(job.get("if", ""))))
+
+
+class TestWorkflowCancellableJobsNoAlways(unittest.TestCase):
+    def setUp(self) -> None:
+        self.documents = {
+            path.name: _load(path) for path in sorted(_WORKFLOWS_DIR.glob("*.yml"))
+        }
+        self.cancellable = _cancellable(self.documents)
+
+    def test_cancellable_jobs_do_not_use_always(self) -> None:
+        offenders = [
+            f"{name}:{job_id}: if: {' '.join(str(job['if']).split())}"
+            for name in sorted(self.cancellable)
+            for job_id, job in _jobs(self.documents[name]).items()
+            if (name, job_id) not in _REPORTERS and _uses_always(job)
+        ]
+        if offenders:
+            self.fail(
+                f"{len(offenders)} job(s) in a cancellable workflow guard with "
+                "always(). A cancel spares them, the run never ends, and the "
+                "run that cancelled it waits on 'pending' behind it. Use "
+                "!cancelled() instead:\n" + "\n".join(offenders)
+            )
+
+    def test_reporters_still_need_the_exemption(self) -> None:
+        stale = [
+            f"{name}:{job_id}"
+            for name, job_id in sorted(_REPORTERS)
+            if not _uses_always(_jobs(self.documents.get(name, {})).get(job_id, {}))
+        ]
+        self.assertEqual(
+            stale, [], "exempted jobs no longer use always(); drop them from _REPORTERS"
+        )
+
+    def test_the_scope_reaches_the_deploy_chunks(self) -> None:
+        self.assertLessEqual(
+            {"call-orchestrator.yml", "call-test-deploy.yml", "call-validation.yml"},
+            self.cancellable,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()