diff --git a/.github/workflows/README.md b/.github/workflows/README.md index e07b082b49..1c013893f4 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -266,10 +266,17 @@ the other entry points take the defaults. flowchart TB prclose["pull_request_target: closed, converted_to_draft"] --> eprcancel["entry-pr-closed-cancel-workflows.yml"] branchdelete["delete: branch"] --> delbranch["entry-delete-branch.yml"] + newcommit["push (not main) / pull_request_target: opened, synchronize, reopened, ready_for_review, labeled"] --> superseded["entry-cancel-superseded.yml"] eprcancel -.->|"cancels concurrency group"| runningci["running entry + child workflow runs"] delbranch -.->|"cancels concurrency group"| runningci + superseded -.->|"cancels via API what the group left behind"| runningci ``` +The concurrency group stays the primary mechanism; it reaps within seconds of a +run being created and needs no runner. `entry-cancel-superseded.yml` is the +fallback for the case where it does not, and carries no group of its own, +because the run holding a group cannot be the run that frees it. + ## Scheduled and standalone ```mermaid diff --git a/.github/workflows/entry-cancel-superseded.yml b/.github/workflows/entry-cancel-superseded.yml new file mode 100644 index 0000000000..a7f35c94fe --- /dev/null +++ b/.github/workflows/entry-cancel-superseded.yml @@ -0,0 +1,74 @@ +# Deliberately without a `concurrency:` block: this workflow frees the very +# groups that block other runs, so a group of its own would deadlock it. + +name: "🚫 Cancel: Superseded Runs" + +on: + push: + branches: + - "feature/**" + - "hotfix/**" + - "fix/**" + - "update/**" + - "chore/**" + - "alert-autofix-*" + pull_request_target: + types: + - opened + - synchronize + - reopened + - ready_for_review + - labeled + +permissions: + contents: read + +jobs: + cancel-branch-runs: + name: 🚫 Cancel superseded branch runs + if: ${{ github.event_name == 'push' && github.ref_name != 'main' }} + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + steps: + - name: ⬇️ Checkout repository + uses: actions/checkout@v7.0.0 + + - name: 🚫 Cancel branch runs superseded by a newer one + shell: bash + env: + BRANCH: ${{ github.ref_name }} + CURRENT_RUN_ID: ${{ github.run_id }} + KEEP_NEWEST_PER: all + INCLUDE_PATHS: | + .github/workflows/entry-push-latest.yml + .github/workflows/entry-manual-steer.yml + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPOSITORY: ${{ github.repository }} + run: bash scripts/github/cancel/branch_runs.sh + + cancel-pull-request-runs: + name: 🚫 Cancel superseded pull request runs + if: ${{ github.event_name == 'pull_request_target' }} + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + pull-requests: read + steps: + - name: ⬇️ Checkout repository + uses: actions/checkout@v7.0.0 + + - name: 🚫 Cancel pull request runs superseded by a newer one + shell: bash + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_HEAD_REF: ${{ github.event.pull_request.head.ref }} + PR_HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} + CURRENT_RUN_ID: ${{ github.run_id }} + KEEP_NEWEST_PER: event + INCLUDE_PATHS: .github/workflows/entry-pr-change-orchestrate.yml + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPOSITORY: ${{ github.repository }} + run: bash scripts/github/cancel/pull_request_runs.sh diff --git a/.github/workflows/entry-delete-branch.yml b/.github/workflows/entry-delete-branch.yml index 080b2b3234..66f1f9e538 100644 --- a/.github/workflows/entry-delete-branch.yml +++ b/.github/workflows/entry-delete-branch.yml @@ -32,4 +32,4 @@ jobs: BRANCH: ${{ github.event.ref }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPOSITORY: ${{ github.repository }} - run: bash scripts/github/cancel/deleted_branch_runs.sh + run: bash scripts/github/cancel/branch_runs.sh diff --git a/.github/workflows/entry-pr-closed-cancel-workflows.yml b/.github/workflows/entry-pr-closed-cancel-workflows.yml index 029568fc84..2fa6071c79 100644 --- a/.github/workflows/entry-pr-closed-cancel-workflows.yml +++ b/.github/workflows/entry-pr-closed-cancel-workflows.yml @@ -13,6 +13,7 @@ concurrency: permissions: actions: write contents: read + pull-requests: read jobs: cancel-pr-runs: diff --git a/scripts/github/cancel/README.md b/scripts/github/cancel/README.md index 41e6f0e5bc..a3e25df650 100644 --- a/scripts/github/cancel/README.md +++ b/scripts/github/cancel/README.md @@ -1,9 +1,33 @@ # Run Cancellation 🚫 -Shell helpers that cancel in-progress GitHub Actions workflow runs in response to repository lifecycle events. +Shell helpers that cancel GitHub Actions workflow runs whose original trigger is no longer relevant. ## Scope 📋 -This directory contains helpers invoked from workflows that listen on `pull_request_target: closed`, `pull_request_target: converted_to_draft`, and `delete`. The helpers call the GitHub Actions API to cancel runs whose original trigger is no longer relevant (a closed PR, a deleted branch). Helpers MUST NOT be called from workflows that have not received the corresponding lifecycle event, because cancelling unrelated runs would mask real failures and waste accumulated work. +This directory contains helpers invoked from workflows that listen on `pull_request_target`, `push`, and `delete`. The helpers call the GitHub Actions API to cancel runs made obsolete by a closed PR, a deleted branch, or a newer commit on the same PR or branch. Helpers MUST NOT be called from workflows that have not received one of those events, because cancelling unrelated runs would mask real failures and waste accumulated work. + +## Superseding runs 🔁 + +On a long orchestrator sweep the concurrency group sometimes fails to reap its occupant: the occupant's run-level status sits on `queued` — the normal state of a run whose matrix jobs are still waiting for runners — and the newer run parks on `pending` behind it until someone cancels the sweep by hand. Whether `queued` is the cause or another symptom cannot be decided from outside GitHub; what is observable is that the group does not always reap. The API cancel does not depend on any of that, so [entry-cancel-superseded.yml](../../../.github/workflows/entry-cancel-superseded.yml) calls these helpers on every new commit as well. That workflow deliberately declares no concurrency group: the run holding a group cannot be the run that frees it. + +The group stays primary. It reaps within seconds of a run being created and needs no runner, while the fallback first has to win a runner slot — from the very pool the superseded sweep is saturating, so it is slowest exactly when it is needed. Measured waits in this repository range from under a minute to about twenty. That is the cost of the fallback, and the reason it is a fallback rather than the mechanism. + +Three variables keep the fallback inside the group's bounds: + +- `KEEP_NEWEST_PER` — keeps the newest run of each group instance alive and cancels only the older ones, which is the rule the concurrency group itself applies. It is what makes the fallback safe against races: the head SHA from the event payload goes stale while the job waits for a runner, and a `[skip ci]` or `GITHUB_TOKEN` push moves the branch tip without creating a run at all. Selecting by "not the current head" would then cancel the newest commit's CI and spare the stale one, or leave the branch with no CI at all. Selecting by "not the newest run" cannot, in any ordering. + + The value must mirror how the backed-up group is keyed. `all` treats every matched run as one instance — correct for `global-ci--`, which spans both `entry-push-latest.yml` and `entry-manual-steer.yml`, so a push must be able to reap a manual sweep. `event` keeps one instance per event name — correct for `pr-ci---pr-`. Grouping any finer than the real key spares a run the group would have reaped: per-workflow grouping leaves the manual sweep alive, because it is the newest run of its own workflow. +- `CURRENT_RUN_ID` — keeps the calling run itself alive. +- `INCLUDE_PATHS` — newline-separated workflow paths, the members of the concurrency group being backed up. Without it the helpers would also cancel runs the group never governed, such as the `cron-*` workflows that listen on `push`. Derive the list from the `concurrency:` blocks, not from what looks related: `entry-delete-branch.yml` keys its group on the *deleted* ref, so it is not a member of a push's group, and listing it would make a push to `main` cancel an unrelated branch cleanup. A value that lists no path at all aborts rather than silently matching nothing. + +Leave `KEEP_NEWEST_PER` and `INCLUDE_PATHS` unset to cancel every matching run, as the lifecycle events do — a closed PR and a deleted branch have no run left that deserves to survive. A fallback also mirrors whatever gates its group: `global-ci-*` cancels on every ref except `main`, so the push job carries the same `github.ref_name != 'main'` guard, while `pr-ci-*` is declared `cancel-in-progress: true` unconditionally, so the pull request job is ungated too. + +Every selected run is attempted before anything is reported; abandoning the rest on the first rejection would leave them burning runners. `HTTP 409` means the run already finished and is not a rejection at all. Any other rejection is logged as an `ERROR` line, but only a denied token fails the job — a rate limit or a run that vanished between listing and cancel is transient, and a red check for those is a red check nobody reads. + +"Newest" means the run that entered the group last, so the helpers sort on `run_started_at` and fall back to `created_at`. A re-run keeps the `created_at` of its first attempt while re-entering the group as the newest member, and sorting on `created_at` would cancel it as the oldest. + +Two open pull requests that share a head repository *and* a head branch, to different bases, cannot be told apart from the run payload: the runs of a fork PR carry an empty `pull_requests` array, the same `head_branch` and the same `head_sha`. The helper therefore asks the API whether any other open pull request shares that head, and drops the branch matcher while one does — cancelling nothing beats cancelling the wrong pull request. If that lookup itself fails the matcher stays on, because a failed lookup is far more likely than a shared head. + +The branch-name matcher normally also requires the run's head repository to be the pull request's own. Two cases relax that: a run whose `head_repository` the API omits, and a pull request whose head fork has been deleted, where the event payload carries no head repository either. Both are matched on the branch name alone, because otherwise their runs would keep going forever — and neither field is something a contributor can influence. The ambiguity check above is what makes that safe: it runs whenever a head branch is known, with or without a repository, so a shared branch name still disables the matcher rather than cancelling the wrong pull request. For the workflow catalog that drives these calls see [workflows.md](../../../docs/contributing/tools/github/actions/workflows.md). diff --git a/scripts/github/cancel/branch_runs.sh b/scripts/github/cancel/branch_runs.sh new file mode 100755 index 0000000000..6e0dbabede --- /dev/null +++ b/scripts/github/cancel/branch_runs.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${BRANCH:?Missing BRANCH}" +: "${GH_TOKEN:?Missing GH_TOKEN}" +: "${REPOSITORY:?Missing REPOSITORY}" +CURRENT_RUN_ID="${CURRENT_RUN_ID:-}" +INCLUDE_PATHS="${INCLUDE_PATHS:-}" +KEEP_NEWEST_PER="${KEEP_NEWEST_PER:-}" + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/github/cancel/lib.sh +source "${SCRIPT_DIR}/lib.sh" + +require_tools +validate_include_paths +validate_keep_newest_per + +echo "Searching active workflow runs for branch: ${BRANCH}" + +runs="$(collect_runs)" + +run_ids="$( + printf '%s\n' "${runs}" | jq -s -r \ + --arg branch "${BRANCH}" \ + --arg current_run_id "${CURRENT_RUN_ID}" \ + --arg include_paths "${INCLUDE_PATHS}" \ + --arg keep_newest "${KEEP_NEWEST_PER}" ' + def drop_newest: sort_by([(.run_started_at // .created_at), .id]) | .[:-1]; + def allowlist: + $include_paths | split("\n") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0)); + + allowlist as $paths + | unique_by(.id) + | map( + select(.path != ".github/workflows/entry-pr-closed-cancel-workflows.yml") + | select($current_run_id == "" or (.id | tostring) != $current_run_id) + | select($include_paths == "" or ((.path // "") | IN($paths[]))) + | select( + .head_branch == $branch + or any(.pull_requests[]?; (.head.ref // "") == $branch) + ) + ) + | (if $keep_newest == "all" then drop_newest + elif $keep_newest == "event" then + group_by(.event // "") | map(drop_newest) | flatten + else . end) + | .[].id + ' +)" + +cancel_all "${run_ids}" diff --git a/scripts/github/cancel/deleted_branch_runs.sh b/scripts/github/cancel/deleted_branch_runs.sh deleted file mode 100755 index 24abde67bc..0000000000 --- a/scripts/github/cancel/deleted_branch_runs.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -: "${BRANCH:?Missing BRANCH}" -: "${GH_TOKEN:?Missing GH_TOKEN}" -: "${REPOSITORY:?Missing REPOSITORY}" - -if ! command -v gh >/dev/null 2>&1; then - echo "ERROR: gh CLI not found." >&2 - exit 1 -fi - -if ! command -v jq >/dev/null 2>&1; then - echo "ERROR: jq not found." >&2 - exit 1 -fi - -echo "Searching active workflow runs for deleted branch: ${BRANCH}" - -cancel_runs_by_status() { - local status="$1" - local run_ids - - run_ids="$( - gh api --paginate \ - -H "Accept: application/vnd.github+json" \ - "/repos/${REPOSITORY}/actions/runs?status=${status}&per_page=100" | - jq -r --arg branch "${BRANCH}" ' - .workflow_runs[] - | select(.path != ".github/workflows/entry-pr-closed-cancel-workflows.yml") - | select( - .head_branch == $branch - or any(.pull_requests[]?; (.head.ref // "") == $branch) - ) - | .id - ' | - sort -u - )" - - if [[ -z "${run_ids}" ]]; then - echo "No ${status} runs found for ${BRANCH}" - return 0 - fi - - while read -r run_id; do - [[ -n "${run_id}" ]] || continue - echo "Cancelling ${status} run ${run_id}" - if ! gh api \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - "/repos/${REPOSITORY}/actions/runs/${run_id}/cancel" \ - >/dev/null; then - echo "Run ${run_id} could not be cancelled, likely because it already completed" - fi - done <<<"${run_ids}" -} - -cancel_runs_by_status requested -cancel_runs_by_status pending -cancel_runs_by_status waiting -cancel_runs_by_status queued -cancel_runs_by_status in_progress diff --git a/scripts/github/cancel/lib.sh b/scripts/github/cancel/lib.sh new file mode 100755 index 0000000000..dcc30846aa --- /dev/null +++ b/scripts/github/cancel/lib.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Shared by branch_runs.sh and pull_request_runs.sh. Expects REPOSITORY, +# GH_TOKEN and INCLUDE_PATHS to be set by the caller. + +RUN_STATUSES=(requested pending waiting queued in_progress) + +require_tools() { + local tool + + for tool in gh jq; do + if ! command -v "${tool}" >/dev/null 2>&1; then + echo "ERROR: ${tool} not found." >&2 + exit 1 + fi + done +} + +# A non-empty INCLUDE_PATHS that trims to nothing would silently select no run +# at all, turning the caller into a no-op that still reports success. +validate_include_paths() { + if [[ -z "${INCLUDE_PATHS}" ]]; then + return 0 + fi + + if [[ -z "$(printf '%s' "${INCLUDE_PATHS}" | tr -d '[:space:]')" ]]; then + echo "ERROR: INCLUDE_PATHS is set but lists no workflow path." >&2 + exit 1 + fi + + echo "Restricted to workflows:" + printf '%s\n' "${INCLUDE_PATHS}" | sed '/^[[:space:]]*$/d;s/^/ /' +} + +# Mirrors the key of the concurrency group being backed up: `all` for a group +# that spans every event on a ref, `event` for one whose key contains the event +# name. Grouping any finer keeps a run alive that the group would have reaped. +validate_keep_newest_per() { + case "${KEEP_NEWEST_PER}" in + "" | all | event) ;; + *) + echo "ERROR: KEEP_NEWEST_PER must be empty, 'all' or 'event'." >&2 + exit 1 + ;; + esac +} + +# Assign the result with `runs="$(collect_runs)"`, never pipe this function +# directly: on the left of a pipeline bash drops its exit status, and a failed +# page would silently narrow the run set instead of aborting. +collect_runs() { + local status + local payload + + for status in "${RUN_STATUSES[@]}"; do + payload="$( + gh api --paginate \ + -H "Accept: application/vnd.github+json" \ + "/repos/${REPOSITORY}/actions/runs?status=${status}&per_page=100" + )" || return 1 + printf '%s' "${payload}" | jq -c '.workflow_runs[]' || return 1 + done +} + +cancel_run() { + local run_id="$1" + local response + + if response="$( + gh api \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + "/repos/${REPOSITORY}/actions/runs/${run_id}/cancel" 2>&1 + )"; then + return 0 + fi + + if [[ "${response}" == *"HTTP 409"* ]]; then + echo "Run ${run_id} was already completed or already cancelling" + return 0 + fi + + if [[ "${response}" == *"Resource not accessible"* ]]; then + CANCEL_PERMISSION_DENIED=1 + fi + + echo "ERROR: cancelling run ${run_id} failed: ${response}" >&2 + return 1 +} + +# One transient failure must not abandon the remaining runs: they would keep +# burning runners. Cancel everything, then report. Only a denied token is +# fatal — a rate limit or a run that vanished between listing and cancel is +# transient, and painting the cleanup red for those trains everyone to ignore +# the red. Counting rejections instead would fail a one-run sweep on a single +# transient, which is the common shape for a branch delete. +cancel_all() { + local run_ids="$1" + local run_id + local failures=0 + local successes=0 + + CANCEL_PERMISSION_DENIED=0 + + if [[ -z "${run_ids}" ]]; then + echo "No superseded runs found" + return 0 + fi + + while read -r run_id; do + [[ -n "${run_id}" ]] || continue + echo "Cancelling run ${run_id}" + if cancel_run "${run_id}"; then + successes=$((successes + 1)) + else + failures=$((failures + 1)) + fi + done <<<"${run_ids}" + + if ((failures == 0)); then + return 0 + fi + + if ((CANCEL_PERMISSION_DENIED)); then + echo "ERROR: the token may not cancel runs; ${failures} of $((failures + successes)) rejected." >&2 + return 1 + fi + + echo "WARNING: ${failures} of $((failures + successes)) run(s) could not be cancelled." >&2 +} diff --git a/scripts/github/cancel/pull_request_runs.sh b/scripts/github/cancel/pull_request_runs.sh index 44eca4fafa..fd47d5c4e7 100755 --- a/scripts/github/cancel/pull_request_runs.sh +++ b/scripts/github/cancel/pull_request_runs.sh @@ -4,81 +4,107 @@ set -euo pipefail : "${PR_NUMBER:?Missing PR_NUMBER}" : "${GH_TOKEN:?Missing GH_TOKEN}" : "${REPOSITORY:?Missing REPOSITORY}" -: "${CURRENT_RUN_ID:?Missing CURRENT_RUN_ID}" PR_HEAD_REF="${PR_HEAD_REF:-}" PR_HEAD_SHA="${PR_HEAD_SHA:-}" PR_HEAD_REPOSITORY="${PR_HEAD_REPOSITORY:-}" +: "${CURRENT_RUN_ID:?Missing CURRENT_RUN_ID}" +INCLUDE_PATHS="${INCLUDE_PATHS:-}" +KEEP_NEWEST_PER="${KEEP_NEWEST_PER:-}" -if ! command -v gh >/dev/null 2>&1; then - echo "ERROR: gh CLI not found." >&2 - exit 1 -fi +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/github/cancel/lib.sh +source "${SCRIPT_DIR}/lib.sh" -if ! command -v jq >/dev/null 2>&1; then - echo "ERROR: jq not found." >&2 - exit 1 -fi +require_tools +validate_include_paths +validate_keep_newest_per + +# A fork PR's runs carry an empty `pull_requests` array, so the branch matcher +# is all that is left — and it cannot tell two pull requests apart that share a +# head repository and head branch. Rather than cancel the wrong one, drop the +# matcher for as long as the head is ambiguous. +disable_ambiguous_branch_fallback() { + local rivals + local -a query + + if [[ -z "${PR_HEAD_REF}" ]]; then + return 0 + fi + + query=(-f state=open -f per_page=100) + if [[ -n "${PR_HEAD_REPOSITORY}" ]]; then + query+=(-f "head=${PR_HEAD_REPOSITORY%%/*}:${PR_HEAD_REF}") + fi + + if ! rivals="$( + gh api --paginate \ + -H "Accept: application/vnd.github+json" \ + -X GET "/repos/${REPOSITORY}/pulls" \ + "${query[@]}" | + jq -s \ + --argjson pr_number "${PR_NUMBER}" \ + --arg head_ref "${PR_HEAD_REF}" \ + '[.[][] | select(.number != $pr_number and (.head.ref // "") == $head_ref)] | length' + )"; then + echo "WARNING: could not list open pull requests for the head; keeping the branch fallback" >&2 + return 0 + fi + + if [[ "${rivals}" != "0" ]]; then + echo "Branch fallback disabled: ${rivals} other open pull request(s) share ${PR_HEAD_REPOSITORY%%/*}:${PR_HEAD_REF}" + PR_HEAD_REF="" + fi +} echo "Searching active workflow runs for PR #${PR_NUMBER}" if [[ -n "${PR_HEAD_SHA}${PR_HEAD_REF}${PR_HEAD_REPOSITORY}" ]]; then echo "Fallback matching enabled for head.sha=${PR_HEAD_SHA:-} head.ref=${PR_HEAD_REF:-} head.repo=${PR_HEAD_REPOSITORY:-}" fi -cancel_runs_by_status() { - local status="$1" - local run_ids +disable_ambiguous_branch_fallback - run_ids="$( - gh api --paginate \ - -H "Accept: application/vnd.github+json" \ - "/repos/${REPOSITORY}/actions/runs?status=${status}&per_page=100" | - jq -r \ - --argjson pr_number "${PR_NUMBER}" \ - --argjson current_run_id "${CURRENT_RUN_ID}" \ - --arg pr_head_ref "${PR_HEAD_REF}" \ - --arg pr_head_sha "${PR_HEAD_SHA}" \ - --arg pr_head_repository "${PR_HEAD_REPOSITORY}" ' - .workflow_runs[] - | select(.id != $current_run_id) - | select(.event == "pull_request" or .event == "pull_request_target") - | select( - any(.pull_requests[]?; (.number // -1) == $pr_number) - or ($pr_head_sha != "" and (.head_sha // "") == $pr_head_sha) - or ( - $pr_head_ref != "" - and (.head_branch // "") == $pr_head_ref - and ( - $pr_head_repository == "" - or (.head_repository.full_name // "") == $pr_head_repository - or (.head_repository.full_name // "") == "" - ) - ) - ) - | .id - ' | - sort -u - )" +runs="$(collect_runs)" - if [[ -z "${run_ids}" ]]; then - echo "No ${status} runs found for PR #${PR_NUMBER}" - return 0 - fi +run_ids="$( + printf '%s\n' "${runs}" | jq -s -r \ + --argjson pr_number "${PR_NUMBER}" \ + --arg current_run_id "${CURRENT_RUN_ID}" \ + --arg pr_head_ref "${PR_HEAD_REF}" \ + --arg pr_head_sha "${PR_HEAD_SHA}" \ + --arg pr_head_repository "${PR_HEAD_REPOSITORY}" \ + --arg include_paths "${INCLUDE_PATHS}" \ + --arg keep_newest "${KEEP_NEWEST_PER}" ' + def drop_newest: sort_by([(.run_started_at // .created_at), .id]) | .[:-1]; + def allowlist: + $include_paths | split("\n") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0)); - while read -r run_id; do - [[ -n "${run_id}" ]] || continue - echo "Cancelling ${status} run ${run_id}" - if ! gh api \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - "/repos/${REPOSITORY}/actions/runs/${run_id}/cancel" \ - >/dev/null; then - echo "Run ${run_id} could not be cancelled, likely because it already completed" - fi - done <<<"${run_ids}" -} + allowlist as $paths + | unique_by(.id) + | map( + select($current_run_id == "" or (.id | tostring) != $current_run_id) + | select($include_paths == "" or ((.path // "") | IN($paths[]))) + | select(.event == "pull_request" or .event == "pull_request_target") + | select( + any(.pull_requests[]?; (.number // -1) == $pr_number) + or ($pr_head_sha != "" and (.head_sha // "") == $pr_head_sha) + or ( + $pr_head_ref != "" + and all(.pull_requests[]?; (.number // $pr_number) == $pr_number) + and (.head_branch // "") == $pr_head_ref + and ( + $pr_head_repository == "" + or (.head_repository.full_name // "") == $pr_head_repository + or (.head_repository.full_name // "") == "" + ) + ) + ) + ) + | (if $keep_newest == "all" then drop_newest + elif $keep_newest == "event" then + group_by(.event // "") | map(drop_newest) | flatten + else . end) + | .[].id + ' +)" -cancel_runs_by_status requested -cancel_runs_by_status pending -cancel_runs_by_status waiting -cancel_runs_by_status queued -cancel_runs_by_status in_progress +cancel_all "${run_ids}" diff --git a/tests/integration/meta/workflows/test_cancel_pr_runs.py b/tests/integration/meta/workflows/test_cancel_pr_runs.py deleted file mode 100644 index 5ab6ad9f9f..0000000000 --- a/tests/integration/meta/workflows/test_cancel_pr_runs.py +++ /dev/null @@ -1,193 +0,0 @@ -import os -import shutil -import subprocess -import tempfile -import textwrap -import unittest -from pathlib import Path - -from utils.cache.files import read_text - -from . import PROJECT_ROOT - -SCRIPT_PATH = PROJECT_ROOT / "scripts" / "github" / "cancel" / "pull_request_runs.sh" -EMPTY_RUNS_JSON = '{"workflow_runs":[]}\n' - - -@unittest.skipUnless( - shutil.which("jq"), "jq is required for the shell script under test" -) -class TestCancelPullRequestRuns(unittest.TestCase): - def _write_fake_gh(self, temp_dir: Path) -> None: - fake_gh = temp_dir / "gh" - fake_gh.write_text( - textwrap.dedent( - """\ - #!/usr/bin/env bash - set -euo pipefail - - if [[ "${1:-}" != "api" ]]; then - echo "Unsupported gh invocation: $*" >&2 - exit 1 - fi - shift - - method="GET" - url="" - while [[ $# -gt 0 ]]; do - case "$1" in - --paginate) - shift - ;; - -H) - shift 2 - ;; - -X) - method="$2" - shift 2 - ;; - /repos/*) - url="$1" - shift - ;; - *) - shift - ;; - esac - done - - if [[ "${method}" == "POST" ]]; then - run_id="$(printf '%s\\n' "${url}" | sed -E 's#.*/actions/runs/([0-9]+)/cancel#\\1#')" - printf '%s\\n' "${run_id}" >> "${GH_FAKE_CANCEL_LOG}" - exit 0 - fi - - status="$(printf '%s\\n' "${url}" | sed -nE 's#.*[?&]status=([^&]+).*#\\1#p')" - cat "${GH_FAKE_RUNS_DIR}/${status}.json" - """ - ), - encoding="utf-8", - ) - fake_gh.chmod(0o755) - - def _run_script( - self, - *, - runs_by_status, - pr_head_ref="feature/makefile-wsl2", - pr_head_sha="deadbeef", - pr_head_repository="AlejandroRomanIbanez/core", - ): - with tempfile.TemporaryDirectory() as temp_dir_str: - temp_dir = Path(temp_dir_str) - runs_dir = temp_dir / "runs" - runs_dir.mkdir() - cancel_log = temp_dir / "cancel.log" - - for status in ["requested", "pending", "waiting", "queued", "in_progress"]: - (runs_dir / f"{status}.json").write_text( - runs_by_status.get(status, EMPTY_RUNS_JSON), - encoding="utf-8", - ) - - self._write_fake_gh(temp_dir) - - env = os.environ.copy() - env.update( - { - "PR_NUMBER": "106", - "PR_HEAD_REF": pr_head_ref, - "PR_HEAD_SHA": pr_head_sha, - "PR_HEAD_REPOSITORY": pr_head_repository, - "GH_TOKEN": "test-token", - "REPOSITORY": "kevinveenbirkenbach/infinito-nexus", - "CURRENT_RUN_ID": "6", - "GH_FAKE_RUNS_DIR": str(runs_dir), - "GH_FAKE_CANCEL_LOG": str(cancel_log), - "PATH": f"{temp_dir}:{env['PATH']}", - } - ) - - result = subprocess.run( - ["bash", str(SCRIPT_PATH)], - cwd=PROJECT_ROOT, - env=env, - capture_output=True, - text=True, - check=True, - ) - - cancelled = [] - if cancel_log.exists(): - cancelled = [ - line.strip() - for line in read_text(str(cancel_log)).splitlines() - if line.strip() - ] - return result, cancelled - - def test_cancels_matching_fork_run_when_pull_request_association_is_missing(self): - in_progress_runs = """\ -{"workflow_runs":[ - { - "id": 297, - "event": "pull_request", - "head_sha": "merge-sha-not-pr-head", - "head_branch": "feature/makefile-wsl2", - "head_repository": {"full_name": "AlejandroRomanIbanez/core"}, - "pull_requests": [] - } -]} -""" - _, cancelled = self._run_script( - runs_by_status={"in_progress": in_progress_runs} - ) - self.assertEqual(cancelled, ["297"]) - - def test_does_not_cancel_run_from_other_fork_with_same_branch_name(self): - in_progress_runs = """\ -{"workflow_runs":[ - { - "id": 297, - "event": "pull_request", - "head_sha": "merge-sha-not-pr-head", - "head_branch": "feature/makefile-wsl2", - "head_repository": {"full_name": "AlejandroRomanIbanez/core"}, - "pull_requests": [] - }, - { - "id": 298, - "event": "pull_request", - "head_sha": "another-merge-sha", - "head_branch": "feature/makefile-wsl2", - "head_repository": {"full_name": "someone-else/core"}, - "pull_requests": [] - } -]} -""" - _, cancelled = self._run_script( - runs_by_status={"in_progress": in_progress_runs} - ) - self.assertEqual(cancelled, ["297"]) - - def test_cancels_run_via_head_sha_when_branch_metadata_differs(self): - in_progress_runs = """\ -{"workflow_runs":[ - { - "id": 299, - "event": "pull_request_target", - "head_sha": "deadbeef", - "head_branch": "unexpected-branch-name", - "head_repository": {"full_name": "base-repo/core"}, - "pull_requests": [] - } -]} -""" - _, cancelled = self._run_script( - runs_by_status={"in_progress": in_progress_runs} - ) - self.assertEqual(cancelled, ["299"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/integration/meta/workflows/test_cancel_runs.py b/tests/integration/meta/workflows/test_cancel_runs.py new file mode 100644 index 0000000000..916289666f --- /dev/null +++ b/tests/integration/meta/workflows/test_cancel_runs.py @@ -0,0 +1,561 @@ +import os +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + +from utils.cache.files import read_text + +from . import PROJECT_ROOT + +SCRIPT_PATH = PROJECT_ROOT / "scripts" / "github" / "cancel" / "pull_request_runs.sh" +BRANCH_SCRIPT_PATH = PROJECT_ROOT / "scripts" / "github" / "cancel" / "branch_runs.sh" +EMPTY_RUNS_JSON = '{"workflow_runs":[]}\n' + + +class CancelScriptMixin: + def _write_fake_gh(self, temp_dir: Path) -> None: + fake_gh = temp_dir / "gh" + fake_gh.write_text( + textwrap.dedent( + """\ + #!/usr/bin/env bash + set -euo pipefail + + if [[ "${1:-}" != "api" ]]; then + echo "Unsupported gh invocation: $*" >&2 + exit 1 + fi + shift + + method="GET" + url="" + while [[ $# -gt 0 ]]; do + case "$1" in + --paginate) + shift + ;; + -H) + shift 2 + ;; + -X) + method="$2" + shift 2 + ;; + /repos/*) + url="$1" + shift + ;; + *) + shift + ;; + esac + done + + if [[ "${method}" == "POST" ]]; then + run_id="$(printf '%s\\n' "${url}" | sed -E 's#.*/actions/runs/([0-9]+)/cancel#\\1#')" + if [[ -n "${GH_FAKE_POST_ERROR:-}" ]]; then + echo "${GH_FAKE_POST_ERROR}" >&2 + exit 1 + fi + if [[ -n "${GH_FAKE_POST_ERROR_ONCE:-}" \\ + && ! -f "${GH_FAKE_CANCEL_LOG}.once" ]]; then + touch "${GH_FAKE_CANCEL_LOG}.once" + echo "${GH_FAKE_POST_ERROR_ONCE}" >&2 + exit 1 + fi + printf '%s\\n' "${run_id}" >> "${GH_FAKE_CANCEL_LOG}" + exit 0 + fi + + if [[ "${url}" == */pulls* ]]; then + cat "${GH_FAKE_RUNS_DIR}/pulls.json" + exit 0 + fi + + status="$(printf '%s\\n' "${url}" | sed -nE 's#.*[?&]status=([^&]+).*#\\1#p')" + if [[ "${status}" == "${GH_FAKE_GET_ERROR_STATUS:-}" ]]; then + echo "gh: simulated API failure for ${status}" >&2 + exit 1 + fi + cat "${GH_FAKE_RUNS_DIR}/${status}.json" + """ + ), + encoding="utf-8", + ) + fake_gh.chmod(0o755) + + def _run_cancel_script( + self, script_path, env_overrides, runs_by_status, check=True + ): + with tempfile.TemporaryDirectory() as temp_dir_str: + temp_dir = Path(temp_dir_str) + runs_dir = temp_dir / "runs" + runs_dir.mkdir() + cancel_log = temp_dir / "cancel.log" + + for status in ["requested", "pending", "waiting", "queued", "in_progress"]: + (runs_dir / f"{status}.json").write_text( + runs_by_status.get(status, EMPTY_RUNS_JSON), + encoding="utf-8", + ) + (runs_dir / "pulls.json").write_text( + runs_by_status.get("pulls", "[]\n"), encoding="utf-8" + ) + + self._write_fake_gh(temp_dir) + + env = os.environ.copy() + env.update( + { + "GH_TOKEN": "test-token", + "REPOSITORY": "kevinveenbirkenbach/infinito-nexus", + "GH_FAKE_RUNS_DIR": str(runs_dir), + "GH_FAKE_CANCEL_LOG": str(cancel_log), + "PATH": f"{temp_dir}:{env['PATH']}", + **env_overrides, + } + ) + + result = subprocess.run( + ["bash", str(script_path)], + cwd=PROJECT_ROOT, + env=env, + capture_output=True, + text=True, + check=check, + ) + + cancelled = [] + if cancel_log.exists(): + cancelled = [ + line.strip() + for line in read_text(str(cancel_log)).splitlines() + if line.strip() + ] + return result, cancelled + + +@unittest.skipUnless( + shutil.which("jq"), "jq is required for the shell script under test" +) +class TestCancelPullRequestRuns(CancelScriptMixin, unittest.TestCase): + def _run_script( + self, + *, + runs_by_status, + pr_head_ref="feature/makefile-wsl2", + pr_head_sha="deadbeef", + pr_head_repository="AlejandroRomanIbanez/core", + include_paths="", + keep_newest_per="", + ): + return self._run_cancel_script( + SCRIPT_PATH, + { + "PR_NUMBER": "106", + "PR_HEAD_REF": pr_head_ref, + "PR_HEAD_SHA": pr_head_sha, + "PR_HEAD_REPOSITORY": pr_head_repository, + "INCLUDE_PATHS": include_paths, + "KEEP_NEWEST_PER": keep_newest_per, + "CURRENT_RUN_ID": "6", + }, + runs_by_status, + ) + + def test_cancels_matching_fork_run_when_pull_request_association_is_missing(self): + in_progress_runs = """\ +{"workflow_runs":[ + { + "id": 297, + "event": "pull_request", + "head_sha": "merge-sha-not-pr-head", + "head_branch": "feature/makefile-wsl2", + "head_repository": {"full_name": "AlejandroRomanIbanez/core"}, + "pull_requests": [] + } +]} +""" + _, cancelled = self._run_script( + runs_by_status={"in_progress": in_progress_runs} + ) + self.assertEqual(cancelled, ["297"]) + + def test_does_not_cancel_run_from_other_fork_with_same_branch_name(self): + in_progress_runs = """\ +{"workflow_runs":[ + { + "id": 297, + "event": "pull_request", + "head_sha": "merge-sha-not-pr-head", + "head_branch": "feature/makefile-wsl2", + "head_repository": {"full_name": "AlejandroRomanIbanez/core"}, + "pull_requests": [] + }, + { + "id": 298, + "event": "pull_request", + "head_sha": "another-merge-sha", + "head_branch": "feature/makefile-wsl2", + "head_repository": {"full_name": "someone-else/core"}, + "pull_requests": [] + } +]} +""" + _, cancelled = self._run_script( + runs_by_status={"in_progress": in_progress_runs} + ) + self.assertEqual(cancelled, ["297"]) + + def test_cancels_run_via_head_sha_when_branch_metadata_differs(self): + in_progress_runs = """\ +{"workflow_runs":[ + { + "id": 299, + "event": "pull_request_target", + "head_sha": "deadbeef", + "head_branch": "unexpected-branch-name", + "head_repository": {"full_name": "base-repo/core"}, + "pull_requests": [] + } +]} +""" + _, cancelled = self._run_script( + runs_by_status={"in_progress": in_progress_runs} + ) + self.assertEqual(cancelled, ["299"]) + + NEWEST_PER_GROUP_RUNS = """\ +{"workflow_runs":[ + { + "id": 300, + "event": "pull_request", + "path": ".github/workflows/entry-pr-change-orchestrate.yml", + "created_at": "2026-08-24T22:25:11Z", + "head_sha": "oldsha", + "head_branch": "feature/makefile-wsl2", + "head_repository": {"full_name": "AlejandroRomanIbanez/core"}, + "pull_requests": [{"number": 106}] + }, + { + "id": 301, + "event": "pull_request", + "path": ".github/workflows/entry-pr-change-orchestrate.yml", + "created_at": "2026-08-25T07:54:16Z", + "head_sha": "newsha", + "head_branch": "feature/makefile-wsl2", + "head_repository": {"full_name": "AlejandroRomanIbanez/core"}, + "pull_requests": [{"number": 106}] + }, + { + "id": 302, + "event": "pull_request_target", + "path": ".github/workflows/entry-pr-change-orchestrate.yml", + "created_at": "2026-08-25T07:54:15Z", + "head_sha": "newsha", + "head_branch": "feature/makefile-wsl2", + "head_repository": {"full_name": "AlejandroRomanIbanez/core"}, + "pull_requests": [{"number": 106}] + } +]} +""" + + FORK_SHAPED_RUNS = """\ +{"workflow_runs":[ + { + "id": 320, + "event": "pull_request", + "created_at": "2026-08-24T22:25:11Z", + "head_sha": "oldsha", + "head_branch": "feature/makefile-wsl2", + "head_repository": {"full_name": "AlejandroRomanIbanez/core"}, + "pull_requests": [] + } +]} +""" + + def test_cancels_a_fork_run_when_the_head_is_unambiguous(self): + _, cancelled = self._run_script( + runs_by_status={"queued": self.FORK_SHAPED_RUNS} + ) + self.assertEqual(cancelled, ["320"]) + + def test_keeps_fork_runs_when_another_pull_request_shares_the_head(self): + result, cancelled = self._run_script( + runs_by_status={ + "queued": self.FORK_SHAPED_RUNS, + "pulls": ( + '[{"number": 106, "head": {"ref": "feature/makefile-wsl2"}},' + ' {"number": 107, "head": {"ref": "feature/makefile-wsl2"}}]\n' + ), + }, + ) + self.assertEqual(cancelled, []) + self.assertIn("Branch fallback disabled", result.stdout) + + def test_cancels_a_run_whose_association_names_this_pull_request(self): + queued_runs = """\ +{"workflow_runs":[ + { + "id": 310, + "event": "pull_request", + "created_at": "2026-08-24T22:25:11Z", + "head_sha": "oldsha", + "head_branch": "feature/makefile-wsl2", + "head_repository": {"full_name": "AlejandroRomanIbanez/core"}, + "pull_requests": [{"no_number_field": true}] + } +]} +""" + _, cancelled = self._run_script(runs_by_status={"queued": queued_runs}) + self.assertEqual(cancelled, ["310"]) + + def test_keeps_a_run_associated_with_a_different_pull_request(self): + queued_runs = """\ +{"workflow_runs":[ + { + "id": 311, + "event": "pull_request", + "created_at": "2026-08-24T22:25:11Z", + "head_sha": "oldsha", + "head_branch": "feature/makefile-wsl2", + "head_repository": {"full_name": "AlejandroRomanIbanez/core"}, + "pull_requests": [{"number": 999}] + } +]} +""" + _, cancelled = self._run_script(runs_by_status={"queued": queued_runs}) + self.assertEqual(cancelled, []) + + DELETED_FORK_RUNS = """\ +{"workflow_runs":[ + { + "id": 330, + "event": "pull_request", + "created_at": "2026-08-24T22:25:11Z", + "head_sha": "oldsha", + "head_branch": "feature/makefile-wsl2", + "head_repository": {"full_name": "AlejandroRomanIbanez/core"}, + "pull_requests": [] + } +]} +""" + + def test_cancels_runs_of_a_pull_request_whose_fork_was_deleted(self): + _, cancelled = self._run_script( + runs_by_status={"queued": self.DELETED_FORK_RUNS}, + pr_head_repository="", + ) + self.assertEqual(cancelled, ["330"]) + + def test_keeps_them_when_another_pull_request_shares_the_branch(self): + result, cancelled = self._run_script( + runs_by_status={ + "queued": self.DELETED_FORK_RUNS, + "pulls": '[{"number": 107, "head": {"ref": "feature/makefile-wsl2"}}]\n', + }, + pr_head_repository="", + ) + self.assertEqual(cancelled, []) + self.assertIn("Branch fallback disabled", result.stdout) + + def test_keeps_the_newest_run_of_every_event(self): + _, cancelled = self._run_script( + runs_by_status={"queued": self.NEWEST_PER_GROUP_RUNS}, + keep_newest_per="event", + ) + self.assertEqual(cancelled, ["300"]) + + def test_keeps_the_newest_run_even_when_the_head_sha_moved_on(self): + _, cancelled = self._run_script( + runs_by_status={"queued": self.NEWEST_PER_GROUP_RUNS}, + keep_newest_per="event", + pr_head_sha="a-third-sha-nobody-has-a-run-for", + ) + self.assertEqual(cancelled, ["300"]) + + def test_keeps_pull_request_runs_outside_the_concurrency_group(self): + queued_runs = """\ +{"workflow_runs":[ + { + "id": 302, + "event": "pull_request", + "path": ".github/workflows/entry-pr-change-orchestrate.yml", + "head_sha": "cafebabe", + "head_branch": "feature/makefile-wsl2", + "head_repository": {"full_name": "AlejandroRomanIbanez/core"}, + "pull_requests": [{"number": 106}] + }, + { + "id": 303, + "event": "pull_request_target", + "path": ".github/workflows/entry-pr-open-dependabot-close.yml", + "head_sha": "cafebabe", + "head_branch": "feature/makefile-wsl2", + "head_repository": {"full_name": "AlejandroRomanIbanez/core"}, + "pull_requests": [{"number": 106}] + } +]} +""" + _, cancelled = self._run_script( + runs_by_status={"queued": queued_runs}, + include_paths=".github/workflows/entry-pr-change-orchestrate.yml", + ) + self.assertEqual(cancelled, ["302"]) + + +@unittest.skipUnless( + shutil.which("jq"), "jq is required for the shell script under test" +) +class TestCancelBranchRuns(CancelScriptMixin, unittest.TestCase): + BRANCH_RUNS = """\ +{"workflow_runs":[ + { + "id": 400, + "path": ".github/workflows/entry-push-latest.yml", + "event": "push", + "created_at": "2026-08-25T07:54:12Z", + "head_sha": "newsha", + "head_branch": "feature/svc-net-tor", + "pull_requests": [] + }, + { + "id": 401, + "path": ".github/workflows/entry-manual-steer.yml", + "event": "workflow_dispatch", + "created_at": "2026-08-24T22:33:34Z", + "head_sha": "oldsha", + "head_branch": "feature/svc-net-tor", + "pull_requests": [] + }, + { + "id": 402, + "path": ".github/workflows/entry-push-latest.yml", + "event": "push", + "created_at": "2026-08-24T22:25:11Z", + "head_sha": "oldsha", + "head_branch": "other-branch", + "pull_requests": [] + }, + { + "id": 403, + "path": ".github/workflows/entry-push-latest.yml", + "event": "push", + "created_at": "2026-08-24T22:25:11Z", + "head_sha": "oldsha", + "head_branch": "feature/svc-net-tor", + "pull_requests": [] + } +]} +""" + + def _run_script(self, check=True, **env_overrides): + return self._run_cancel_script( + BRANCH_SCRIPT_PATH, + {"BRANCH": "feature/svc-net-tor", **env_overrides}, + {"queued": self.BRANCH_RUNS}, + check=check, + ) + + def test_tolerates_whitespace_around_allowlist_entries(self): + _, cancelled = self._run_script( + INCLUDE_PATHS=" .github/workflows/entry-manual-steer.yml \n\n", + ) + self.assertEqual(cancelled, ["401"]) + + def test_fails_loudly_when_a_cancel_is_rejected(self): + result, cancelled = self._run_script( + check=False, + GH_FAKE_POST_ERROR="gh: HTTP 403: Resource not accessible by integration", + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(cancelled, []) + self.assertIn("ERROR: cancelling run", result.stderr) + + def test_accepts_a_run_that_already_finished(self): + result, cancelled = self._run_script( + check=False, + GH_FAKE_POST_ERROR="gh: HTTP 409: Conflict", + ) + self.assertEqual(result.returncode, 0) + self.assertEqual(cancelled, []) + self.assertIn("already completed", result.stdout) + + def test_keeps_only_the_newest_run_of_the_whole_group(self): + _, cancelled = self._run_script(KEEP_NEWEST_PER="all") + self.assertEqual(sorted(cancelled), ["401", "403"]) + + def test_rejects_an_unknown_grouping_mode(self): + result, cancelled = self._run_script(check=False, KEEP_NEWEST_PER="true") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(cancelled, []) + self.assertIn("KEEP_NEWEST_PER must be", result.stderr) + + def test_cancels_every_branch_run_without_exclusions(self): + _, cancelled = self._run_script() + self.assertEqual(cancelled, ["400", "401", "403"]) + + def test_keeps_runs_outside_the_concurrency_group(self): + _, cancelled = self._run_script( + INCLUDE_PATHS=".github/workflows/entry-manual-steer.yml\n", + ) + self.assertEqual(cancelled, ["401"]) + + def test_aborts_on_an_allowlist_that_lists_nothing(self): + result, cancelled = self._run_script(check=False, INCLUDE_PATHS=" \n \n") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(cancelled, []) + self.assertIn("lists no workflow path", result.stderr) + + def test_stays_green_when_only_some_cancels_are_rejected(self): + result, cancelled = self._run_script( + check=False, + GH_FAKE_POST_ERROR_ONCE="gh: HTTP 403: rate limit", + ) + self.assertEqual(result.returncode, 0) + self.assertEqual(len(cancelled), 2) + self.assertIn("WARNING: 1 of 3 run(s) could not be cancelled", result.stderr) + + def test_a_single_transient_rejection_does_not_fail_the_job(self): + result, cancelled = self._run_script( + check=False, + INCLUDE_PATHS=".github/workflows/entry-manual-steer.yml\n", + GH_FAKE_POST_ERROR="gh: HTTP 404: Not Found", + ) + self.assertEqual(result.returncode, 0) + self.assertEqual(cancelled, []) + self.assertEqual(result.stdout.count("Cancelling run "), 1) + + def test_fails_loudly_when_a_run_listing_fails(self): + result, cancelled = self._run_script( + check=False, + GH_FAKE_GET_ERROR_STATUS="queued", + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(cancelled, []) + + def test_attempts_every_run_and_stays_green_on_transient_rejections(self): + result, cancelled = self._run_script( + check=False, + GH_FAKE_POST_ERROR="gh: HTTP 500: Internal Server Error", + ) + self.assertEqual(result.returncode, 0) + self.assertEqual(cancelled, []) + self.assertEqual(result.stdout.count("Cancelling run "), 3) + self.assertIn("3 of 3 run(s) could not be cancelled", result.stderr) + + def test_fails_when_the_token_may_not_cancel(self): + result, cancelled = self._run_script( + check=False, + GH_FAKE_POST_ERROR="gh: Resource not accessible by integration (HTTP 403)", + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(cancelled, []) + self.assertIn("token may not cancel runs", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/meta/workflows/test_cancel_superseded_workflow.py b/tests/integration/meta/workflows/test_cancel_superseded_workflow.py new file mode 100644 index 0000000000..d705619e87 --- /dev/null +++ b/tests/integration/meta/workflows/test_cancel_superseded_workflow.py @@ -0,0 +1,180 @@ +import re +import unittest + +from utils.cache.files import read_text +from utils.cache.yaml import load_yaml_any + +from . import PROJECT_ROOT + +WORKFLOWS = PROJECT_ROOT / ".github" / "workflows" +SUPERSEDED = WORKFLOWS / "entry-cancel-superseded.yml" +PUSH_ENTRY = WORKFLOWS / "entry-push-latest.yml" +PR_ENTRY = WORKFLOWS / "entry-pr-change-orchestrate.yml" +CANCEL_JOBS = { + "cancel-branch-runs": "scripts/github/cancel/branch_runs.sh", + "cancel-pull-request-runs": "scripts/github/cancel/pull_request_runs.sh", +} + + +def load(path): + return load_yaml_any(str(path), default_if_missing={}) + + +def workflow_files(): + yield from sorted(WORKFLOWS.glob("*.yml")) + yield from sorted(WORKFLOWS.glob("*.yaml")) + + +def concurrency_groups(workflow): + """Every group string a workflow declares, workflow-level and job-level. + + GitHub accepts both a bare string and a mapping, so a test that only + understands the mapping form goes blind exactly when someone adds the + other one. + """ + found = [] + for block in [workflow, *workflow.get("jobs", {}).values()]: + if not isinstance(block, dict): + continue + concurrency = block.get("concurrency") + if isinstance(concurrency, str): + found.append(concurrency) + elif isinstance(concurrency, dict): + found.append(concurrency.get("group", "")) + return found + + +PR_GROUP = load(PR_ENTRY)["concurrency"]["group"] + + +class TestCancelSupersededWorkflow(unittest.TestCase): + def setUp(self): + self.workflow = load(SUPERSEDED) + + def test_declares_no_concurrency_group(self): + self.assertEqual( + concurrency_groups(self.workflow), + [], + "entry-cancel-superseded.yml frees blocked concurrency groups; a group " + "of its own would queue it behind the very run it has to cancel", + ) + + def test_push_allowlist_matches_the_group_it_backs_up(self): + group_members = set() + for path in workflow_files(): + for group in concurrency_groups(load(path)): + if "global-ci-" in group and "github.ref_name" in group: + group_members.add(f".github/workflows/{path.name}") + + allowlist = { + line.strip() + for line in self._env("cancel-branch-runs", "INCLUDE_PATHS").splitlines() + if line.strip() + } + self.assertEqual( + allowlist, + group_members, + "the push allowlist must be exactly the members of the concurrency " + "group it backs up, otherwise it over- or under-cancels", + ) + + def test_each_job_groups_the_way_its_concurrency_group_is_keyed(self): + expected = { + "cancel-branch-runs": ("all", load(PUSH_ENTRY)["concurrency"]["group"]), + "cancel-pull-request-runs": ("event", PR_GROUP), + } + for job, (mode, group) in expected.items(): + keyed_on_event = "github.event_name" in group + self.assertEqual( + mode == "event", + keyed_on_event, + f"{job}: the grouping must mirror the concurrency group key {group}", + ) + self.assertEqual( + self._env(job, "KEEP_NEWEST_PER"), + mode, + f"{job} must select by 'not the newest run in the group', not by head " + "SHA: the payload SHA goes stale while the job waits for a runner, " + "and a skipped or token push moves the tip without creating a run", + ) + + def test_push_job_mirrors_the_cancel_rule_of_its_group(self): + gate = load(PUSH_ENTRY)["concurrency"]["cancel-in-progress"] + inner = gate.removeprefix("${{").removesuffix("}}").strip() + self.assertEqual( + self.workflow["jobs"]["cancel-branch-runs"]["if"], + "${{ github.event_name == 'push' && " + inner + " }}", + "the fallback mirrors the cancel rule of the group it backs up; a " + "changed polarity or a dropped event guard must fail here", + ) + + def test_covers_every_trigger_of_the_workflows_it_backs_up(self): + triggers = self.workflow[True] + self.assertEqual( + sorted(triggers["push"]["branches"]), + sorted( + branch + for branch in load(PUSH_ENTRY)[True]["push"]["branches"] + if branch != "main" + ), + "a branch that starts CI but no fallback keeps the deadlock; main is " + "excluded because runs on main are never cancelled", + ) + pr_types = set() + for event, block in load(PR_ENTRY)[True].items(): + if str(event).startswith("pull_request"): + pr_types.update(block["types"]) + self.assertEqual( + set(triggers["pull_request_target"]["types"]), + pr_types, + "an event that starts a PR run but no fallback keeps the deadlock", + ) + + def test_pull_request_job_runs_for_pull_request_target_only(self): + self.assertEqual( + self.workflow["jobs"]["cancel-pull-request-runs"]["if"], + "${{ github.event_name == 'pull_request_target' }}", + ) + + def _step(self, job_name): + run_suffix = CANCEL_JOBS[job_name] + steps = [ + step + for step in self.workflow["jobs"][job_name]["steps"] + if run_suffix in step.get("run", "") + ] + self.assertEqual( + len(steps), 1, f"expected one step running {run_suffix} in job {job_name}" + ) + return steps[0] + + def _env(self, job_name, key): + return self._step(job_name)["env"][key] + + +class TestCancelHelperReferences(unittest.TestCase): + def test_every_referenced_shell_script_exists(self): + pattern = re.compile(r"scripts/github/[A-Za-z0-9_./-]+\.sh") + for path in workflow_files(): + for match in pattern.findall(read_text(str(path))): + self.assertTrue( + (PROJECT_ROOT / match).is_file(), + f"{path.name} references missing {match}", + ) + + def test_every_sourced_helper_exists(self): + pattern = re.compile(r'source "\$\{SCRIPT_DIR\}/([A-Za-z0-9_.-]+)"') + cancel_dir = PROJECT_ROOT / "scripts" / "github" / "cancel" + sourced = set() + for path in sorted(cancel_dir.glob("*.sh")): + for name in pattern.findall(read_text(str(path))): + sourced.add(name) + self.assertTrue( + (cancel_dir / name).is_file(), + f"{path.name} sources missing {name}", + ) + self.assertIn("lib.sh", sourced) + + +if __name__ == "__main__": + unittest.main()