From 7fd495cdfc4e829d0d98e7215aab1dbe177e2624 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 31 Aug 2026 16:57:04 +0200 Subject: [PATCH] feat(ci): replay a role's README instructions on its smallest deploy row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The instructions test held a job of its own that picked one random role per run, so a README's Production block was proven roughly once per 265 roles. The replay now rides on a deploy row instead: no extra runner, and every role the sweep touches gets its documented one-command deploy exercised. The row is the role's smallest variant -- fewest services enabled -- because that is the shape the README describes, and it is chosen among the variants the sweep actually deploys, so a row cut as redundant coverage cannot silently drop the role from the test. A role qualifies only when its README carries a Production block and the guide's own mode is one the role runs and tests. The marked entry carries the guide mode in `instructions` and a πŸ“– in its job title, between the variant number and a priority ⭐. The guide mode is independent of the mode the rotation gave the row: a swarm row still replays in compose. The step runs last in the deploy job, after the swarm teardown, because the replay deploys the role a second time and needs the runner's ports. call-test-instructions.yml, cli.meta.ci.guide_select and the `instructions` dispatch input go with it. `redundant()` moves next to the marker so the chunker, the plan table and the marker share one definition. The plan table trades its πŸ“Š Weight column for πŸ“– Instructions; nothing read the weight any more, so it leaves the matrix entries too. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/README.md | 4 - .github/workflows/call-orchestrator.yml | 19 --- .github/workflows/call-test-deploy.yml | 15 +++ .github/workflows/call-test-instructions.yml | 105 ---------------- .github/workflows/entry-manual-steer.yml | 13 +- cli/administration/deploy/ci/runs.py | 9 +- cli/meta/ci/guide_select.py | 81 ------------- cli/meta/ci/matrix.py | 13 +- cli/meta/ci/plan.py | 12 +- .../tools/github/actions/README.md | 1 + .../tools/github/actions/instructions.md | 27 +++++ .../tools/github/actions/workflows.md | 5 +- .../workflows/test_ci_retrigger_inputs.py | 1 - .../cli/administration/deploy/ci/test_runs.py | 1 - .../python/cli/meta/ci/test_guide_select.py | 53 -------- tests/unit/python/cli/meta/ci/test_plan.py | 13 +- .../unit/python/utils/github/test_run_name.py | 1 - tests/unit/python/utils/roles/test_guide.py | 77 ++++++++++++ utils/github/variant/axes.py | 8 +- utils/github/variant/instructions.py | 61 ++++++++++ utils/roles/guide.py | 113 ++++++++++++++++++ utils/symbol_glossary.py | 1 + 22 files changed, 330 insertions(+), 303 deletions(-) delete mode 100644 .github/workflows/call-test-instructions.yml delete mode 100644 cli/meta/ci/guide_select.py create mode 100644 docs/contributing/tools/github/actions/instructions.md delete mode 100644 tests/unit/python/cli/meta/ci/test_guide_select.py create mode 100644 tests/unit/python/utils/roles/test_guide.py create mode 100644 utils/github/variant/instructions.py create mode 100644 utils/roles/guide.py diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 1c013893f4..1384cf75af 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -47,15 +47,12 @@ flowchart TB instmake["test-install-make.yml"] instpkgmgr["test-install-pkgmgr.yml"] mirror --> devenv["test-workspace: test-workspace.yml"] - buildci --> testguide["test-instructions.yml"] - mirror --> testguide chain --> donegate["done"] smoke --> donegate instmake --> donegate instpkgmgr --> donegate devenv --> donegate - testguide --> donegate end chunk0 --> deploy["call-test-deploy.yml"] @@ -294,7 +291,6 @@ flowchart TB relhighest -.->|"gh workflow run"| relver["release-version.yml"] relver --> imgbuildci["images-build-ci.yml"] manual["workflow_dispatch"] --> mirrorcleanup["images-mirror-cleanup.yml"] - manual --> deploywf["test-instructions.yml: run a role README Production command"] ``` Also manually dispatchable: `cron-images-mirror-all.yml`, `cron-images-cleanup-ci.yml`, diff --git a/.github/workflows/call-orchestrator.yml b/.github/workflows/call-orchestrator.yml index c3e4d88d97..81c0b61b07 100644 --- a/.github/workflows/call-orchestrator.yml +++ b/.github/workflows/call-orchestrator.yml @@ -72,11 +72,6 @@ on: required: false type: string default: "rotate" - instructions: - description: "Run the instructions test (role README Production command). 'auto' runs it on a global scope only, 'true'/'false' force it." - required: false - type: string - default: "auto" tor: description: "Tor axis of the deploy matrix: 'auto' rotates the onion over the tor-capable rows, 'enforced' onions all of them, 'exclusive' also drops the rows that cannot, 'disabled' onions none." required: false @@ -285,19 +280,6 @@ jobs: - test secrets: inherit - test-instructions: - if: ${{ inputs.instructions == 'true' || (inputs.instructions == 'auto' && (inputs.whitelist == '' || inputs.whitelist == '__ALL__')) }} - uses: ./.github/workflows/call-test-instructions.yml - needs: - - images-mirror-missing - - build-ci-images - - lint - - test - - validate-selection - with: - distros: "arch debian ubuntu fedora centos" - secrets: inherit - report-main-failures: name: 🐞 Report main role failures needs: @@ -331,7 +313,6 @@ jobs: - test-deploy-chunk-2 - test-runner-smoke - test-workspace - - test-instructions - test - lint - security-codeql diff --git a/.github/workflows/call-test-deploy.yml b/.github/workflows/call-test-deploy.yml index bba124dc5f..f61a757647 100644 --- a/.github/workflows/call-test-deploy.yml +++ b/.github/workflows/call-test-deploy.yml @@ -417,3 +417,18 @@ jobs: if: ${{ always() && matrix.mode == 'swarm' }} timeout-minutes: 15 run: scripts/tests/deploy/swarm/utils/clean/teardown.sh + + - name: "πŸ“– Replay the ${{ matrix.apps }} README instructions" + if: ${{ matrix.instructions != '' }} + env: + GUIDE_ROLE: ${{ matrix.apps }} + GUIDE_MODE: ${{ matrix.instructions }} + run: ./scripts/tests/deploy/distros.sh scripts/github/guide/one.sh + + - name: "⬆️ Upload instructions rescue diagnostics" + if: ${{ always() && matrix.instructions != '' && !env.ACT }} + uses: actions/upload-artifact@v7 + with: + name: rescue-diagnostics-instructions-${{ matrix.artifact }} + path: ${{ env.INFINITO_RESCUE_DIAGNOSTICS_BASE }}/ + if-no-files-found: ignore diff --git a/.github/workflows/call-test-instructions.yml b/.github/workflows/call-test-instructions.yml deleted file mode 100644 index fe3bc1be60..0000000000 --- a/.github/workflows/call-test-instructions.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: "πŸ“– Test: Instructions" - -on: - workflow_call: - inputs: - distros: - description: "Space-separated distro list" - required: true - type: string - role: - description: "Role to test; empty picks one at random" - required: false - type: string - default: "" - priority: - description: "Space-separated app ids; a random one is preferred over whitelist" - required: false - type: string - default: "" - whitelist: - description: "Space-separated app ids; a random one is used when no role or priority is given" - required: false - type: string - default: "" - workflow_dispatch: - inputs: - distros: - description: "🐧 Space-separated distro list" - required: true - type: string - role: - description: "πŸ“– Role to test; empty picks one at random" - required: false - type: string - default: "" - priority: - description: "⭐ App ids; a random one is preferred over the whitelist" - required: false - type: string - default: "" - whitelist: - description: "🎯 App ids; a random one runs when no role or priority is given" - required: false - type: string - default: "" - -permissions: - contents: read - packages: read - -jobs: - deploy: - name: πŸ“– Instructions - runs-on: ubuntu-latest - timeout-minutes: 345 - concurrency: - group: infinito-instructions - cancel-in-progress: false - queue: max - env: - INFINITO_IMAGE_TAG: ci-${{ github.sha }} - INFINITO_DISTROS: ${{ inputs.distros }} - steps: - - if: ${{ !env.ACT }} - uses: actions/checkout@v7.0.0 - - - name: "πŸ“₯ Load rescue-diagnostics SPOT into job env" - shell: bash - run: grep '^INFINITO_RESCUE_DIAGNOSTICS_BASE=' default.env >> "$GITHUB_ENV" - - - name: 🐍 Install instructions + compose deps (Act runner only) - if: ${{ env.ACT }} - shell: bash - run: | - PYTHON=python3 PIP_BREAK_SYSTEM_PACKAGES=1 bash scripts/install/python.sh - apt-get update -qq && apt-get install -y -qq gettext-base jq - - - name: 🎲 Select the role and deploy mode - shell: bash - run: | - set -euo pipefail - python3 -m cli.meta.ci.guide_select \ - --role "${{ inputs.role }}" \ - --priority "${{ inputs.priority }}" \ - --whitelist "${{ inputs.whitelist }}" | tee -a "$GITHUB_ENV" - - - name: πŸ”‘ Login to GHCR - if: ${{ !env.ACT }} - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: πŸ“– Replay the instructions across ${{ inputs.distros }} - shell: bash - run: ./scripts/tests/deploy/distros.sh scripts/github/guide/one.sh - - - name: ⬆️ Upload rescue diagnostics - if: ${{ always() && !env.ACT }} - uses: actions/upload-artifact@v7 - with: - name: rescue-diagnostics-instructions-${{ env.GUIDE_ROLE }} - path: ${{ env.INFINITO_RESCUE_DIAGNOSTICS_BASE }}/ - if-no-files-found: ignore diff --git a/.github/workflows/entry-manual-steer.yml b/.github/workflows/entry-manual-steer.yml index af8ab6b095..8fd1c6f438 100644 --- a/.github/workflows/entry-manual-steer.yml +++ b/.github/workflows/entry-manual-steer.yml @@ -1,5 +1,5 @@ name: "πŸ•ΉοΈ Manual" -run-name: "πŸ•ΉοΈ ${{ inputs.chunk_gate && 'πŸ›‘ ' || '' }}${{ inputs.offset != '0' && inputs.offset != '' && format('⏩{0} ', inputs.offset) || '' }}${{ inputs.workspace != 'auto' && format('πŸ§‘β€πŸ’»{0} ', inputs.workspace == 'true' && '是' || '否') || '' }}${{ inputs.instructions != 'auto' && format('πŸ“–{0} ', inputs.instructions == 'true' && '是' || '否') || '' }}${{ inputs.tor != 'auto' && format('{0} ', inputs.tor == 'disabled' && '🌐' || inputs.tor == 'enforced' && 'πŸ§…εΌΊεˆΆ' || 'πŸ§…η‹¬ε ') || '' }}${{ inputs.filesystem != '' && format('πŸ—„οΈ{0} ', inputs.filesystem) || '' }}${{ inputs.distros != '' && format('🐧{0} ', inputs.distros) || '' }}${{ inputs.lifecycles != '' && format('♻️{0} ', inputs.lifecycles) || '' }}${{ inputs.mode != 'auto' && format('πŸš€{0} ', inputs.mode) || '' }}${{ inputs.priority != '' && format('⭐{0} ', inputs.priority) || '' }}${{ inputs.whitelist != '' && format('🎯{0}', inputs.whitelist) || 'πŸ”€ diff (origin/main)' }}" +run-name: "πŸ•ΉοΈ ${{ inputs.chunk_gate && 'πŸ›‘ ' || '' }}${{ inputs.offset != '0' && inputs.offset != '' && format('⏩{0} ', inputs.offset) || '' }}${{ inputs.workspace != 'auto' && format('πŸ§‘β€πŸ’»{0} ', inputs.workspace == 'true' && '是' || '否') || '' }}${{ inputs.tor != 'auto' && format('{0} ', inputs.tor == 'disabled' && '🌐' || inputs.tor == 'enforced' && 'πŸ§…εΌΊεˆΆ' || 'πŸ§…η‹¬ε ') || '' }}${{ inputs.filesystem != '' && format('πŸ—„οΈ{0} ', inputs.filesystem) || '' }}${{ inputs.distros != '' && format('🐧{0} ', inputs.distros) || '' }}${{ inputs.lifecycles != '' && format('♻️{0} ', inputs.lifecycles) || '' }}${{ inputs.mode != 'auto' && format('πŸš€{0} ', inputs.mode) || '' }}${{ inputs.priority != '' && format('⭐{0} ', inputs.priority) || '' }}${{ inputs.whitelist != '' && format('🎯{0}', inputs.whitelist) || 'πŸ”€ diff (origin/main)' }}" concurrency: group: global-ci-${{ github.repository }}-${{ github.ref_name }} @@ -86,16 +86,6 @@ on: - "rotate" - "compose" - "swarm" - instructions: - description: "πŸ“– Run the instructions test. 'auto': on a global scope only" - required: false - default: "auto" - type: choice - options: - - "auto" - - "true" - - "false" - permissions: contents: read @@ -121,6 +111,5 @@ jobs: filesystem: ${{ inputs.filesystem }} workspace: ${{ inputs.workspace }} workspace_track: ${{ inputs.workspace_track }} - instructions: ${{ inputs.instructions }} tor: ${{ inputs.tor }} secrets: inherit diff --git a/cli/administration/deploy/ci/runs.py b/cli/administration/deploy/ci/runs.py index 1d793a14bd..64e17f301c 100644 --- a/cli/administration/deploy/ci/runs.py +++ b/cli/administration/deploy/ci/runs.py @@ -84,7 +84,6 @@ CONFIG_INPUTS = ( "filesystem", "chunk_gate", "workspace", - "instructions", ) LOG_INPUTS = ("tor",) @@ -370,13 +369,13 @@ def config_from_title(title: str) -> dict[str, str]: return {name: recorded[name] for name in CONFIG_INPUTS if name in recorded} -SUITE_JOB_IDS = {"workspace": "test-workspace", "instructions": "test-instructions"} +SUITE_JOB_IDS = {"workspace": "test-workspace"} def suite_passed(jobs: list[dict], suite: str) -> bool: - """Whether ``suite`` ('workspace' | 'instructions') ran to success in this - run. A suite that was skipped, cancelled or never dispatched has not - passed, so a retrigger keeps asking for it.""" + """Whether ``suite`` ('workspace') ran to success in this run. A suite that + was skipped, cancelled or never dispatched has not passed, so a retrigger + keeps asking for it.""" job_id = SUITE_JOB_IDS[suite] outcomes = [ _effective(job) diff --git a/cli/meta/ci/guide_select.py b/cli/meta/ci/guide_select.py deleted file mode 100644 index fb96e9c496..0000000000 --- a/cli/meta/ci/guide_select.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Pick the role and deploy mode for the Guide test. - -Role selection: an explicit ``--role`` wins; otherwise a random pick from -``--priority``, then ``--whitelist``, then every invokable role inside the -tested lifecycle envelope (``INFINITO_LIFECYCLES`` in ``default.env``) whose -guide deploy mode is enabled in ``meta/services.yml`` and not skipped via -``meta/tests.yml``. The mode is ``host`` for a role that ships no container -stack (installed straight onto the machine), else ``compose``. The distro is not picked here: the role is replayed on every -distro by ``scripts/tests/deploy/distros.sh``. Output is ``key=value`` lines -for ``$GITHUB_ENV``. -""" - -from __future__ import annotations - -import argparse -import random -import sys -from typing import TYPE_CHECKING - -from utils.cache.files import PROJECT_ROOT -from utils.roles.deploy import role_has_stack -from utils.roles.lifecycle import tested_lifecycles -from utils.roles.meta_lookup import get_role_mode_enabled, get_role_test_skips -from utils.roles.validation.invokable import ( - _get_invokable_paths, - _is_role_invokable, - _role_lifecycle, -) - -if TYPE_CHECKING: - from pathlib import Path - - -def _tokens(raw: str) -> list[str]: - return [t for t in raw.split() if t] - - -def _guide_mode(role_dir: Path) -> str: - return "compose" if role_has_stack(role_dir) else "host" - - -def _testable_roles() -> list[str]: - paths = _get_invokable_paths() - roles_dir = PROJECT_ROOT / "roles" - tested = set(tested_lifecycles()) - return sorted( - d.name - for d in roles_dir.iterdir() - if d.is_dir() - and _is_role_invokable(d.name, paths) - and _role_lifecycle(d) in tested - and _guide_mode(d) not in get_role_test_skips(d, role_name=d.name) - and get_role_mode_enabled(d, mode=_guide_mode(d), role_name=d.name) - ) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--role", default="") - parser.add_argument("--priority", default="") - parser.add_argument("--whitelist", default="") - args = parser.parse_args(argv) - - if args.role.strip(): - role = args.role.strip() - else: - pool = _tokens(args.priority) or _tokens(args.whitelist) or _testable_roles() - if not pool: - print("guide_select: no candidate roles", file=sys.stderr) - return 1 - role = random.choice(pool) # noqa: S311 - test-role pick, not cryptographic - - mode = _guide_mode(PROJECT_ROOT / "roles" / role) - - print(f"GUIDE_ROLE={role}") - print(f"GUIDE_MODE={mode}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/cli/meta/ci/matrix.py b/cli/meta/ci/matrix.py index ba20407b99..554a60cdf3 100644 --- a/cli/meta/ci/matrix.py +++ b/cli/meta/ci/matrix.py @@ -35,10 +35,10 @@ import sys from cli.meta.ci import chunks, query, slots from utils.cache.applications import get_variants -from utils.github.variant import axes, pools, selection, tor +from utils.github.variant import axes, instructions, pools, selection, tor from utils.roles.display import display_names -DROPPED = ("priority", "weight", "id", "covered", "clone") +DROPPED = ("priority", "id", "covered", "clone") """Entry keys the plan table reads but the matrix JSON withholds, so a deploy job cannot reference one: they describe the row's rank, not its deployment.""" @@ -149,13 +149,6 @@ def offset_index(raw: int | str | None, regular: list[dict[str, str]]) -> int: ) -def redundant(entry: dict[str, str]) -> bool: - """Whether the sweep drops this row as coverage an earlier row already has.""" - if entry["priority"] == "true": - return False - return entry["clone"] == "true" or entry["covered"] != "0" - - def chunks_of( entries: list[dict[str, str]], offset: int | str | None = 0 ) -> list[list[dict[str, str]]]: @@ -168,7 +161,7 @@ def chunks_of( regular = [ entry for entry in entries - if entry["priority"] != "true" and not redundant(entry) + if entry["priority"] != "true" and not instructions.redundant(entry) ] return [ sorted(chunk, key=axes.sort_key) diff --git a/cli/meta/ci/plan.py b/cli/meta/ci/plan.py index f7c3294a82..8fc8a80988 100644 --- a/cli/meta/ci/plan.py +++ b/cli/meta/ci/plan.py @@ -15,6 +15,10 @@ sweep's budget (raise ``--offset`` to reach it), βœ‚οΈ cut as redundant coverag a clone or a row an earlier one already embeds, which no sweep deploys. ``--cli`` renders fixed-width terminal tables instead of Markdown. +πŸ“– marks the row that replays the role's README instructions after its deploy, +and names the mode the replay runs in -- which is the guide's own, not the +row's (:mod:`utils.roles.guide`). + πŸ†” numbers the rows of this table, 1..n, and πŸ”° names the πŸ†” of the earlier row that already embeds this one -- empty when nothing does. A row with a πŸ”° is redundant coverage: whatever it would prove, the row it points at proves first. πŸ‘ marks the other reason a row is cut: it @@ -29,7 +33,7 @@ import sys from cli.meta.ci import matrix, query, slots from cli.meta.roles.applications.complexity.render import _dwidth -from utils.github.variant import axes, pools, tor +from utils.github.variant import axes, instructions, pools, tor from utils.roles.display import display_names from utils.symbol_glossary import to_emoji @@ -49,7 +53,7 @@ _COLUMNS = ( "tor", "triggered", "covered_by", - "weight", + "instructions", "clone", ) _GLYPH = {"triggered": "enabled", "distro": "distros"} @@ -98,7 +102,7 @@ def cells( positions.setdefault(entry.get("id", ""), str(counter)) for counter, entry in enumerate(entries, start=1): chunk = _chunk_of(entry, plan) - if matrix.redundant(entry): + if instructions.redundant(entry): status, where = _CUT, "" elif chunk is None: status, where = _OFF, "" @@ -118,7 +122,7 @@ def cells( to_emoji("tor" if entry["tor"] == "true" else "clearnet"), status, positions.get(covered, covered) if covered not in ("", "0") else "", - entry["weight"], + to_emoji(entry["instructions"]) if entry["instructions"] else "", to_emoji("clone") if entry.get("clone") == "true" else "", ) ) diff --git a/docs/contributing/tools/github/actions/README.md b/docs/contributing/tools/github/actions/README.md index e06e358fcb..10ddffc8fc 100644 --- a/docs/contributing/tools/github/actions/README.md +++ b/docs/contributing/tools/github/actions/README.md @@ -3,5 +3,6 @@ This directory documents GitHub Actions in this repository. - [workflows.md](workflows.md): catalog of every workflow file under `.github/workflows/`. +- [instructions.md](instructions.md): how the role README's Production block is replayed inside a deploy row. - [configuration.md](configuration.md): repository variables that control CI behaviour. - [secrets.md](secrets.md): repository secrets that workflows consume, and the failure mode when they are missing. diff --git a/docs/contributing/tools/github/actions/instructions.md b/docs/contributing/tools/github/actions/instructions.md new file mode 100644 index 0000000000..ff2ed50334 --- /dev/null +++ b/docs/contributing/tools/github/actions/instructions.md @@ -0,0 +1,27 @@ +# Instructions replay πŸ“– + +How CI proves that a role README's `### Production` block still deploys the role. + +For the workflow catalog see [workflows.md](workflows.md). + +## Where it runs πŸ—οΈ + +The replay rides along on a deploy row of the sweep instead of holding a job of its own. `utils/roles/guide.py` marks one row per role: + +- the role's **smallest variant**, the one whose `meta/variants.yml` overlay enables the fewest services, because that is the shape the README describes; +- only when the README carries a `### Production` heading; +- only when the guide's own mode (`compose` for a role shipping a container stack, `host` otherwise) is enabled in `meta/services.yml` and not listed under `skip` in `meta/tests.yml`. + +The marked row carries `instructions: ` in the matrix JSON and a πŸ“– in its job title, between the variant number and the ⭐ of a priority row. + +## What the row does πŸš€ + +`.github/workflows/call-test-deploy.yml` runs the replay as the job's last step, after the row's own deploy and after the swarm teardown: the replay deploys the role a second time and needs the runner's ports. It calls `scripts/tests/deploy/distros.sh scripts/github/guide/one.sh` with `GUIDE_ROLE` set to the row's app and `GUIDE_MODE` to the mode the marker carries, which is independent of the deploy mode the rotation gave the row. A swarm row therefore still replays a `compose` guide. + +A red deploy skips the replay: the step carries no `always()`, so the runner state a failed deploy leaves behind never reaches the guide. + +## Reading a failure πŸ” + +The replay writes its rescue snapshots under `INFINITO_RESCUE_DIAGNOSTICS_BASE//`, uploaded as `rescue-diagnostics-instructions-`. The deploy's own snapshots live in `rescue-diagnostics-`. + +To reproduce one locally, export `GUIDE_ROLE`, `GUIDE_MODE`, `INFINITO_DISTRO` and run `scripts/github/guide/one.sh`. diff --git a/docs/contributing/tools/github/actions/workflows.md b/docs/contributing/tools/github/actions/workflows.md index ad9c4ff858..bdd8982bb7 100644 --- a/docs/contributing/tools/github/actions/workflows.md +++ b/docs/contributing/tools/github/actions/workflows.md @@ -36,7 +36,7 @@ Trigger column legend: **auto** = fires automatically (push, pull_request, sched | Workflow | Description | Trigger | Inputs | |---|---|---|---| -| [call-orchestrator.yml](../../../../../.github/workflows/call-orchestrator.yml): `🎡 CI: Orchestrator` | Central coordinator. Runs fork-prereq wait, security, linting, CI image build, code tests, code-quality gate, DNS tests, mirror, deploy tests as a serial chain of chunk blocks (each chunk's discover job renders the sweep's plan table via `cli.meta.ci.plan` into the run summary), install tests, `test-workspace`, `test-instructions`, final `done` gate. | reusable | `distros` (optional; empty spreads the rows over every declared distro), `filesystem` (optional; empty spreads them over `zfs`, `btrfs`, `ext4`), `whitelist` (optional), `priority` (optional; app ids sorted to the head of the list and cut into chunks of their own, so every priority row is deployed before the first regular one starts, and deployed in every combination it can take: each variant Γ— each offered mode Γ— Tor and clearnet, in one sweep), `mode` (default `auto`), `sweep`, `resume_from_chunk` (default `0`), `chunk_gate` (default `true`), `image_wait_attempts` (default `1980`), `image_wait_sleep_seconds` (default `10`), `workspace` (default `auto`), `instructions` (default `auto`) | +| [call-orchestrator.yml](../../../../../.github/workflows/call-orchestrator.yml): `🎡 CI: Orchestrator` | Central coordinator. Runs fork-prereq wait, security, linting, CI image build, code tests, code-quality gate, DNS tests, mirror, deploy tests as a serial chain of chunk blocks (each chunk's discover job renders the sweep's plan table via `cli.meta.ci.plan` into the run summary), install tests, `test-workspace`, final `done` gate. | reusable | `distros` (optional; empty spreads the rows over every declared distro), `filesystem` (optional; empty spreads them over `zfs`, `btrfs`, `ext4`), `whitelist` (optional), `priority` (optional; app ids sorted to the head of the list and cut into chunks of their own, so every priority row is deployed before the first regular one starts, and deployed in every combination it can take: each variant Γ— each offered mode Γ— Tor and clearnet, in one sweep), `mode` (default `auto`), `sweep`, `resume_from_chunk` (default `0`), `chunk_gate` (default `true`), `image_wait_attempts` (default `1980`), `image_wait_sleep_seconds` (default `10`), `workspace` (default `auto`) | ### Security and linting πŸ›‘οΈπŸ” @@ -71,8 +71,7 @@ Trigger column legend: **auto** = fires automatically (push, pull_request, sched | Workflow | Description | Trigger | Inputs | |---|---|---|---| | [call-test-dns.yml](../../../../../.github/workflows/call-test-dns.yml): `πŸ’¬ Test: DNS` | Validates DNS resolution across target distributions. | reusable, manual | `distros` (required on call; default `debian` on manual) | -| [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml): `πŸš€ Test: Deploy` | Deploys one chunk of the sweep. Every matrix row is a single `role#variant` carrying the deploy mode, onion state, distro and filesystem the rotation assigned it: `swarm` rows get a simulated 3-node DinD cluster with an NFS server plus a drain/reschedule state-survival assertion, `compose` and `host` rows run the single-node drill. When `whitelist` is empty, the discover job derives one from the branch's diff vs `origin/main`; see the "Diff-driven app selection" subsection in [pipeline.md](../../../artefact/git/pipeline.md). | reusable | `index`, `sweep` (required), `distros` (optional pool), `filesystem` (optional pool), `whitelist` (optional; explicit value wins over diff), `priority`, `lifecycles`, `modes` (default `auto`), `tor`, `marker` | -| [call-test-instructions.yml](../../../../../.github/workflows/call-test-instructions.yml): `πŸ“– Test: Instructions` | Extracts the `### Production` command from a role README (a random role when `role` is empty) and executes it against the runner: `` becomes `localhost` (ansible_connection local) and the Docker socket replaces the TTY flag. Verifies the documented one-command deploy actually works, seeding the runΒ΄s freshly built CI image (needs `call-images-build-ci.yml`) instead of the published default. Gates the final `done` job. | reusable, manual | `distros` (required), `role` (optional; empty picks one at random), `priority` (optional), `whitelist` (optional) | +| [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml): `πŸš€ Test: Deploy` | Deploys one chunk of the sweep. Every matrix row is a single `role#variant` carrying the deploy mode, onion state, distro and filesystem the rotation assigned it: `swarm` rows get a simulated 3-node DinD cluster with an NFS server plus a drain/reschedule state-survival assertion, `compose` and `host` rows run the single-node drill. When `whitelist` is empty, the discover job derives one from the branch's diff vs `origin/main`; see the "Diff-driven app selection" subsection in [pipeline.md](../../../artefact/git/pipeline.md). A row marked πŸ“– -- the role's smallest variant, see [instructions.md](instructions.md) -- replays the role README's `### Production` block on the same runner once the deploy has released it. | reusable | `index`, `sweep` (required), `distros` (optional pool), `filesystem` (optional pool), `whitelist` (optional; explicit value wins over diff), `priority`, `lifecycles`, `modes` (default `auto`), `tor`, `marker` | | [call-test-install-make.yml](../../../../../.github/workflows/call-test-install-make.yml): `πŸ“₯ Test: Install Make` | Validates the `make install` entry points. | reusable | none | | [call-test-install-pkgmgr.yml](../../../../../.github/workflows/call-test-install-pkgmgr.yml): `πŸ“₯ Test: Install Package Manager` | Validates installation via the upstream package manager. | reusable | none | | [call-test-workspace.yml](../../../../../.github/workflows/call-test-workspace.yml): `πŸ’» Test: Workspace` | Builds and exercises the dev-runtime image matrix. | reusable, manual | none | diff --git a/tests/integration/meta/workflows/test_ci_retrigger_inputs.py b/tests/integration/meta/workflows/test_ci_retrigger_inputs.py index 16470e7c05..20c937aac2 100644 --- a/tests/integration/meta/workflows/test_ci_retrigger_inputs.py +++ b/tests/integration/meta/workflows/test_ci_retrigger_inputs.py @@ -47,7 +47,6 @@ _VALUES = { "sweep": "7", "workspace": "true", "workspace_track": "compose", - "instructions": "false", } """A value for every dispatch input, distinct from that input's default so a dropped one shows up as a missing key rather than as a coincidence.""" diff --git a/tests/unit/python/cli/administration/deploy/ci/test_runs.py b/tests/unit/python/cli/administration/deploy/ci/test_runs.py index 4538838f5d..3d37e7e97c 100644 --- a/tests/unit/python/cli/administration/deploy/ci/test_runs.py +++ b/tests/unit/python/cli/administration/deploy/ci/test_runs.py @@ -395,7 +395,6 @@ class TestConfigFromTitle(unittest.TestCase): "filesystem": "ext4", "chunk_gate": "false", "workspace": "true", - "instructions": "false", } self.assertEqual(runs.config_from_title(render(dispatched)), dispatched) diff --git a/tests/unit/python/cli/meta/ci/test_guide_select.py b/tests/unit/python/cli/meta/ci/test_guide_select.py deleted file mode 100644 index 1af990e3ac..0000000000 --- a/tests/unit/python/cli/meta/ci/test_guide_select.py +++ /dev/null @@ -1,53 +0,0 @@ -from __future__ import annotations - -import textwrap -import unittest -from pathlib import Path -from tempfile import TemporaryDirectory - -from cli.meta.ci import guide_select -from utils.cache.files import PROJECT_ROOT -from utils.roles.mapping import ROLE_FILE_META_TESTS -from utils.roles.meta_lookup import MetaServicesShapeError, get_role_test_skips - - -class GuideModeTests(unittest.TestCase): - def test_role_without_stack_deploys_in_host_mode(self): - self.assertEqual( - guide_select._guide_mode(PROJECT_ROOT / "roles" / "svc-storage-nfs-client"), - "host", - ) - - def test_role_with_stack_deploys_in_compose_mode(self): - self.assertEqual( - guide_select._guide_mode(PROJECT_ROOT / "roles" / "web-app-yourls"), - "compose", - ) - - -class TestableRolesTests(unittest.TestCase): - def test_pool_excludes_roles_whose_guide_mode_is_skipped(self): - pool = guide_select._testable_roles() - self.assertNotIn("svc-storage-nfs-client", pool) - self.assertTrue(pool) - - def test_malformed_tests_yml_fails_selection_loudly(self): - with TemporaryDirectory() as tmp: - role_dir = Path(tmp) / "web-app-broken" - tests_path = role_dir / ROLE_FILE_META_TESTS - tests_path.parent.mkdir(parents=True) - tests_path.write_text( - textwrap.dedent( - """ - --- - skip: not-a-list - """ - ), - encoding="utf-8", - ) - with self.assertRaises(MetaServicesShapeError): - get_role_test_skips(role_dir, role_name=role_dir.name) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/python/cli/meta/ci/test_plan.py b/tests/unit/python/cli/meta/ci/test_plan.py index d8cb6e38ec..9b7d512418 100644 --- a/tests/unit/python/cli/meta/ci/test_plan.py +++ b/tests/unit/python/cli/meta/ci/test_plan.py @@ -15,6 +15,7 @@ def _entry( identifier: str = "0", covered: str = "0", clone: bool = False, + instructions: str = "", ) -> dict: return { "apps": app, @@ -23,7 +24,7 @@ def _entry( "tor": "true" if variant == "0" else "false", "disable": "", "priority": "true" if priority else "false", - "weight": "42", + "instructions": instructions, "id": identifier, "covered": covered, "clone": "true" if clone else "false", @@ -68,6 +69,16 @@ class TestCells(unittest.TestCase): self.assertEqual(rows[0][clone], "") self.assertEqual(rows[-1][clone], to_emoji("clone")) + def test_the_instructions_column_names_the_mode_the_replay_runs_in(self) -> None: + entries = [ + *_ENTRIES, + _entry("web-app-d", "0", "swarm", instructions="compose"), + ] + rows = plan.cells(entries, [_PRIORITY, _REGULAR]) + column = plan._COLUMNS.index("instructions") + self.assertEqual(rows[0][column], "") + self.assertEqual(rows[-1][column], to_emoji("compose")) + def test_the_mode_is_rendered_as_its_glyph(self) -> None: rows = plan.cells(_ENTRIES, [_PRIORITY, _REGULAR]) mode = plan._COLUMNS.index("mode") diff --git a/tests/unit/python/utils/github/test_run_name.py b/tests/unit/python/utils/github/test_run_name.py index 6e61ba98c3..6813b4b976 100644 --- a/tests/unit/python/utils/github/test_run_name.py +++ b/tests/unit/python/utils/github/test_run_name.py @@ -128,7 +128,6 @@ class DeclaredTemplateTests(unittest.TestCase): dispatched = { "chunk_gate": "true", "workspace": "true", - "instructions": "false", "filesystem": "ext4", "distros": "debian arch", "lifecycles": "stable", diff --git a/tests/unit/python/utils/roles/test_guide.py b/tests/unit/python/utils/roles/test_guide.py new file mode 100644 index 0000000000..49c3bcf1de --- /dev/null +++ b/tests/unit/python/utils/roles/test_guide.py @@ -0,0 +1,77 @@ +import unittest +from typing import ClassVar +from unittest import mock + +from utils.cache.applications import get_variants +from utils.roles.guide import guide_deployable, guide_variant, smallest_variant + + +def _variant(*enabled: str) -> dict: + return {"services": {name: {"enabled": True} for name in enabled}} + + +class TestSmallestVariant(unittest.TestCase): + def test_the_variant_with_the_fewest_enabled_services_wins(self) -> None: + variants = [_variant("a", "b", "c"), _variant("a"), _variant("a", "b")] + self.assertEqual(smallest_variant(variants), "1") + + def test_a_tie_goes_to_the_lower_index(self) -> None: + self.assertEqual(smallest_variant([_variant("a"), _variant("b")]), "0") + + def test_a_disabled_service_does_not_count(self) -> None: + variants = [ + {"services": {"a": {"enabled": True}, "b": {"enabled": False}}}, + _variant("a", "b"), + ] + self.assertEqual(smallest_variant(variants), "0") + + def test_a_variant_without_services_counts_as_empty(self) -> None: + self.assertEqual(smallest_variant([_variant("a"), {}]), "1") + + def test_a_variant_the_sweep_cuts_hands_the_replay_on(self) -> None: + """A row no sweep deploys would drop the role from the test entirely.""" + variants = [_variant("a"), _variant("a", "b"), _variant("a", "b", "c")] + self.assertEqual(smallest_variant(variants, {"1", "2"}), "1") + + def test_a_role_with_no_deployed_variant_carries_nothing(self) -> None: + self.assertEqual(smallest_variant([_variant("a")], set()), "") + + +class TestGuideVariant(unittest.TestCase): + _VARIANTS: ClassVar[dict] = {"web-app-x": [_variant("a", "b"), _variant("a")]} + _ALL: ClassVar[set] = {"0", "1"} + + def test_the_smallest_deployed_variant_carries_the_replay(self) -> None: + with mock.patch("utils.roles.guide.guide_deployable", return_value="compose"): + self.assertEqual( + guide_variant("web-app-x", self._VARIANTS, self._ALL), ("1", "compose") + ) + + def test_a_role_the_guide_cannot_deploy_carries_nothing(self) -> None: + with mock.patch("utils.roles.guide.guide_deployable", return_value=""): + self.assertEqual( + guide_variant("web-app-x", self._VARIANTS, self._ALL), ("", "") + ) + + def test_without_variant_data_no_row_is_the_smallest(self) -> None: + self.assertEqual(guide_variant("web-app-x", None, self._ALL), ("", "")) + self.assertEqual(guide_variant("web-app-x", {}, self._ALL), ("", "")) + + +class TestGuideDeployable(unittest.TestCase): + """Against the real roles, which is what the marker is computed over.""" + + def test_a_role_without_a_readme_production_block_is_excluded(self) -> None: + self.assertEqual(guide_deployable("does-not-exist"), "") + + def test_the_mode_follows_whether_the_role_ships_a_stack(self) -> None: + self.assertEqual(guide_deployable("web-app-nextcloud"), "compose") + + def test_the_replay_reaches_more_roles_than_the_single_random_pick(self) -> None: + """The job it replaced tested exactly one role per run.""" + marked = [app for app in get_variants() if guide_deployable(app)] + self.assertGreater(len(marked), 100) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/github/variant/axes.py b/utils/github/variant/axes.py index 8caafe11c2..f00940b12d 100644 --- a/utils/github/variant/axes.py +++ b/utils/github/variant/axes.py @@ -42,6 +42,7 @@ import os import re from typing import TYPE_CHECKING, NamedTuple +from utils.github.variant import instructions from utils.github.variant.pools import DISTROS, FILESYSTEMS, rotate from utils.github.variant.tor import ( TOR_DEPLOY_MODES, @@ -63,7 +64,8 @@ MODES = ("compose", "swarm", "host") LOCAL_GLYPH = to_emoji("test_host") _AXIS_GLYPHS = ( - "".join(to_emoji(word) for word in ("tor", "clearnet", "priority")) + LOCAL_GLYPH + "".join(to_emoji(word) for word in ("tor", "clearnet", "priority", "instructions")) + + LOCAL_GLYPH ) @@ -393,7 +395,7 @@ def assign( else "false", "disable": "" if enabled else "tor", "priority": "true" if priority else "false", - "weight": str(row.get("weight", 0)), + "instructions": "", "id": str(row.get("id", 0)), "covered": str(row.get("covered_by", 0)), "clone": "true" if row.get("clone") else "false", @@ -404,7 +406,7 @@ def assign( + (f" {to_emoji('priority')}" if priority else ""), } ) - return _one_per_deploy(entries) + return instructions.mark(_one_per_deploy(entries), variants_per_app) def _one_per_deploy(entries: list[dict[str, str]]) -> list[dict[str, str]]: diff --git a/utils/github/variant/instructions.py b/utils/github/variant/instructions.py new file mode 100644 index 0000000000..5b274a404b --- /dev/null +++ b/utils/github/variant/instructions.py @@ -0,0 +1,61 @@ +"""Which matrix entry replays its role's README instructions. + +The replay used to be a job of its own, picking one random role per run. It +rides on a deploy row instead: the role's smallest variant among the rows this +sweep actually deploys, marked with ``instructions`` (the mode the replay runs +in) and a πŸ“– in the job title. :mod:`utils.roles.guide` decides which variant +that is; this module is what turns the decision into matrix entries. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from utils.roles.guide import guide_variant +from utils.symbol_glossary import to_emoji + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + from typing import Any + + +def redundant(entry: Mapping[str, str]) -> bool: + """Whether the sweep drops this row as coverage an earlier row already has.""" + if entry["priority"] == "true": + return False + return entry["clone"] == "true" or entry["covered"] != "0" + + +def mark( + entries: list[dict[str, str]], + variants_per_app: Mapping[str, Sequence[Mapping[str, Any]]] | None, +) -> list[dict[str, str]]: + """Hand one entry per role the replay, and give its label the πŸ“–. + + Only the first entry of the chosen variant: a priority row runs every mode + and onion state it offers, and replaying the same README block in each of + them proves nothing the first replay did not. The glyph goes between the + variant number and the ⭐ of a priority row, with no space between the two: + they are one marker block, not two axes. + """ + deployed: dict[str, set[str]] = {} + for entry in entries: + if not redundant(entry): + deployed.setdefault(entry["apps"], set()).add(entry["variant"]) + star = f" {to_emoji('priority')}" + marked: set[str] = set() + for entry in entries: + app = entry["apps"] + if app in marked or redundant(entry): + continue + variant, mode = guide_variant(app, variants_per_app, deployed.get(app, set())) + if not mode or entry["variant"] != variant: + continue + entry["instructions"] = mode + entry["label"] = ( + entry["label"].removesuffix(star) + + f" {to_emoji('instructions')}" + + (to_emoji("priority") if entry["priority"] == "true" else "") + ) + marked.add(app) + return entries diff --git a/utils/roles/guide.py b/utils/roles/guide.py new file mode 100644 index 0000000000..96fa27f586 --- /dev/null +++ b/utils/roles/guide.py @@ -0,0 +1,113 @@ +"""Which deploy row replays the role's README instructions, and in what mode. + +The instructions test used to be a job of its own that picked one random role +per run. Riding along on a deploy row instead costs no extra runner and covers +every role the sweep touches. The row it rides on is the role's smallest +variant -- the one enabling the fewest services -- because that is the shape +the README's Production block describes: the bare role, with the optional +providers switched off. + +The replay is a second deploy on the same runner, so the row it picks must be +one the guide can actually deploy: the README has to carry a Production block, +and the guide's own mode (``compose`` for a role that ships a stack, ``host`` +for one installed onto the machine) has to be a mode the role runs and tests. +""" + +from __future__ import annotations + +from functools import cache +from typing import TYPE_CHECKING + +from utils.cache.files import PROJECT_ROOT +from utils.roles.deploy import role_has_stack +from utils.roles.meta_lookup import get_role_mode_enabled, get_role_test_skips + +if TYPE_CHECKING: + from collections.abc import Collection, Mapping, Sequence + from typing import Any + +PRODUCTION_HEADING = "### Production" +"""The README section :file:`scripts/github/guide/compose_deploy.sh` replays. +A role without it has no instructions to test.""" + + +def _enabled_services(variant: Mapping[str, Any]) -> int: + services = variant.get("services") or {} + return sum( + 1 + for config in services.values() + if isinstance(config, dict) and config.get("enabled") + ) + + +def smallest_variant( + variants: Sequence[Mapping[str, Any]], allowed: Collection[str] | None = None +) -> str: + """The variant that enables the fewest services, as the matrix writes it; + the lowest index wins a tie, and ``''`` means none qualifies. + + Args: + variants: the role's rendered variant configs, in index order. + allowed: the variants this sweep actually deploys. A variant the sweep + cuts as redundant coverage never reaches a runner, so hanging the + replay on it would silently drop the role from the instructions + test; the next-smallest deployed variant carries it instead. + """ + candidates = [ + index + for index in range(len(variants)) + if allowed is None or str(index) in allowed + ] + if not candidates: + return "" + return str( + min(candidates, key=lambda index: (_enabled_services(variants[index]), index)) + ) + + +@cache +def guide_deployable(app: str) -> str: + """The mode the guide would replay *app* in, or ``''`` when it cannot. + + Independent of the variant, and cached: the answer is a property of the + role's files, and :func:`utils.github.variant.axes.assign` asks it once per + row of every chunk. + """ + role_dir = PROJECT_ROOT / "roles" / app + readme = role_dir / "README.md" + if not readme.is_file() or PRODUCTION_HEADING not in readme.read_text( + encoding="utf-8" + ): + return "" + mode = "compose" if role_has_stack(role_dir) else "host" + if mode in get_role_test_skips(role_dir, role_name=app): + return "" + if not get_role_mode_enabled(role_dir, mode=mode, role_name=app): + return "" + return mode + + +def guide_variant( + app: str, + variants_per_app: Mapping[str, Sequence[Mapping[str, Any]]] | None, + allowed: Collection[str], +) -> tuple[str, str]: + """The variant of *app* that carries the replay, and the mode it runs in. + + ``('', '')`` when the role has no instructions to replay or this sweep + deploys no variant that could carry them. + + Args: + app: the role. + variants_per_app: rendered variant configs per app; without them no + variant can be told to be the smallest one. + allowed: the variants this sweep deploys, as the matrix writes them. + """ + variants = (variants_per_app or {}).get(app) + if not variants: + return "", "" + mode = guide_deployable(app) + if not mode: + return "", "" + variant = smallest_variant(variants, allowed) + return (variant, mode) if variant else ("", "") diff --git a/utils/symbol_glossary.py b/utils/symbol_glossary.py index eff690c4a3..004cc846b4 100644 --- a/utils/symbol_glossary.py +++ b/utils/symbol_glossary.py @@ -16,6 +16,7 @@ SYMBOLS: dict[str, str] = { "disabled": "❌", "skip": "⏭️", "priority": "⭐", + "instructions": "πŸ“–", "distros": "🐧", "arch": "🏹", "debian": "πŸŒ€",