diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 85e739661c..9ffd29ff4d 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -21,50 +21,52 @@ flowchart TB eman --> orch subgraph orchestrator["ci-orchestrator.yml jobs"] - plan["plan: run-summary table"] waitfork["wait-fork-prereq-run"] --> forkready["fork-prereqs-ready"] - lintwf["lint.yml: make lint + hadolint"] --> qualitygate["code-quality-gate"] - testwf["test.yml: make test"] --> qualitygate - codeql["security-codeql.yml"] --> qualitygate - - qualitygate --> buildci["build-ci-images: images-build-ci.yml"] - buildci --> dns["test-dns.yml"] + lintwf["lint.yml: make lint + hadolint"] + testwf["test.yml: make test"] + codeql["security-codeql.yml"] + buildci["build-ci-images: images-build-ci.yml"] --> dns["test-dns.yml"] mirror["images-mirror-missing.yml"] + seq["sequencing: serial or parallel, per line"] - dns --> snprio["test-deploy-single-node-priority"] - mirror --> snprio - buildci --> swarmprio["test-deploy-swarm-priority"] - mirror --> swarmprio + subgraph prio["priority line, skipped without a priority input"] + swarmprio["test-deploy-swarm-priority"] + swarmprio -->|"serial"| composeprio["test-deploy-compose-priority"] + composeprio -->|"serial"| hostprio["test-deploy-host-priority"] + snprio["test-deploy-single-node-priority: parallel"] + end - snprio -->|"all priority jobs green"| snreg["test-deploy-single-node"] - swarmprio -->|"all priority jobs green"| snreg - snprio -->|"all priority jobs green"| swarmreg["test-deploy-swarm"] - swarmprio -->|"all priority jobs green"| swarmreg - dns --> snreg - mirror --> snreg - buildci --> swarmreg - mirror --> swarmreg + subgraph reg["regular line"] + swarmreg["test-deploy-swarm"] + swarmreg -->|"serial"| composereg["test-deploy-compose"] + composereg -->|"serial"| hostreg["test-deploy-host"] + snreg["test-deploy-single-node: parallel"] + end + + lintwf --> prio + testwf --> prio + dns --> prio + mirror --> prio + seq --> prio + prio -->|"all priority jobs green"| reg swarmreg --> smoke["test-runner-smoke.yml"] + prio --> report["report-main-failures"] + reg --> report - qualitygate --> instmake["test-install-make.yml"] - qualitygate --> instpkgmgr["test-install-pkgmgr.yml"] - instmake --> instgate["test-install-gate"] - instpkgmgr --> instgate - - qualitygate --> devenv["test-workspace: test-workspace.yml"] - mirror --> devenv - + instmake["test-install-make.yml"] + instpkgmgr["test-install-pkgmgr.yml"] + mirror --> devenv["test-workspace: test-workspace.yml"] buildci --> testguide["test-instructions.yml"] mirror --> testguide - snprio --> donegate["done"] - swarmprio --> donegate - snreg --> donegate - swarmreg --> donegate + prio --> donegate["done"] + reg --> donegate + seq --> donegate smoke --> donegate - instgate --> donegate + instmake --> donegate + instpkgmgr --> donegate devenv --> donegate testguide --> donegate end @@ -73,14 +75,107 @@ flowchart TB snreg --> singlenode singlenode --> deploycompose["test-deploy-compose.yml"] singlenode --> deployhost["test-deploy-host.yml"] + composeprio --> deploycompose + composereg --> deploycompose + hostprio --> deployhost + hostreg --> deployhost swarmprio --> deployswarm["test-deploy-swarm.yml"] swarmreg --> deployswarm ``` -`test-deploy-single-node-priority` and `test-deploy-swarm-priority` run only -when the orchestrator's `priority` input is set; with it empty they are -skipped and the regular deploy jobs start directly. The regular jobs receive -the priority ids as `blacklist`, so each role deploys in exactly one line. +The priority line runs only when the orchestrator's `priority` input is set; +with it empty every priority job is skipped and the regular line starts +directly. The regular jobs receive the priority ids as `blacklist`, so each +role deploys in exactly one line. + +## Deploy-line sequencing + +A **line** is one half of a run: the priority line (`โญ`, the roles named in +`priority`) or the regular line (`๐Ÿ”`, everything else). Each line deploys up +to three **modes**: swarm, compose, host. The lines are always sequential +relative to each other; this section is about the order *inside* one line. + +### Why the order exists + +GitHub cancels a job that has sat queued for 24 hours. That clock starts when +the job is queued, not when the run starts, and a `needs:` edge delays queueing +until the dependency completes. A line that starts all three modes at once +queues its whole matrix in one go, so with more roles than runner slots the +tail of that matrix waits past the cut and dies unrun. Deploying the modes one +after the other restarts the clock per mode. + +Serialising costs wall-clock, so it is only worth it above a certain size. The +run does not pay it on a small diff. + +### The decision + +The `sequencing` job resolves the effective whitelist (the same +`scripts/github/resolve/effective_whitelist.sh` the discover steps use) and +calls `cli.meta.ci.sequencing` once per line, emitting `serial` or `parallel` +as a job output. + +Job counts are taken on the row basis each mode actually runs on: swarm +selections are `role#variant` tokens that map 1:1 onto jobs, compose and host +selections are whole roles whose variants pack into bundles +(`utils.github.variant.bundles`). The count deliberately omits the +runner-storage filter that `scripts/meta/resolve/apps.sh` applies inside +GitHub Actions, so it can only overestimate โ€” erring towards the sequential +layout, which cannot be cancelled. + +| `sequencing` input | Behaviour | +|---|---| +| `auto` (default) | `serial` above `INFINITO_CI_SEQUENTIAL_THRESHOLD` jobs in that line, `parallel` at or below it | +| `serial` | forced; skips the count entirely | +| `parallel` | forced; skips the count entirely | + +The threshold lives in [`default.env`](../../default.env) as +`INFINITO_CI_SEQUENTIAL_THRESHOLD`. Both lines are decided independently: a +small priority line can run parallel while the regular line behind it runs +serial. + +### The two layouts + +Serial order is **swarm, then compose, then host** โ€” heaviest mode first, so +the longest queue drains while the run is youngest. + +- **parallel** โ€” `test-deploy-swarm-priority` plus + `test-deploy-single-node-priority`, which fans compose and host out + together. This is the layout the pipeline had before sequencing existed. +- **serial** โ€” `test-deploy-swarm-priority` โ†’ + `test-deploy-compose-priority` โ†’ `test-deploy-host-priority`, each chained + on the previous through `needs:`. + +The regular line mirrors this with the unsuffixed job names. + +A `needs:` edge is static, so a workflow cannot make one conditional. Both +layouts therefore exist as separate jobs and the `sequencing` output skips +one of them. Because a skipped dependency otherwise skips its dependents, the +serial jobs carry `always()` plus explicit `needs..result` checks on the +real gates. The swarm job depends on no sequencing decision at all: it leads +in either layout. + +### Stopping on failure + +`mode_fail_fast` (default `true`) decides whether a serial line stops at its +first failed mode: + +| Value | Behaviour | +|---|---| +| `true` | swarm fails โ†’ compose and host of that line are skipped | +| `false` | every mode deploys and reports; the run still ends red | + +`skipped` counts as passed, so a mode absent from `modes` never blocks the +chain. The parallel layout ignores the switch โ€” with no order there is nothing +to stop. The setting is a checkbox on `entry-manual.yml`; the other entry +points take the default. + +### Job budget + +`cli.meta.ci.slots` divides the run's 256-job cap between the deploy +matrices and reads the orchestrator to do it, so every deploy caller โ€” +including the serial twins โ€” must be listed in its `_DEPLOY_CALLERS`. +A caller missing there is charged a guessed dynamic-matrix estimate instead +of its single discover job. ## Cancellation diff --git a/.github/workflows/ci-orchestrator.yml b/.github/workflows/ci-orchestrator.yml index 70ac76f499..6d48bc1a2c 100644 --- a/.github/workflows/ci-orchestrator.yml +++ b/.github/workflows/ci-orchestrator.yml @@ -26,6 +26,16 @@ on: required: false type: string default: "" + sequencing: + description: "๐Ÿšฆ Deploy-mode order per line. 'serial': swarm, then compose, then host. 'parallel': all at once. 'auto': serial above the job threshold." + required: false + type: string + default: "auto" + mode_fail_fast: + description: "๐Ÿ›‘ Stop a serial line at its first failed mode. Off: keep deploying the remaining modes." + required: false + type: boolean + default: true filesystem: description: "๐Ÿ—„๏ธ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three." required: false @@ -162,22 +172,65 @@ jobs: image_wait_sleep_seconds: ${{ inputs.image_wait_sleep_seconds }} secrets: inherit - test-deploy-single-node-priority: - name: โญ Priority ยท single-node - if: ${{ inputs.priority != '' && (contains(inputs.modes, 'compose') || contains(inputs.modes, 'host')) }} - needs: - - test-dns - - images-mirror-missing - - lint - - test - uses: ./.github/workflows/test-deploy-single-node.yml - with: - distros: ${{ inputs.distros }} - whitelist: ${{ inputs.priority }} - lifecycles: ${{ inputs.lifecycles }} - filesystem: ${{ inputs.filesystem }} - modes: ${{ inputs.modes }} - marker: " โญ" + sequencing: + name: ๐Ÿšฆ Sequencing + runs-on: ubuntu-latest + timeout-minutes: 60 + outputs: + priority: ${{ steps.priority.outputs.sequencing }} + regular: ${{ steps.regular.outputs.sequencing }} + env: + INFINITO_APP_DISCOVERY_RUNNER: host + INFINITO_LIFECYCLES: ${{ inputs.lifecycles }} + INFINITO_HOST_EXCLUDE_COMPOSE: ${{ contains(inputs.modes, 'compose') && 'true' || 'false' }} + steps: + - uses: actions/checkout@v7.0.0 + with: + fetch-depth: 0 + + - name: ๐Ÿ”ง Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: ๐Ÿ“ฆ Install project (host venv) + shell: bash + run: make install + + - name: ๐Ÿ” Resolve effective whitelist (diff vs origin/main) + id: resolve_wl + shell: bash + env: + INPUT_WHITELIST: ${{ inputs.whitelist }} + run: ./scripts/github/resolve/effective_whitelist.sh + + - name: โญ Decide the priority line + id: priority + if: ${{ inputs.priority != '' }} + shell: bash + run: | + source scripts/meta/env/load.sh + "${PYTHON}" -m cli.meta.ci.sequencing \ + --modes "${{ inputs.modes }}" \ + --whitelist "${{ inputs.priority }}" \ + --lifecycles "${{ inputs.lifecycles }}" \ + --choice "${{ inputs.sequencing }}" \ + | tee -a "$GITHUB_OUTPUT" + + - name: ๐Ÿ” Decide the regular line + id: regular + shell: bash + run: | + source scripts/meta/env/load.sh + "${PYTHON}" -m cli.meta.ci.sequencing \ + --modes "${{ inputs.modes }}" \ + --whitelist "${{ steps.resolve_wl.outputs.whitelist }}" \ + --blacklist "${{ inputs.priority }}" \ + --lifecycles "${{ inputs.lifecycles }}" \ + --choice "${{ inputs.sequencing }}" \ + | tee -a "$GITHUB_OUTPUT" test-deploy-swarm-priority: name: โญ Priority ยท swarm @@ -195,21 +248,135 @@ jobs: filesystem: ${{ inputs.filesystem }} marker: " โญ" + test-deploy-single-node-priority: + name: โญ Priority ยท single-node + if: >- + ${{ inputs.priority != '' + && needs.sequencing.outputs.priority == 'parallel' + && (contains(inputs.modes, 'compose') || contains(inputs.modes, 'host')) }} + needs: + - test-dns + - images-mirror-missing + - lint + - test + - sequencing + uses: ./.github/workflows/test-deploy-single-node.yml + with: + distros: ${{ inputs.distros }} + whitelist: ${{ inputs.priority }} + lifecycles: ${{ inputs.lifecycles }} + filesystem: ${{ inputs.filesystem }} + modes: ${{ inputs.modes }} + marker: " โญ" + + test-deploy-compose-priority: + name: โญ Priority ยท compose + if: >- + ${{ always() + && inputs.priority != '' + && needs.sequencing.outputs.priority == 'serial' + && contains(inputs.modes, 'compose') + && needs.lint.result == 'success' + && needs.test.result == 'success' + && needs.test-dns.result == 'success' + && needs.images-mirror-missing.result == 'success' + && (!inputs.mode_fail_fast + || contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result)) }} + needs: + - test-dns + - images-mirror-missing + - lint + - test + - sequencing + - test-deploy-swarm-priority + uses: ./.github/workflows/test-deploy-compose.yml + with: + distros: ${{ inputs.distros }} + whitelist: ${{ inputs.priority }} + lifecycles: ${{ inputs.lifecycles }} + filesystem: ${{ inputs.filesystem }} + marker: " โญ" + + test-deploy-host-priority: + name: โญ Priority ยท host + if: >- + ${{ always() + && inputs.priority != '' + && needs.sequencing.outputs.priority == 'serial' + && contains(inputs.modes, 'host') + && needs.lint.result == 'success' + && needs.test.result == 'success' + && needs.test-dns.result == 'success' + && needs.images-mirror-missing.result == 'success' + && (!inputs.mode_fail_fast + || contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)) }} + needs: + - test-dns + - images-mirror-missing + - lint + - test + - sequencing + - test-deploy-compose-priority + uses: ./.github/workflows/test-deploy-host.yml + with: + distros: ${{ inputs.distros }} + whitelist: ${{ inputs.priority }} + lifecycles: ${{ inputs.lifecycles }} + filesystem: ${{ inputs.filesystem }} + marker: " โญ" + exclude_compose_covered: ${{ contains(inputs.modes, 'compose') && 'true' || 'false' }} + + test-deploy-swarm: + name: ๐Ÿ” Regular ยท swarm + if: >- + ${{ always() + && contains(inputs.modes, 'swarm') + && needs.lint.result == 'success' + && needs.test.result == 'success' + && needs.build-ci-images.result == 'success' + && needs.images-mirror-missing.result == 'success' + && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result) + && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result) + && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result) + && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) }} + needs: + - build-ci-images + - images-mirror-missing + - test-deploy-single-node-priority + - test-deploy-compose-priority + - test-deploy-host-priority + - test-deploy-swarm-priority + - lint + - test + uses: ./.github/workflows/test-deploy-swarm.yml + with: + distros: ${{ inputs.distros }} + whitelist: ${{ inputs.whitelist }} + lifecycles: ${{ inputs.lifecycles }} + filesystem: ${{ inputs.filesystem }} + blacklist: ${{ inputs.priority }} + test-deploy-single-node: name: ๐Ÿ” Regular ยท single-node if: >- ${{ always() + && needs.sequencing.outputs.regular == 'parallel' && (contains(inputs.modes, 'compose') || contains(inputs.modes, 'host')) && needs.lint.result == 'success' && needs.test.result == 'success' && needs.test-dns.result == 'success' && needs.images-mirror-missing.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result) + && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result) + && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result) && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) }} needs: - test-dns - images-mirror-missing + - sequencing - test-deploy-single-node-priority + - test-deploy-compose-priority + - test-deploy-host-priority - test-deploy-swarm-priority - lint - test @@ -222,25 +389,34 @@ jobs: modes: ${{ inputs.modes }} blacklist: ${{ inputs.priority }} - test-deploy-swarm: - name: ๐Ÿ” Regular ยท swarm + test-deploy-compose: + name: ๐Ÿ” Regular ยท compose if: >- ${{ always() - && contains(inputs.modes, 'swarm') + && needs.sequencing.outputs.regular == 'serial' + && contains(inputs.modes, 'compose') && needs.lint.result == 'success' && needs.test.result == 'success' - && needs.build-ci-images.result == 'success' + && needs.test-dns.result == 'success' && needs.images-mirror-missing.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result) - && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) }} + && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result) + && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result) + && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) + && (!inputs.mode_fail_fast + || contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm.result)) }} needs: - - build-ci-images + - test-dns - images-mirror-missing + - sequencing - test-deploy-single-node-priority + - test-deploy-compose-priority + - test-deploy-host-priority - test-deploy-swarm-priority + - test-deploy-swarm - lint - test - uses: ./.github/workflows/test-deploy-swarm.yml + uses: ./.github/workflows/test-deploy-compose.yml with: distros: ${{ inputs.distros }} whitelist: ${{ inputs.whitelist }} @@ -248,6 +424,42 @@ jobs: filesystem: ${{ inputs.filesystem }} blacklist: ${{ inputs.priority }} + test-deploy-host: + name: ๐Ÿ” Regular ยท host + if: >- + ${{ always() + && needs.sequencing.outputs.regular == 'serial' + && contains(inputs.modes, 'host') + && needs.lint.result == 'success' + && needs.test.result == 'success' + && needs.test-dns.result == 'success' + && needs.images-mirror-missing.result == 'success' + && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result) + && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result) + && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result) + && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) + && (!inputs.mode_fail_fast + || contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose.result)) }} + needs: + - test-dns + - images-mirror-missing + - sequencing + - test-deploy-single-node-priority + - test-deploy-compose-priority + - test-deploy-host-priority + - test-deploy-swarm-priority + - test-deploy-compose + - lint + - test + uses: ./.github/workflows/test-deploy-host.yml + with: + distros: ${{ inputs.distros }} + whitelist: ${{ inputs.whitelist }} + lifecycles: ${{ inputs.lifecycles }} + filesystem: ${{ inputs.filesystem }} + blacklist: ${{ inputs.priority }} + exclude_compose_covered: ${{ contains(inputs.modes, 'compose') && 'true' || 'false' }} + test-runner-smoke: needs: - test-deploy-swarm @@ -283,8 +495,12 @@ jobs: name: ๐Ÿ› Report main role failures needs: - test-deploy-single-node-priority + - test-deploy-compose-priority + - test-deploy-host-priority - test-deploy-swarm-priority - test-deploy-single-node + - test-deploy-compose + - test-deploy-host - test-deploy-swarm if: ${{ always() && github.ref == 'refs/heads/main' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }} runs-on: ubuntu-latest @@ -309,9 +525,14 @@ jobs: - test-dns - test-install-make - test-install-pkgmgr + - sequencing - test-deploy-single-node-priority + - test-deploy-compose-priority + - test-deploy-host-priority - test-deploy-swarm-priority - test-deploy-single-node + - test-deploy-compose + - test-deploy-host - test-deploy-swarm - test-runner-smoke - test-workspace diff --git a/.github/workflows/entry-manual.yml b/.github/workflows/entry-manual.yml index 40ef4925be..535241c187 100644 --- a/.github/workflows/entry-manual.yml +++ b/.github/workflows/entry-manual.yml @@ -38,6 +38,20 @@ on: required: false default: "" type: string + sequencing: + description: "๐Ÿšฆ Deploy-mode order per line. 'serial': swarm, then compose, then host. 'parallel': all at once. 'auto': serial above 65 jobs per line." + required: false + default: "auto" + type: choice + options: + - "auto" + - "serial" + - "parallel" + mode_fail_fast: + description: "๐Ÿ›‘ Stop a serial line at its first failed mode. Off: keep deploying the remaining modes." + required: false + default: true + type: boolean workspace: description: "๐Ÿง‘โ€๐Ÿ’ป Run the workspace test (dev-runtime image matrix). 'auto' runs it on a global scope only." required: false @@ -74,6 +88,8 @@ jobs: lifecycles: ${{ inputs.lifecycles }} modes: ${{ inputs.modes }} priority: ${{ inputs.priority }} + sequencing: ${{ inputs.sequencing }} + mode_fail_fast: ${{ inputs.mode_fail_fast }} filesystem: ${{ inputs.filesystem }} workspace: ${{ inputs.workspace }} instructions: ${{ inputs.instructions }} diff --git a/cli/meta/ci/sequencing.py b/cli/meta/ci/sequencing.py new file mode 100644 index 0000000000..fae50f9a9e --- /dev/null +++ b/cli/meta/ci/sequencing.py @@ -0,0 +1,102 @@ +"""Decide whether a CI line deploys its modes in sequence or in parallel. + +Usage: + python -m cli.meta.ci.sequencing [--modes "swarm compose host"] + [--whitelist "..."] [--blacklist "..."] [--lifecycles "..."] + [--choice auto|serial|parallel] + +A *line* is the priority (โญ) or the regular (๐Ÿ”) half of an orchestrator +run. Its job count is the sum of the deploy-matrix entries every active +mode contributes, counted on the row basis the discover steps use: swarm +selections are ``role#variant`` tokens mapping 1:1 onto jobs, compose and +host selections are whole roles whose variants pack into bundles +(utils.github.variant.bundles, the same packing the matrix runs). + +GitHub cancels a job that sat queued for 24 hours. A line that starts every +mode at once floods the runner pool, so the tail of its matrix waits past +that cut and dies unrun. ``auto`` therefore serialises the modes -- swarm, +then compose, then host, heaviest first -- once the line exceeds +``INFINITO_CI_SEQUENTIAL_THRESHOLD`` jobs, which restarts the queue clock +per mode. Below the threshold the modes run in parallel for wall-clock. + +The count omits the runner-storage filter scripts/meta/resolve/apps.sh +applies inside GitHub Actions, so it can only overestimate, erring towards +the sequential layout that cannot be cancelled. + +Writes ``jobs=`` and ``sequencing=``, ready to append to +``$GITHUB_OUTPUT``. +""" + +from __future__ import annotations + +import argparse +import os +import sys + +from cli.meta.ci.query import MODES, discover, expands_variants +from utils.cache.applications import get_variants +from utils.github.variant.bundles import compose_bundle_counts + +CHOICES = ("auto", "serial", "parallel") + + +def mode_jobs(mode: str, *, whitelist: str, blacklist: str, lifecycles: str) -> int: + """Deploy-matrix jobs *mode* contributes to the line.""" + selection = discover( + mode, whitelist=whitelist, blacklist=blacklist, lifecycles=lifecycles + ) + if expands_variants(mode): + return len(selection) + return sum(compose_bundle_counts(selection, get_variants()).values()) + + +def line_jobs(modes: str, *, whitelist: str, blacklist: str, lifecycles: str) -> int: + return sum( + mode_jobs(mode, whitelist=whitelist, blacklist=blacklist, lifecycles=lifecycles) + for mode in MODES + if mode in modes.split() + ) + + +def threshold() -> int: + return int(os.environ["INFINITO_CI_SEQUENTIAL_THRESHOLD"]) + + +def decide(jobs: int | None, choice: str) -> str: + """``choice`` verbatim unless it is ``auto``, which compares *jobs* + against the threshold.""" + if choice != "auto": + return choice + return "serial" if jobs > threshold() else "parallel" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Decide serial or parallel deploy modes for one CI line." + ) + parser.add_argument("--modes", default=" ".join(MODES)) + parser.add_argument("--whitelist", default="") + parser.add_argument("--blacklist", default="") + parser.add_argument("--lifecycles", default="") + parser.add_argument("--choice", default="auto", choices=CHOICES) + args = parser.parse_args(argv) + + if args.lifecycles.strip(): + os.environ["INFINITO_LIFECYCLES"] = args.lifecycles + + jobs = None + if args.choice == "auto": + jobs = line_jobs( + args.modes, + whitelist=args.whitelist, + blacklist=args.blacklist, + lifecycles=args.lifecycles, + ) + + print(f"jobs={'' if jobs is None else jobs}") + print(f"sequencing={decide(jobs, args.choice)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/cli/meta/ci/slots.py b/cli/meta/ci/slots.py index b7ed0c83a5..170a40ebdb 100644 --- a/cli/meta/ci/slots.py +++ b/cli/meta/ci/slots.py @@ -50,6 +50,10 @@ _DEPLOY_CALLERS = frozenset( { "test-deploy-single-node", "test-deploy-single-node-priority", + "test-deploy-compose", + "test-deploy-compose-priority", + "test-deploy-host", + "test-deploy-host-priority", "test-deploy-swarm", "test-deploy-swarm-priority", } diff --git a/default.env b/default.env index 067cc0454d..e4433a2d13 100644 --- a/default.env +++ b/default.env @@ -88,6 +88,9 @@ INFINITO_CI_DISTRO_BUDGET_SECONDS=20400 # Job budget per workflow run cli.meta.ci.slots divides between the deploy matrices (GitHub matrix cap: 256). INFINITO_CI_JOB_LIMIT=256 +# Deploy jobs a CI line may queue at once before cli.meta.ci.sequencing serialises its modes (GitHub cancels a job queued for 24h). +INFINITO_CI_SEQUENTIAL_THRESHOLD=65 + # Compose service container name; generator overrides as infinito_nexus_${INFINITO_DISTRO}. INFINITO_CONTAINER=infinito_nexus_debian diff --git a/tests/integration/meta/workflows/test_ci_deploy_sequencing.py b/tests/integration/meta/workflows/test_ci_deploy_sequencing.py new file mode 100644 index 0000000000..7b8d83d494 --- /dev/null +++ b/tests/integration/meta/workflows/test_ci_deploy_sequencing.py @@ -0,0 +1,107 @@ +"""The orchestrator's two deploy layouts must stay mutually exclusive. + +Each line renders swarm plus either the parallel single-node job or the +serial compose -> host chain; a `needs:` edge cannot be conditional, so both +layouts exist as jobs and the sequencing decision skips one. Nothing at +runtime proves the ten hand-written `if` expressions still agree with each +other, and the failure is silent in both directions: two layouts deploying +the same roles twice, or neither deploying anything. + +These are structural assertions, not an expression evaluator. They catch the +mistakes this shape actually invites -- a copied sentinel, a missing chain +edge, a fail-fast guard pointing at the wrong predecessor. +""" + +from __future__ import annotations + +import unittest + +from tests.utils import PROJECT_ROOT +from utils.cache.yaml import load_yaml + +ORCHESTRATOR = PROJECT_ROOT / ".github" / "workflows" / "ci-orchestrator.yml" + +LINES = {"priority": "-priority", "regular": ""} +CHAIN = ("test-deploy-swarm", "test-deploy-compose", "test-deploy-host") +PARALLEL = "test-deploy-single-node" + + +def _jobs() -> dict: + return load_yaml(str(ORCHESTRATOR))["jobs"] + + +def _job(name: str, suffix: str) -> dict: + return _jobs()[f"{name}{suffix}"] + + +def _condition(name: str, suffix: str) -> str: + return " ".join(str(_job(name, suffix).get("if", "")).split()) + + +def _needs(name: str, suffix: str) -> list[str]: + declared = _job(name, suffix).get("needs", []) + return [declared] if isinstance(declared, str) else list(declared) + + +class TestLayoutsAreMutuallyExclusive(unittest.TestCase): + def test_the_serial_chain_selects_on_the_serial_sentinel(self) -> None: + for line, suffix in LINES.items(): + for name in CHAIN[1:]: + with self.subTest(line=line, job=name): + self.assertIn( + f"needs.sequencing.outputs.{line} == 'serial'", + _condition(name, suffix), + ) + + def test_the_parallel_job_selects_on_the_parallel_sentinel(self) -> None: + for line, suffix in LINES.items(): + with self.subTest(line=line): + self.assertIn( + f"needs.sequencing.outputs.{line} == 'parallel'", + _condition(PARALLEL, suffix), + ) + + def test_a_skipped_dependency_cannot_skip_a_serial_job(self) -> None: + for line, suffix in LINES.items(): + for name in CHAIN[1:]: + with self.subTest(line=line, job=name): + self.assertIn("always()", _condition(name, suffix)) + + +class TestSerialChain(unittest.TestCase): + def test_each_mode_waits_for_its_predecessor(self) -> None: + for line, suffix in LINES.items(): + for predecessor, successor in zip(CHAIN, CHAIN[1:], strict=False): + with self.subTest(line=line, job=successor): + self.assertIn( + f"{predecessor}{suffix}", _needs(successor, suffix) + ) + + def test_swarm_leads_and_waits_for_no_decision(self) -> None: + """It runs first in either layout, so making it wait for the + sequencing job would delay every run for nothing.""" + for line, suffix in LINES.items(): + with self.subTest(line=line): + self.assertNotIn("sequencing", _needs(CHAIN[0], suffix)) + + def test_fail_fast_guards_the_immediate_predecessor(self) -> None: + for line, suffix in LINES.items(): + for predecessor, successor in zip(CHAIN, CHAIN[1:], strict=False): + with self.subTest(line=line, job=successor): + condition = _condition(successor, suffix) + self.assertIn("inputs.mode_fail_fast", condition) + self.assertIn( + f"needs.{predecessor}{suffix}.result", condition + ) + + +class TestRegularLineWaitsForPriority(unittest.TestCase): + def test_every_priority_job_gates_every_regular_job(self) -> None: + priority = [f"{name}-priority" for name in (PARALLEL, *CHAIN)] + for name in (PARALLEL, *CHAIN): + with self.subTest(job=name): + self.assertLessEqual(set(priority), set(_needs(name, ""))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/cli/meta/ci/test_sequencing.py b/tests/unit/cli/meta/ci/test_sequencing.py new file mode 100644 index 0000000000..ff6be0710a --- /dev/null +++ b/tests/unit/cli/meta/ci/test_sequencing.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import unittest +from unittest import mock + +from cli.meta.ci import sequencing + +_SELECTIONS = { + "swarm": ["a#0", "a#1", "b#0"], + "compose": ["a", "b"], + "host": ["c"], +} + + +def _patched(modes: str) -> int: + with ( + mock.patch.object(sequencing, "discover", lambda mode, **_: _SELECTIONS[mode]), + mock.patch.object(sequencing, "get_variants", dict), + mock.patch.object( + sequencing, + "compose_bundle_counts", + lambda apps, _variants: dict.fromkeys(apps, 2), + ), + ): + return sequencing.line_jobs(modes, whitelist="", blacklist="", lifecycles="") + + +class TestLineJobs(unittest.TestCase): + def test_swarm_tokens_map_one_to_one_and_roles_bundle(self) -> None: + self.assertEqual(_patched("swarm compose host"), 3 + 4 + 2) + + def test_inactive_modes_do_not_count(self) -> None: + self.assertEqual(_patched("swarm"), 3) + + +class TestDecide(unittest.TestCase): + def test_auto_serialises_above_the_threshold(self) -> None: + with mock.patch.dict("os.environ", {"INFINITO_CI_SEQUENTIAL_THRESHOLD": "65"}): + self.assertEqual(sequencing.decide(66, "auto"), "serial") + self.assertEqual(sequencing.decide(65, "auto"), "parallel") + + def test_a_forced_choice_skips_the_threshold(self) -> None: + self.assertEqual(sequencing.decide(None, "serial"), "serial") + self.assertEqual(sequencing.decide(None, "parallel"), "parallel") + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/env/handlers/passthrough.py b/utils/env/handlers/passthrough.py index fc36343caf..549111f6e4 100644 --- a/utils/env/handlers/passthrough.py +++ b/utils/env/handlers/passthrough.py @@ -84,6 +84,7 @@ STATIC_KEYS = ( "INFINITO_BLACKLIST", "INFINITO_CI_DISTRO_BUDGET_SECONDS", "INFINITO_CI_JOB_LIMIT", + "INFINITO_CI_SEQUENTIAL_THRESHOLD", "INFINITO_LIMIT_HOST", "INFINITO_RUN_FLAGS", "INFINITO_WORKER_ENABLED",