feat(ci): serialise a big deploy line's modes

GitHub cancels a job queued for 24h. A line starting swarm, compose and
host at once queues its whole matrix in one go, so with more roles than
runner slots the tail waits past that cut and dies unrun -- 54 of 241
jobs in run 30937913632.

Deploy the modes one after the other instead, heaviest first, which
restarts the queue clock per mode. cli.meta.ci.sequencing counts a
line's matrix jobs on the row basis the discover steps use and
serialises above INFINITO_CI_SEQUENTIAL_THRESHOLD; below it the modes
still start together, so a small diff pays no wall-clock for this.

A needs: edge is static, so both layouts exist as jobs and the decision
skips one. Their conditions are structurally asserted, not evaluated:
ten near-identical if expressions invite a copied sentinel or a missing
chain edge, and both failures are silent at runtime.

mode_fail_fast stops a serial line at its first failed mode, on by
default.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

.github/workflows/README.md

@@ -21,50 +21,52 @@ flowchart TB
     eman --> orch
 
     subgraph orchestrator["ci-orchestrator.yml jobs"]
-        plan["plan: run-summary table"]
         waitfork["wait-fork-prereq-run"] --> forkready["fork-prereqs-ready"]
 
-        lintwf["lint.yml: make lint + hadolint"] --> qualitygate["code-quality-gate"]
-        testwf["test.yml: make test"] --> qualitygate
-        codeql["security-codeql.yml"] --> qualitygate
-
-        qualitygate --> buildci["build-ci-images: images-build-ci.yml"]
-        buildci --> dns["test-dns.yml"]
+        lintwf["lint.yml: make lint + hadolint"]
+        testwf["test.yml: make test"]
+        codeql["security-codeql.yml"]
+        buildci["build-ci-images: images-build-ci.yml"] --> dns["test-dns.yml"]
         mirror["images-mirror-missing.yml"]
+        seq["sequencing: serial or parallel, per line"]
 
-        dns --> snprio["test-deploy-single-node-priority"]
-        mirror --> snprio
-        buildci --> swarmprio["test-deploy-swarm-priority"]
-        mirror --> swarmprio
+        subgraph prio["priority line, skipped without a priority input"]
+            swarmprio["test-deploy-swarm-priority"]
+            swarmprio -->|"serial"| composeprio["test-deploy-compose-priority"]
+            composeprio -->|"serial"| hostprio["test-deploy-host-priority"]
+            snprio["test-deploy-single-node-priority: parallel"]
+        end
 
-        snprio -->|"all priority jobs green"| snreg["test-deploy-single-node"]
-        swarmprio -->|"all priority jobs green"| snreg
-        snprio -->|"all priority jobs green"| swarmreg["test-deploy-swarm"]
-        swarmprio -->|"all priority jobs green"| swarmreg
-        dns --> snreg
-        mirror --> snreg
-        buildci --> swarmreg
-        mirror --> swarmreg
+        subgraph reg["regular line"]
+            swarmreg["test-deploy-swarm"]
+            swarmreg -->|"serial"| composereg["test-deploy-compose"]
+            composereg -->|"serial"| hostreg["test-deploy-host"]
+            snreg["test-deploy-single-node: parallel"]
+        end
+
+        lintwf --> prio
+        testwf --> prio
+        dns --> prio
+        mirror --> prio
+        seq --> prio
+        prio -->|"all priority jobs green"| reg
 
         swarmreg --> smoke["test-runner-smoke.yml"]
+        prio --> report["report-main-failures"]
+        reg --> report
 
-        qualitygate --> instmake["test-install-make.yml"]
-        qualitygate --> instpkgmgr["test-install-pkgmgr.yml"]
-        instmake --> instgate["test-install-gate"]
-        instpkgmgr --> instgate
-
-        qualitygate --> devenv["test-workspace: test-workspace.yml"]
-        mirror --> devenv
-
+        instmake["test-install-make.yml"]
+        instpkgmgr["test-install-pkgmgr.yml"]
+        mirror --> devenv["test-workspace: test-workspace.yml"]
         buildci --> testguide["test-instructions.yml"]
         mirror --> testguide
 
-        snprio --> donegate["done"]
-        swarmprio --> donegate
-        snreg --> donegate
-        swarmreg --> donegate
+        prio --> donegate["done"]
+        reg --> donegate
+        seq --> donegate
         smoke --> donegate
-        instgate --> donegate
+        instmake --> donegate
+        instpkgmgr --> donegate
         devenv --> donegate
         testguide --> donegate
     end
@@ -73,14 +75,107 @@ flowchart TB
     snreg --> singlenode
     singlenode --> deploycompose["test-deploy-compose.yml"]
     singlenode --> deployhost["test-deploy-host.yml"]
+    composeprio --> deploycompose
+    composereg --> deploycompose
+    hostprio --> deployhost
+    hostreg --> deployhost
     swarmprio --> deployswarm["test-deploy-swarm.yml"]
     swarmreg --> deployswarm
 ```
 
-`test-deploy-single-node-priority` and `test-deploy-swarm-priority` run only
-when the orchestrator's `priority` input is set; with it empty they are
-skipped and the regular deploy jobs start directly. The regular jobs receive
-the priority ids as `blacklist`, so each role deploys in exactly one line.
+The priority line runs only when the orchestrator's `priority` input is set;
+with it empty every priority job is skipped and the regular line starts
+directly. The regular jobs receive the priority ids as `blacklist`, so each
+role deploys in exactly one line.
+
+## Deploy-line sequencing
+
+A **line** is one half of a run: the priority line (`⭐`, the roles named in
+`priority`) or the regular line (`🔁`, everything else). Each line deploys up
+to three **modes**: swarm, compose, host. The lines are always sequential
+relative to each other; this section is about the order *inside* one line.
+
+### Why the order exists
+
+GitHub cancels a job that has sat queued for 24 hours. That clock starts when
+the job is queued, not when the run starts, and a `needs:` edge delays queueing
+until the dependency completes. A line that starts all three modes at once
+queues its whole matrix in one go, so with more roles than runner slots the
+tail of that matrix waits past the cut and dies unrun. Deploying the modes one
+after the other restarts the clock per mode.
+
+Serialising costs wall-clock, so it is only worth it above a certain size. The
+run does not pay it on a small diff.
+
+### The decision
+
+The `sequencing` job resolves the effective whitelist (the same
+`scripts/github/resolve/effective_whitelist.sh` the discover steps use) and
+calls `cli.meta.ci.sequencing` once per line, emitting `serial` or `parallel`
+as a job output.
+
+Job counts are taken on the row basis each mode actually runs on: swarm
+selections are `role#variant` tokens that map 1:1 onto jobs, compose and host
+selections are whole roles whose variants pack into bundles
+(`utils.github.variant.bundles`). The count deliberately omits the
+runner-storage filter that `scripts/meta/resolve/apps.sh` applies inside
+GitHub Actions, so it can only overestimate — erring towards the sequential
+layout, which cannot be cancelled.
+
+| `sequencing` input | Behaviour |
+|---|---|
+| `auto` (default) | `serial` above `INFINITO_CI_SEQUENTIAL_THRESHOLD` jobs in that line, `parallel` at or below it |
+| `serial` | forced; skips the count entirely |
+| `parallel` | forced; skips the count entirely |
+
+The threshold lives in [`default.env`](../../default.env) as
+`INFINITO_CI_SEQUENTIAL_THRESHOLD`. Both lines are decided independently: a
+small priority line can run parallel while the regular line behind it runs
+serial.
+
+### The two layouts
+
+Serial order is **swarm, then compose, then host** — heaviest mode first, so
+the longest queue drains while the run is youngest.
+
+- **parallel** — `test-deploy-swarm-priority` plus
+  `test-deploy-single-node-priority`, which fans compose and host out
+  together. This is the layout the pipeline had before sequencing existed.
+- **serial** — `test-deploy-swarm-priority` →
+  `test-deploy-compose-priority` → `test-deploy-host-priority`, each chained
+  on the previous through `needs:`.
+
+The regular line mirrors this with the unsuffixed job names.
+
+A `needs:` edge is static, so a workflow cannot make one conditional. Both
+layouts therefore exist as separate jobs and the `sequencing` output skips
+one of them. Because a skipped dependency otherwise skips its dependents, the
+serial jobs carry `always()` plus explicit `needs.<job>.result` checks on the
+real gates. The swarm job depends on no sequencing decision at all: it leads
+in either layout.
+
+### Stopping on failure
+
+`mode_fail_fast` (default `true`) decides whether a serial line stops at its
+first failed mode:
+
+| Value | Behaviour |
+|---|---|
+| `true` | swarm fails → compose and host of that line are skipped |
+| `false` | every mode deploys and reports; the run still ends red |
+
+`skipped` counts as passed, so a mode absent from `modes` never blocks the
+chain. The parallel layout ignores the switch — with no order there is nothing
+to stop. The setting is a checkbox on `entry-manual.yml`; the other entry
+points take the default.
+
+### Job budget
+
+`cli.meta.ci.slots` divides the run's 256-job cap between the deploy
+matrices and reads the orchestrator to do it, so every deploy caller —
+including the serial twins — must be listed in its `_DEPLOY_CALLERS`.
+A caller missing there is charged a guessed dynamic-matrix estimate instead
+of its single discover job.
 
 ## Cancellation
 

.github/workflows/ci-orchestrator.yml

@@ -26,6 +26,16 @@ on:
         required: false
         type: string
         default: ""
+      sequencing:
+        description: "🚦 Deploy-mode order per line. 'serial': swarm, then compose, then host. 'parallel': all at once. 'auto': serial above the job threshold."
+        required: false
+        type: string
+        default: "auto"
+      mode_fail_fast:
+        description: "🛑 Stop a serial line at its first failed mode. Off: keep deploying the remaining modes."
+        required: false
+        type: boolean
+        default: true
       filesystem:
         description: "🗄️ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three."
         required: false
@@ -162,22 +172,65 @@ jobs:
       image_wait_sleep_seconds: ${{ inputs.image_wait_sleep_seconds }}
     secrets: inherit
 
-  test-deploy-single-node-priority:
-    name: ⭐ Priority · single-node
-    if: ${{ inputs.priority != '' && (contains(inputs.modes, 'compose') || contains(inputs.modes, 'host')) }}
-    needs:
-      - test-dns
-      - images-mirror-missing
-      - lint
-      - test
-    uses: ./.github/workflows/test-deploy-single-node.yml
-    with:
-      distros: ${{ inputs.distros }}
-      whitelist: ${{ inputs.priority }}
-      lifecycles: ${{ inputs.lifecycles }}
-      filesystem: ${{ inputs.filesystem }}
-      modes: ${{ inputs.modes }}
-      marker: " ⭐"
+  sequencing:
+    name: 🚦 Sequencing
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    outputs:
+      priority: ${{ steps.priority.outputs.sequencing }}
+      regular: ${{ steps.regular.outputs.sequencing }}
+    env:
+      INFINITO_APP_DISCOVERY_RUNNER: host
+      INFINITO_LIFECYCLES: ${{ inputs.lifecycles }}
+      INFINITO_HOST_EXCLUDE_COMPOSE: ${{ contains(inputs.modes, 'compose') && 'true' || 'false' }}
+    steps:
+      - uses: actions/checkout@v7.0.0
+        with:
+          fetch-depth: 0
+
+      - name: 🔧 Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: 📦 Install project (host venv)
+        shell: bash
+        run: make install
+
+      - name: 🔍 Resolve effective whitelist (diff vs origin/main)
+        id: resolve_wl
+        shell: bash
+        env:
+          INPUT_WHITELIST: ${{ inputs.whitelist }}
+        run: ./scripts/github/resolve/effective_whitelist.sh
+
+      - name: ⭐ Decide the priority line
+        id: priority
+        if: ${{ inputs.priority != '' }}
+        shell: bash
+        run: |
+          source scripts/meta/env/load.sh
+          "${PYTHON}" -m cli.meta.ci.sequencing \
+            --modes "${{ inputs.modes }}" \
+            --whitelist "${{ inputs.priority }}" \
+            --lifecycles "${{ inputs.lifecycles }}" \
+            --choice "${{ inputs.sequencing }}" \
+            | tee -a "$GITHUB_OUTPUT"
+
+      - name: 🔁 Decide the regular line
+        id: regular
+        shell: bash
+        run: |
+          source scripts/meta/env/load.sh
+          "${PYTHON}" -m cli.meta.ci.sequencing \
+            --modes "${{ inputs.modes }}" \
+            --whitelist "${{ steps.resolve_wl.outputs.whitelist }}" \
+            --blacklist "${{ inputs.priority }}" \
+            --lifecycles "${{ inputs.lifecycles }}" \
+            --choice "${{ inputs.sequencing }}" \
+            | tee -a "$GITHUB_OUTPUT"
 
   test-deploy-swarm-priority:
     name: ⭐ Priority · swarm
@@ -195,21 +248,135 @@ jobs:
       filesystem: ${{ inputs.filesystem }}
       marker: " ⭐"
 
+  test-deploy-single-node-priority:
+    name: ⭐ Priority · single-node
+    if: >-
+      ${{ inputs.priority != ''
+          && needs.sequencing.outputs.priority == 'parallel'
+          && (contains(inputs.modes, 'compose') || contains(inputs.modes, 'host')) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - lint
+      - test
+      - sequencing
+    uses: ./.github/workflows/test-deploy-single-node.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.priority }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      modes: ${{ inputs.modes }}
+      marker: " ⭐"
+
+  test-deploy-compose-priority:
+    name: ⭐ Priority · compose
+    if: >-
+      ${{ always()
+          && inputs.priority != ''
+          && needs.sequencing.outputs.priority == 'serial'
+          && contains(inputs.modes, 'compose')
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.test-dns.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && (!inputs.mode_fail_fast
+              || contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result)) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - lint
+      - test
+      - sequencing
+      - test-deploy-swarm-priority
+    uses: ./.github/workflows/test-deploy-compose.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.priority }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      marker: " ⭐"
+
+  test-deploy-host-priority:
+    name: ⭐ Priority · host
+    if: >-
+      ${{ always()
+          && inputs.priority != ''
+          && needs.sequencing.outputs.priority == 'serial'
+          && contains(inputs.modes, 'host')
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.test-dns.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && (!inputs.mode_fail_fast
+              || contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - lint
+      - test
+      - sequencing
+      - test-deploy-compose-priority
+    uses: ./.github/workflows/test-deploy-host.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.priority }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      marker: " ⭐"
+      exclude_compose_covered: ${{ contains(inputs.modes, 'compose') && 'true' || 'false' }}
+
+  test-deploy-swarm:
+    name: 🔁 Regular · swarm
+    if: >-
+      ${{ always()
+          && contains(inputs.modes, 'swarm')
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.build-ci-images.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) }}
+    needs:
+      - build-ci-images
+      - images-mirror-missing
+      - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
+      - test-deploy-swarm-priority
+      - lint
+      - test
+    uses: ./.github/workflows/test-deploy-swarm.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.whitelist }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      blacklist: ${{ inputs.priority }}
+
   test-deploy-single-node:
     name: 🔁 Regular · single-node
     if: >-
       ${{ always()
+          && needs.sequencing.outputs.regular == 'parallel'
           && (contains(inputs.modes, 'compose') || contains(inputs.modes, 'host'))
           && needs.lint.result == 'success'
           && needs.test.result == 'success'
           && needs.test-dns.result == 'success'
           && needs.images-mirror-missing.result == 'success'
           && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result)
           && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) }}
     needs:
       - test-dns
       - images-mirror-missing
+      - sequencing
       - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
       - test-deploy-swarm-priority
       - lint
       - test
@@ -222,25 +389,34 @@ jobs:
       modes: ${{ inputs.modes }}
       blacklist: ${{ inputs.priority }}
 
-  test-deploy-swarm:
-    name: 🔁 Regular · swarm
+  test-deploy-compose:
+    name: 🔁 Regular · compose
     if: >-
       ${{ always()
-          && contains(inputs.modes, 'swarm')
+          && needs.sequencing.outputs.regular == 'serial'
+          && contains(inputs.modes, 'compose')
           && needs.lint.result == 'success'
           && needs.test.result == 'success'
-          && needs.build-ci-images.result == 'success'
+          && needs.test-dns.result == 'success'
           && needs.images-mirror-missing.result == 'success'
           && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
-          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) }}
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result)
+          && (!inputs.mode_fail_fast
+              || contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm.result)) }}
     needs:
-      - build-ci-images
+      - test-dns
       - images-mirror-missing
+      - sequencing
       - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
       - test-deploy-swarm-priority
+      - test-deploy-swarm
       - lint
       - test
-    uses: ./.github/workflows/test-deploy-swarm.yml
+    uses: ./.github/workflows/test-deploy-compose.yml
     with:
       distros: ${{ inputs.distros }}
       whitelist: ${{ inputs.whitelist }}
@@ -248,6 +424,42 @@ jobs:
       filesystem: ${{ inputs.filesystem }}
       blacklist: ${{ inputs.priority }}
 
+  test-deploy-host:
+    name: 🔁 Regular · host
+    if: >-
+      ${{ always()
+          && needs.sequencing.outputs.regular == 'serial'
+          && contains(inputs.modes, 'host')
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.test-dns.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result)
+          && (!inputs.mode_fail_fast
+              || contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose.result)) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - sequencing
+      - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
+      - test-deploy-swarm-priority
+      - test-deploy-compose
+      - lint
+      - test
+    uses: ./.github/workflows/test-deploy-host.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.whitelist }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      blacklist: ${{ inputs.priority }}
+      exclude_compose_covered: ${{ contains(inputs.modes, 'compose') && 'true' || 'false' }}
+
   test-runner-smoke:
     needs:
       - test-deploy-swarm
@@ -283,8 +495,12 @@ jobs:
     name: 🐛 Report main role failures
     needs:
       - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
       - test-deploy-swarm-priority
       - test-deploy-single-node
+      - test-deploy-compose
+      - test-deploy-host
       - test-deploy-swarm
     if: ${{ always() && github.ref == 'refs/heads/main' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}
     runs-on: ubuntu-latest
@@ -309,9 +525,14 @@ jobs:
       - test-dns
       - test-install-make
       - test-install-pkgmgr
+      - sequencing
       - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
       - test-deploy-swarm-priority
       - test-deploy-single-node
+      - test-deploy-compose
+      - test-deploy-host
       - test-deploy-swarm
       - test-runner-smoke
       - test-workspace

.github/workflows/entry-manual.yml

@@ -38,6 +38,20 @@ on:
         required: false
         default: ""
         type: string
+      sequencing:
+        description: "🚦 Deploy-mode order per line. 'serial': swarm, then compose, then host. 'parallel': all at once. 'auto': serial above 65 jobs per line."
+        required: false
+        default: "auto"
+        type: choice
+        options:
+          - "auto"
+          - "serial"
+          - "parallel"
+      mode_fail_fast:
+        description: "🛑 Stop a serial line at its first failed mode. Off: keep deploying the remaining modes."
+        required: false
+        default: true
+        type: boolean
       workspace:
         description: "🧑‍💻 Run the workspace test (dev-runtime image matrix). 'auto' runs it on a global scope only."
         required: false
@@ -74,6 +88,8 @@ jobs:
       lifecycles: ${{ inputs.lifecycles }}
       modes: ${{ inputs.modes }}
       priority: ${{ inputs.priority }}
+      sequencing: ${{ inputs.sequencing }}
+      mode_fail_fast: ${{ inputs.mode_fail_fast }}
       filesystem: ${{ inputs.filesystem }}
       workspace: ${{ inputs.workspace }}
       instructions: ${{ inputs.instructions }}