Merge upstream/main into feature/role-stalwart-mail-swarm

2026-11-11 21:26:44 +00:00 · 2026-08-18 01:42:23 +04:00
commit 7332bd66ae
@@ -10,8 +10,8 @@ inputs and descriptions: [workflows.md](../../docs/contributing/tools/github/act
 ```mermaid
 flowchart TB
    push["push: main, feature/**, hotfix/**, fix/**"] --> epl["entry-push-latest.yml"]
-    pr["pull_request: opened, synchronize, reopened, ready_for_review"] --> eprc["entry-pull-request-change.yml"]
-    dispatch["workflow_dispatch"] --> eman["entry-manual.yml"]
+    pr["pull_request: opened, synchronize, reopened, ready_for_review"] --> eprc["entry-pr-change-orchestrate.yml"]
+    dispatch["workflow_dispatch"] --> eman["entry-manual-steer.yml"]

    epl --> orch["ci-orchestrator.yml"]
    epl -->|"version tag on main"| relv["release-version.yml"]
@@ -21,50 +21,52 @@ flowchart TB
    eman --> orch

    subgraph orchestrator["ci-orchestrator.yml jobs"]
-        plan["plan: run-summary table"]
        waitfork["wait-fork-prereq-run"] --> forkready["fork-prereqs-ready"]

-        lintwf["lint.yml: make lint + hadolint"] --> qualitygate["code-quality-gate"]
-        testwf["test.yml: make test"] --> qualitygate
-        codeql["security-codeql.yml"] --> qualitygate
-
-        qualitygate --> buildci["build-ci-images: images-build-ci.yml"]
-        buildci --> dns["test-dns.yml"]
+        lintwf["lint.yml: make lint + hadolint"]
+        testwf["test.yml: make test"]
+        codeql["cron-security-codeql.yml"]
+        buildci["build-ci-images: images-build-ci.yml"] --> dns["test-dns.yml"]
        mirror["images-mirror-missing.yml"]
+        seq["sequencing: serial or parallel, per line"]

-        dns --> snprio["test-deploy-single-node-priority"]
-        mirror --> snprio
-        buildci --> swarmprio["test-deploy-swarm-priority"]
-        mirror --> swarmprio
+        subgraph prio["priority line, skipped without a priority input"]
+            swarmprio["test-deploy-swarm-priority"]
+            swarmprio -->|"serial"| composeprio["test-deploy-compose-priority"]
+            composeprio -->|"serial"| hostprio["test-deploy-host-priority"]
+            snprio["test-deploy-single-node-priority: parallel"]
+        end

-        snprio -->|"all priority jobs green"| snreg["test-deploy-single-node"]
-        swarmprio -->|"all priority jobs green"| snreg
-        snprio -->|"all priority jobs green"| swarmreg["test-deploy-swarm"]
-        swarmprio -->|"all priority jobs green"| swarmreg
-        dns --> snreg
-        mirror --> snreg
-        buildci --> swarmreg
-        mirror --> swarmreg
+        subgraph reg["regular line"]
+            swarmreg["test-deploy-swarm"]
+            swarmreg -->|"serial"| composereg["test-deploy-compose"]
+            composereg -->|"serial"| hostreg["test-deploy-host"]
+            snreg["test-deploy-single-node: parallel"]
+        end
+
+        lintwf --> prio
+        testwf --> prio
+        dns --> prio
+        mirror --> prio
+        seq --> prio
+        prio -->|"all priority jobs green"| reg

        swarmreg --> smoke["test-runner-smoke.yml"]
+        prio --> report["report-main-failures"]
+        reg --> report

-        qualitygate --> instmake["test-install-make.yml"]
-        qualitygate --> instpkgmgr["test-install-pkgmgr.yml"]
-        instmake --> instgate["test-install-gate"]
-        instpkgmgr --> instgate
-
-        qualitygate --> devenv["test-development: test-environment.yml"]
-        mirror --> devenv
-
-        buildci --> testguide["test-guide.yml"]
+        instmake["test-install-make.yml"]
+        instpkgmgr["test-install-pkgmgr.yml"]
+        mirror --> devenv["test-workspace: test-workspace.yml"]
+        buildci --> testguide["test-instructions.yml"]
        mirror --> testguide

-        snprio --> donegate["done"]
-        swarmprio --> donegate
-        snreg --> donegate
-        swarmreg --> donegate
+        prio --> donegate["done"]
+        reg --> donegate
+        seq --> donegate
        smoke --> donegate
-        instgate --> donegate
+        instmake --> donegate
+        instpkgmgr --> donegate
        devenv --> donegate
        testguide --> donegate
    end
@@ -73,21 +75,114 @@ flowchart TB
    snreg --> singlenode
    singlenode --> deploycompose["test-deploy-compose.yml"]
    singlenode --> deployhost["test-deploy-host.yml"]
+    composeprio --> deploycompose
+    composereg --> deploycompose
+    hostprio --> deployhost
+    hostreg --> deployhost
    swarmprio --> deployswarm["test-deploy-swarm.yml"]
    swarmreg --> deployswarm
 ```

-`test-deploy-single-node-priority` and `test-deploy-swarm-priority` run only
-when the orchestrator's `priority` input is set; with it empty they are
-skipped and the regular deploy jobs start directly. The regular jobs receive
-the priority ids as `blacklist`, so each role deploys in exactly one line.
+The priority line runs only when the orchestrator's `priority` input is set;
+with it empty every priority job is skipped and the regular line starts
+directly. The regular jobs receive the priority ids as `blacklist`, so each
+role deploys in exactly one line.
+
+## Deploy-line sequencing
+
+A **line** is one half of a run: the priority line (`⭐`, the roles named in
+`priority`) or the regular line (`🔁`, everything else). Each line deploys up
+to three **modes**: swarm, compose, host. The lines are always sequential
+relative to each other; this section is about the order *inside* one line.
+
+### Why the order exists
+
+GitHub cancels a job that has sat queued for 24 hours. That clock starts when
+the job is queued, not when the run starts, and a `needs:` edge delays queueing
+until the dependency completes. A line that starts all three modes at once
+queues its whole matrix in one go, so with more roles than runner slots the
+tail of that matrix waits past the cut and dies unrun. Deploying the modes one
+after the other restarts the clock per mode.
+
+Serialising costs wall-clock, so it is only worth it above a certain size. The
+run does not pay it on a small diff.
+
+### The decision
+
+The `sequencing` job resolves the effective whitelist (the same
+`scripts/github/resolve/effective_whitelist.sh` the discover steps use) and
+calls `cli.meta.ci.sequencing` once per line, emitting `serial` or `parallel`
+as a job output.
+
+Job counts are taken on the row basis each mode actually runs on: swarm
+selections are `role#variant` tokens that map 1:1 onto jobs, compose and host
+selections are whole roles whose variants pack into bundles
+(`utils.github.variant.bundles`). The count deliberately omits the
+runner-storage filter that `scripts/meta/resolve/apps.sh` applies inside
+GitHub Actions, so it can only overestimate — erring towards the sequential
+layout, which cannot be cancelled.
+
+| `sequencing` input | Behaviour |
+|---|---|
+| `auto` (default) | `serial` above `INFINITO_CI_SEQUENTIAL_THRESHOLD` jobs in that line, `parallel` at or below it |
+| `serial` | forced; skips the count entirely |
+| `parallel` | forced; skips the count entirely |
+
+The threshold lives in [`default.env`](../../default.env) as
+`INFINITO_CI_SEQUENTIAL_THRESHOLD`. Both lines are decided independently: a
+small priority line can run parallel while the regular line behind it runs
+serial.
+
+### The two layouts
+
+Serial order is **swarm, then compose, then host** — heaviest mode first, so
+the longest queue drains while the run is youngest.
+
+- **parallel** — `test-deploy-swarm-priority` plus
+  `test-deploy-single-node-priority`, which fans compose and host out
+  together. This is the layout the pipeline had before sequencing existed.
+- **serial** — `test-deploy-swarm-priority` →
+  `test-deploy-compose-priority` → `test-deploy-host-priority`, each chained
+  on the previous through `needs:`.
+
+The regular line mirrors this with the unsuffixed job names.
+
+A `needs:` edge is static, so a workflow cannot make one conditional. Both
+layouts therefore exist as separate jobs and the `sequencing` output skips
+one of them. Because a skipped dependency otherwise skips its dependents, the
+serial jobs carry `always()` plus explicit `needs.<job>.result` checks on the
+real gates. The swarm job depends on no sequencing decision at all: it leads
+in either layout.
+
+### Stopping on failure
+
+`mode_fail_fast` (default `true`) decides whether a serial line stops at its
+first failed mode:
+
+| Value | Behaviour |
+|---|---|
+| `true` | swarm fails → compose and host of that line are skipped |
+| `false` | every mode deploys and reports; the run still ends red |
+
+`skipped` counts as passed, so a mode absent from `modes` never blocks the
+chain. The parallel layout ignores the switch — with no order there is nothing
+to stop. The setting is a checkbox on `entry-manual-steer.yml`; the other entry
+points take the default.
+
+### Job budget
+
+`cli.meta.ci.slots` divides the run's 256-job cap between the deploy
+matrices and reads the orchestrator to do it, so every deploy caller —
+including the serial twins — must be listed in its `_DEPLOY_CALLERS`.
+A caller missing there is charged a guessed dynamic-matrix estimate instead
+of its single discover job.

 ## Cancellation

 ```mermaid
 flowchart TB
-    prclose["pull_request_target: closed, converted_to_draft"] --> eprcancel["entry-pull-request-cancel.yml"]
-    branchdelete["delete: branch"] --> delbranch["delete-branch.yml"]
+    prclose["pull_request_target: closed, converted_to_draft"] --> eprcancel["entry-pr-closed-cancel-workflows.yml"]
+    branchdelete["delete: branch"] --> delbranch["entry-delete-branch.yml"]
    eprcancel -.->|"cancels concurrency group"| runningci["running entry + child workflow runs"]
    delbranch -.->|"cancels concurrency group"| runningci
 ```
@@ -96,23 +191,23 @@ flowchart TB

 ```mermaid
 flowchart TB
-    daily["schedule: daily 00:00 UTC"] --> mirrorall["images-mirror-all.yml"]
-    daily --> stale["cleanup-stale.yml"]
-    daily --> relhighest["release-highest.yml"]
-    weekly["schedule: weekly Sat 00:00 UTC"] --> updatewf["update.yml"]
-    weekly --> cleanupci["images-cleanup-ci.yml"]
-    weeklymon["schedule: weekly Mon 00:00 UTC"] --> scorecard["security-scorecard.yml"]
+    daily["schedule: daily 00:00 UTC"] --> mirrorall["cron-images-mirror-all.yml"]
+    daily --> stale["cron-cleanup-stale.yml"]
+    daily --> relhighest["cron-release-highest.yml"]
+    weekly["schedule: weekly Sat 00:00 UTC"] --> updatewf["cron-update.yml"]
+    weekly --> cleanupci["cron-images-cleanup-ci.yml"]
+    weeklymon["schedule: weekly Mon 00:00 UTC"] --> scorecard["cron-security-scorecard.yml"]
    branchprot["branch_protection_rule"] --> scorecard
    pushmain["push: main"] --> updatewf
-    prtarget["pull_request_target: opened, reopened"] --> depclose["dependabot-close.yml"]
+    prtarget["pull_request_target: opened, reopened"] --> depclose["entry-pr-open-dependabot-close.yml"]

    relhighest -.->|"gh workflow run"| relver["release-version.yml"]
    relver --> imgbuildci["images-build-ci.yml"]
    manual["workflow_dispatch"] --> mirrorcleanup["images-mirror-cleanup.yml"]
-    manual --> deploywf["test-guide.yml: run a role README Production command"]
+    manual --> deploywf["test-instructions.yml: run a role README Production command"]
 ```

-Also manually dispatchable: `images-mirror-all.yml`, `images-cleanup-ci.yml`,
-`cleanup-stale.yml`, `update.yml`, `release-highest.yml`, `release-version.yml`,
-`lint.yml`, `test.yml`, `test-deploy-swarm.yml`, `test-dns.yml`,
-`test-environment.yml`, `test-runner-smoke.yml`.
+Also manually dispatchable: `cron-images-mirror-all.yml`, `cron-images-cleanup-ci.yml`,
+`cron-cleanup-stale.yml`, `cron-update.yml`, `cron-release-highest.yml`, `call-release-version.yml`,
+`call-lint.yml`, `call-test.yml`, `call-test-deploy-swarm.yml`, `call-test-dns.yml`,
+`test-workspace.yml`, `test-runner-smoke.yml`.
@@ -43,7 +43,7 @@ concurrency:

 jobs:
  prepare:
-    name: 🏷️ Label images
+    name: 📋 Build Matrix
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.out.outputs.matrix }}
@@ -88,17 +88,25 @@ jobs:
        shell: bash
        run: bash scripts/github/runner/disk_pressure_report.sh START

+      - name: 🔍 Resolve buildkit image ref
+        id: buildkit
+        shell: bash
+        run: |
+          set -euo pipefail
+          ref="$(bash scripts/meta/resolve/image/service_ref.sh svc-runner buildkit mirror)"
+          echo "ref=${ref}" >> "$GITHUB_OUTPUT"
+
      - name: ⬇️ Pre-pull buildkit image (retry 7x, 20s)
        shell: bash
        env:
          MAX_ATTEMPTS: "7"
          RETRY_DELAY_SECONDS: "20"
-        run: bash scripts/github/runner/pull_with_retry.sh moby/buildkit:buildx-stable-1
+        run: bash scripts/github/runner/pull_with_retry.sh "${{ steps.buildkit.outputs.ref }}"

      - name: 🔧 Set up Docker Buildx
        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
        with:
-          driver-opts: image=moby/buildkit:buildx-stable-1
+          driver-opts: image=${{ steps.buildkit.outputs.ref }}

      - name: 🔑 Login to GHCR
        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
@@ -116,9 +116,7 @@ jobs:
          cache-dependency-path: pyproject.toml

      - name: 📦 Install Python deps
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install PyYAML
+        run: PYTHON=python bash scripts/install/python.sh

      - name: 📦 Install skopeo
        run: ./scripts/install/apt.sh skopeo
@@ -0,0 +1,572 @@
+name: "🎵 CI: Orchestrator"
+on:
+  workflow_call:
+    inputs:
+      distros:
+        description: "Space-separated distro list"
+        required: true
+        type: string
+      whitelist:
+        description: "Optional space-separated whitelist of app ids"
+        required: false
+        type: string
+        default: ""
+      lifecycles:
+        description: "Lifecycle envelope for test-deploy discovery (comma/space-separated)."
+        required: false
+        type: string
+        default: ""
+      modes:
+        description: "Deploy modes to run (space-separated: 'swarm compose host'). Only listed modes run their deploy job."
+        required: false
+        type: string
+        default: "swarm compose host"
+      priority:
+        description: "Optional space-separated app ids for the priority line: deployed first in every mode; the regular line runs the remaining roles only after every priority job succeeded."
+        required: false
+        type: string
+        default: ""
+      sequencing:
+        description: "🐛 Deploy-mode order per line. 'serial': swarm, then compose, then host. 'parallel': all at once. 'auto': serial above the job threshold."
+        required: false
+        type: string
+        default: "auto"
+      mode_fail_fast:
+        description: "🛑 Stop a serial line at its first failed mode. Off: keep deploying the remaining modes."
+        required: false
+        type: boolean
+        default: true
+      filesystem:
+        description: "🗄️ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three."
+        required: false
+        type: string
+        default: ""
+      image_wait_attempts:
+        description: "Polling attempts for waiting on prebuilt or mirrored images"
+        required: false
+        type: string
+        default: "1980"
+      image_wait_sleep_seconds:
+        description: "Polling interval in seconds for waiting on prebuilt or mirrored images"
+        required: false
+        type: string
+        default: "10"
+      workspace:
+        description: "Run the workspace test (dev-runtime image matrix). 'auto' runs it on a global scope only, 'true'/'false' force it."
+        required: false
+        type: string
+        default: "auto"
+      instructions:
+        description: "Run the instructions test (role README Production command). 'auto' runs it on a global scope only, 'true'/'false' force it."
+        required: false
+        type: string
+        default: "auto"
+      test_migration:
+        description: "'true' runs the opt-in mail-provider migration scenario (Mailu → Stalwart)."
+        required: false
+        type: string
+        default: "false"
+      trusted_pr:
+        description: "True when the PR carries the maintainer '🛡️ Trusted' label; skips the fork-prereq wait."
+        required: false
+        type: boolean
+        default: false
+      force_cancel_in_progress:
+        description: "When true, a new run always cancels the running orchestrator on the same ref, ignoring the CI_CANCEL_IN_PROGRESS repository variable."
+        required: false
+        type: boolean
+        default: false
+
+concurrency:
+  group: infinito-test-${{ github.ref }}
+  cancel-in-progress: ${{ inputs.force_cancel_in_progress || vars.CI_CANCEL_IN_PROGRESS != 'false' }}
+
+permissions:
+  actions: write
+  contents: read
+  packages: write
+  security-events: write
+  issues: write
+
+jobs:
+  wait-fork-prereq-run:
+    name: ⏳ Wait for fork run
+    runs-on: ubuntu-latest
+    env:
+      WAIT_REQUIRED: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && !inputs.trusted_pr }}
+    steps:
+      - name: 🏷️ Mark fork prerequisites as ready outside fork PRs or for trusted PRs
+        if: ${{ env.WAIT_REQUIRED != 'true' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          echo "Fork prerequisites are implicitly ready (non-fork PR or '🛡️ Trusted' label)."
+
+      - name: ⬇️ Checkout trusted workflow repo (helpers)
+        if: ${{ env.WAIT_REQUIRED == 'true' }}
+        uses: actions/checkout@v7.0.0
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          persist-credentials: false
+
+      - name: 🔑 Login to GHCR for CI image checks
+        if: ${{ env.WAIT_REQUIRED == 'true' }}
+        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: ⏳ Wait for privileged fork CI images
+        if: ${{ env.WAIT_REQUIRED == 'true' }}
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          WORKFLOW_FILE: entry-pr-change-orchestrate.yml
+          TARGET_EVENT: pull_request_target
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          PR_UPDATED_AT: ${{ github.event.pull_request.updated_at }}
+          INFINITO_DISTROS: "arch debian ubuntu fedora centos"
+          IMAGE_TAG: ci-${{ github.sha }}
+          WAIT_ATTEMPTS: ${{ inputs.image_wait_attempts }}
+          WAIT_SLEEP_SECONDS: ${{ inputs.image_wait_sleep_seconds }}
+        run: ./scripts/meta/wait/images.sh
+
+  fork-prereqs-ready:
+    name: 🚦 Gate Dependencies
+    needs: wait-fork-prereq-run
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7.0.0
+      - name: ✅ Assert privileged prerequisites are ready
+        shell: bash
+        env:
+          RESULT: ${{ needs.wait-fork-prereq-run.result }}
+        run: ./scripts/github/gate/fork_prereqs.sh
+
+  security-codeql:
+    uses: ./.github/workflows/cron-security-codeql.yml
+
+  lint:
+    uses: ./.github/workflows/call-lint.yml
+
+  test:
+    uses: ./.github/workflows/call-test.yml
+
+  build-ci-images:
+    uses: ./.github/workflows/call-images-build-ci.yml
+    with:
+      distros: "arch debian ubuntu fedora centos"
+      concurrency_channel: ci-orchestrator-build-consumer
+    secrets: inherit
+  
+  test-dns:
+    needs: 
+      - build-ci-images
+    uses: ./.github/workflows/call-test-dns.yml
+    with:
+      distros: "arch debian ubuntu fedora centos"
+  
+  images-mirror-missing:
+    uses: ./.github/workflows/call-images-mirror-missing.yml
+    with:
+      concurrency_channel: ci-orchestrator-mirror-consumer
+      image_wait_attempts: ${{ inputs.image_wait_attempts }}
+      image_wait_sleep_seconds: ${{ inputs.image_wait_sleep_seconds }}
+    secrets: inherit
+
+  sequencing:
+    name: 🐛 Sequencing
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    outputs:
+      priority: ${{ steps.priority.outputs.sequencing }}
+      regular: ${{ steps.regular.outputs.sequencing }}
+    env:
+      INFINITO_APP_DISCOVERY_RUNNER: host
+      INFINITO_LIFECYCLES: ${{ inputs.lifecycles }}
+      INFINITO_HOST_EXCLUDE_COMPOSE: ${{ contains(inputs.modes, 'compose') && 'true' || 'false' }}
+    steps:
+      - uses: actions/checkout@v7.0.0
+        with:
+          fetch-depth: 0
+
+      - name: 🔧 Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: 📦 Install project (host venv)
+        shell: bash
+        run: make install
+
+      - name: 🔍 Resolve effective whitelist (diff vs origin/main)
+        id: resolve_wl
+        shell: bash
+        env:
+          INPUT_WHITELIST: ${{ inputs.whitelist }}
+        run: ./scripts/github/resolve/effective_whitelist.sh
+
+      - name: ⭐ Decide the priority line
+        id: priority
+        if: ${{ inputs.priority != '' }}
+        shell: bash
+        run: |
+          source scripts/meta/env/load.sh
+          "${PYTHON}" -m cli.meta.ci.sequencing \
+            --modes "${{ inputs.modes }}" \
+            --whitelist "${{ inputs.priority }}" \
+            --lifecycles "${{ inputs.lifecycles }}" \
+            --choice "${{ inputs.sequencing }}" \
+            | tee -a "$GITHUB_OUTPUT"
+
+      - name: 🔁 Decide the regular line
+        id: regular
+        shell: bash
+        run: |
+          source scripts/meta/env/load.sh
+          "${PYTHON}" -m cli.meta.ci.sequencing \
+            --modes "${{ inputs.modes }}" \
+            --whitelist "${{ steps.resolve_wl.outputs.whitelist }}" \
+            --blacklist "${{ inputs.priority }}" \
+            --lifecycles "${{ inputs.lifecycles }}" \
+            --choice "${{ inputs.sequencing }}" \
+            | tee -a "$GITHUB_OUTPUT"
+
+  test-deploy-swarm-priority:
+    name: ⭐ Priority · swarm
+    if: ${{ inputs.priority != '' && contains(inputs.modes, 'swarm') }}
+    needs:
+      - lint
+      - test
+      - build-ci-images
+      - images-mirror-missing
+    uses: ./.github/workflows/call-test-deploy-swarm.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.priority }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      marker: " ⭐"
+
+  test-deploy-single-node-priority:
+    name: ⭐ Priority · single-node
+    if: >-
+      ${{ inputs.priority != ''
+          && needs.sequencing.outputs.priority == 'parallel'
+          && (contains(inputs.modes, 'compose') || contains(inputs.modes, 'host')) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - lint
+      - test
+      - sequencing
+    uses: ./.github/workflows/call-test-deploy-single-node.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.priority }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      modes: ${{ inputs.modes }}
+      marker: " ⭐"
+
+  test-deploy-compose-priority:
+    name: ⭐ Priority · compose
+    if: >-
+      ${{ always()
+          && inputs.priority != ''
+          && needs.sequencing.outputs.priority == 'serial'
+          && contains(inputs.modes, 'compose')
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.test-dns.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && (!inputs.mode_fail_fast
+              || contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result)) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - lint
+      - test
+      - sequencing
+      - test-deploy-swarm-priority
+    uses: ./.github/workflows/call-test-deploy-compose.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.priority }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      marker: " ⭐"
+
+  test-deploy-host-priority:
+    name: ⭐ Priority · host
+    if: >-
+      ${{ always()
+          && inputs.priority != ''
+          && needs.sequencing.outputs.priority == 'serial'
+          && contains(inputs.modes, 'host')
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.test-dns.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && (!inputs.mode_fail_fast
+              || contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - lint
+      - test
+      - sequencing
+      - test-deploy-compose-priority
+    uses: ./.github/workflows/call-test-deploy-host.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.priority }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      marker: " ⭐"
+      exclude_compose_covered: ${{ contains(inputs.modes, 'compose') && 'true' || 'false' }}
+
+  test-deploy-swarm:
+    name: 🔁 Regular · swarm
+    if: >-
+      ${{ always()
+          && contains(inputs.modes, 'swarm')
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.build-ci-images.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) }}
+    needs:
+      - build-ci-images
+      - images-mirror-missing
+      - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
+      - test-deploy-swarm-priority
+      - lint
+      - test
+    uses: ./.github/workflows/call-test-deploy-swarm.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.whitelist }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      blacklist: ${{ inputs.priority }}
+
+  test-deploy-single-node:
+    name: 🔁 Regular · single-node
+    if: >-
+      ${{ always()
+          && needs.sequencing.outputs.regular == 'parallel'
+          && (contains(inputs.modes, 'compose') || contains(inputs.modes, 'host'))
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.test-dns.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - sequencing
+      - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
+      - test-deploy-swarm-priority
+      - lint
+      - test
+    uses: ./.github/workflows/call-test-deploy-single-node.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.whitelist }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      modes: ${{ inputs.modes }}
+      blacklist: ${{ inputs.priority }}
+
+  test-deploy-compose:
+    name: 🔁 Regular · compose
+    if: >-
+      ${{ always()
+          && needs.sequencing.outputs.regular == 'serial'
+          && contains(inputs.modes, 'compose')
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.test-dns.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result)
+          && (!inputs.mode_fail_fast
+              || contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm.result)) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - sequencing
+      - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
+      - test-deploy-swarm-priority
+      - test-deploy-swarm
+      - lint
+      - test
+    uses: ./.github/workflows/call-test-deploy-compose.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.whitelist }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      blacklist: ${{ inputs.priority }}
+
+  test-deploy-host:
+    name: 🔁 Regular · host
+    if: >-
+      ${{ always()
+          && needs.sequencing.outputs.regular == 'serial'
+          && contains(inputs.modes, 'host')
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.test-dns.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result)
+          && (!inputs.mode_fail_fast
+              || contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose.result)) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - sequencing
+      - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
+      - test-deploy-swarm-priority
+      - test-deploy-compose
+      - lint
+      - test
+    uses: ./.github/workflows/call-test-deploy-host.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.whitelist }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      blacklist: ${{ inputs.priority }}
+      exclude_compose_covered: ${{ contains(inputs.modes, 'compose') && 'true' || 'false' }}
+
+  # Exception: opt-in job — the called workflow gates itself on the forwarded
+  # test_migration input or the INFINITO_TEST_MIGRATION repository variable.
+  test-deploy-migration:
+    needs:
+      - test-dns
+      - images-mirror-missing
+    uses: ./.github/workflows/call-test-deploy-migration.yml
+    with:
+      test_migration: ${{ inputs.test_migration }}
+
+  test-runner-smoke:
+    needs:
+      - test-deploy-swarm
+    if: ${{ always() && vars.CI_SELF_HOSTED_RUNNER_COUNT != '0' && vars.CI_SELF_HOSTED_RUNNER_COUNT != '' && (needs.test-deploy-swarm.result == 'success' || needs.test-deploy-swarm.result == 'failure') }}
+    uses: ./.github/workflows/call-test-runner-smoke.yml
+    with:
+      ref: ${{ github.sha }}
+    secrets: inherit
+
+  test-install-make:
+    uses: ./.github/workflows/call-test-install-make.yml
+  test-install-pkgmgr:
+    uses: ./.github/workflows/call-test-install-pkgmgr.yml
+
+  test-workspace:
+    if: ${{ inputs.workspace == 'true' || (inputs.workspace == 'auto' && (inputs.whitelist == '' || inputs.whitelist == '__ALL__')) }}
+    uses: ./.github/workflows/call-test-workspace.yml
+    needs:
+      - images-mirror-missing
+      - lint
+      - test
+    secrets: inherit
+
+  test-instructions:
+    if: ${{ inputs.instructions == 'true' || (inputs.instructions == 'auto' && (inputs.whitelist == '' || inputs.whitelist == '__ALL__')) }}
+    uses: ./.github/workflows/call-test-instructions.yml
+    needs:
+      - images-mirror-missing
+      - build-ci-images
+      - lint
+      - test
+    with:
+      distros: ${{ inputs.distros }}
+    secrets: inherit
+
+  report-main-failures:
+    name: 🐞 Report main role failures
+    needs:
+      - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
+      - test-deploy-swarm-priority
+      - test-deploy-single-node
+      - test-deploy-compose
+      - test-deploy-host
+      - test-deploy-swarm
+    if: ${{ always() && github.ref == 'refs/heads/main' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      actions: read
+      contents: read
+    steps:
+      - uses: actions/checkout@v7.0.0
+      - name: 🐞 Open or append issues for failed roles
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: python3 -m cli.meta.ci.report_failures --run-id "${{ github.run_id }}" --repo "${{ github.repository }}"
+
+  done:
+    name: 🏁 Finish pipeline
+    needs:
+      - wait-fork-prereq-run
+      - fork-prereqs-ready
+      - build-ci-images
+      - images-mirror-missing
+      - test-dns
+      - test-install-make
+      - test-install-pkgmgr
+      - sequencing
+      - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
+      - test-deploy-swarm-priority
+      - test-deploy-single-node
+      - test-deploy-compose
+      - test-deploy-host
+      - test-deploy-swarm
+      - test-deploy-migration
+      - test-runner-smoke
+      - test-workspace
+      - test-instructions
+      - test
+      - lint
+      - security-codeql
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: 🛑 Fail if any upstream job failed or was cancelled
+        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
+        run: |
+          echo "Pipeline failed: at least one upstream job failed or was cancelled." >&2
+          exit 1
+      - name: ✅ Done
+        run: echo "All checks passed."
@@ -102,7 +102,7 @@ jobs:
    name: 🏗️ Build missing CI images
    needs: check-ci-images
    if: needs.check-ci-images.outputs.missing == 'true'
-    uses: ./.github/workflows/images-build-ci.yml
+    uses: ./.github/workflows/call-images-build-ci.yml
    with:
      distros: ${{ needs.check-ci-images.outputs.distros }}
      checkout_ref: ${{ needs.check-ci-images.outputs.version_tag }}
@@ -138,17 +138,25 @@ jobs:
          ref: ${{ github.sha }}
          persist-credentials: false

+      - name: 🔍 Resolve buildkit image ref
+        id: buildkit
+        shell: bash
+        run: |
+          set -euo pipefail
+          ref="$(bash scripts/meta/resolve/image/service_ref.sh svc-runner buildkit mirror)"
+          echo "ref=${ref}" >> "$GITHUB_OUTPUT"
+
      - name: ⬇️ Pre-pull buildkit image (retry 7x, 20s)
        shell: bash
        env:
          MAX_ATTEMPTS: "7"
          RETRY_DELAY_SECONDS: "20"
-        run: bash scripts/github/runner/pull_with_retry.sh moby/buildkit:buildx-stable-1
+        run: bash scripts/github/runner/pull_with_retry.sh "${{ steps.buildkit.outputs.ref }}"

      - name: 🔧 Set up Docker Buildx
        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
        with:
-          driver-opts: image=moby/buildkit:buildx-stable-1
+          driver-opts: image=${{ steps.buildkit.outputs.ref }}

      - name: 🔑 Login to GHCR
        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
@@ -27,6 +27,11 @@ on:
        required: false
        type: string
        default: ""
+      filesystem:
+        description: "🗄️ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three."
+        required: false
+        type: string
+        default: ""

 permissions:
  contents: read
@@ -41,6 +46,7 @@ env:
  INFINITO_PLAYWRIGHT_KEEP: ${{ vars.INFINITO_PLAYWRIGHT_KEEP }}
  INFINITO_APP_DISCOVERY_RUNNER: host
  INFINITO_DEPLOY_MODE: compose
+  INFINITO_DOCKER_FILESYSTEM_ALLOWED: ${{ inputs.filesystem }}
  INFINITO_LIFECYCLES: ${{ inputs.lifecycles }}
  INFINITO_BLACKLIST: ${{ inputs.blacklist }}
 jobs:
@@ -108,12 +114,17 @@ jobs:
            >> "$GITHUB_STEP_SUMMARY"

  compose:
-    name: 🐳 ${{ matrix.apps }}${{ matrix.variant && format(' {0}',matrix.variant) || '' }}
+    name: 🐳${{ matrix.label }}
    if: needs.discover.outputs.apps != '' && needs.discover.outputs.apps != '[]'
    runs-on: ubuntu-latest
-    timeout-minutes: 345
+    timeout-minutes: 355
    needs: discover

+    concurrency:
+      group: "infinito-deploy-compose-${{ matrix.apps }}#${{ matrix.variant_slug }}"
+      cancel-in-progress: false
+      queue: max
+
    strategy:
      fail-fast: false
      matrix:
@@ -157,7 +168,7 @@ jobs:
        run: ./scripts/install/apt.sh jq

      - name: 🚀 Deploy app
-        timeout-minutes: 300
+        timeout-minutes: 350
        shell: bash
        env:
          apps: ${{ matrix.apps }}
@@ -190,7 +201,7 @@ jobs:

      - name: 🩺 Dump runner state (on failure)
        if: failure()
-        timeout-minutes: 10
+        timeout-minutes: 5
        shell: bash
        run: ./scripts/github/runner/dump_state.sh "post-deploy failure (${{ matrix.apps }})"

@@ -27,6 +27,11 @@ on:
        required: false
        type: string
        default: ""
+      filesystem:
+        description: "🗄️ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three."
+        required: false
+        type: string
+        default: ""
      exclude_compose_covered:
        description: "Drop roles the same run's compose line already deploys"
        required: false
@@ -46,6 +51,7 @@ env:
  INFINITO_PLAYWRIGHT_KEEP: ${{ vars.INFINITO_PLAYWRIGHT_KEEP }}
  INFINITO_APP_DISCOVERY_RUNNER: host
  INFINITO_DEPLOY_MODE: host
+  INFINITO_DOCKER_FILESYSTEM_ALLOWED: ${{ inputs.filesystem }}
  INFINITO_LIFECYCLES: ${{ inputs.lifecycles }}
  INFINITO_BLACKLIST: ${{ inputs.blacklist }}
  INFINITO_HOST_EXCLUDE_COMPOSE: ${{ inputs.exclude_compose_covered }}
@@ -114,12 +120,17 @@ jobs:
            >> "$GITHUB_STEP_SUMMARY"

  host:
-    name: 💻 ${{ matrix.apps }}${{ matrix.variant && format(' {0}',matrix.variant) || '' }}
+    name: 💻${{ matrix.label }}
    if: needs.discover.outputs.apps != '' && needs.discover.outputs.apps != '[]'
    runs-on: ubuntu-latest
-    timeout-minutes: 345
+    timeout-minutes: 355
    needs: discover

+    concurrency:
+      group: "infinito-deploy-host-${{ matrix.apps }}#${{ matrix.variant_slug }}"
+      cancel-in-progress: false
+      queue: max
+
    strategy:
      fail-fast: false
      matrix:
@@ -163,7 +174,7 @@ jobs:
        run: ./scripts/install/apt.sh jq

      - name: 🚀 Deploy host role
-        timeout-minutes: 300
+        timeout-minutes: 350
        shell: bash
        env:
          apps: ${{ matrix.apps }}
@@ -196,7 +207,7 @@ jobs:

      - name: 🩺 Dump runner state (on failure)
        if: failure()
-        timeout-minutes: 10
+        timeout-minutes: 5
        shell: bash
        run: ./scripts/github/runner/dump_state.sh "post-deploy failure (${{ matrix.apps }})"

@@ -1,5 +1,3 @@
-name: "📧 Test: Mail-Provider Migration (Mailu → Stalwart)"
-
 # Opt-in migration scenario: deploys Mailu as the ACTIVE provider, then
 # redeploys with the default provider (Stalwart) over kept state — the exact
 # upgrade path of an existing Mailu installation. See
@@ -8,7 +6,16 @@ name: "📧 Test: Mail-Provider Migration (Mailu → Stalwart)"
 # Gated OFF by default (INFINITO_TEST_MIGRATION=false in default.env):
 # enable per run via the manual CI workflow's `test_migration` field, per
 # dispatch of this workflow, or repo-wide via the INFINITO_TEST_MIGRATION
-# GitHub repository variable.
+# GitHub repository variable. The job-level gate lets the dispatch/call input
+# win; the repository variable enables it fleet-wide.
+#
+# Step order: the standalone data-migration test runs first — it seeds a
+# Mailu-layout maildir stump, migrates it into a single pinned Stalwart
+# container and verifies the data with no platform deploy involved, so it
+# fails in ~1 minute when the migration script breaks. The full platform
+# cutover (Mailu-active deploy → default-provider takeover over kept state)
+# runs second.
+name: "📧 Test: Mail-Provider Migration (Mailu → Stalwart)"

 on:
  workflow_dispatch:
@@ -39,7 +46,6 @@ env:
 jobs:
  mail-migration:
    name: 🚀 Mailu → Stalwart migration
-    # The gate: dispatch/call input wins; the repo variable enables it fleet-wide.
    if: ${{ inputs.test_migration == true || inputs.test_migration == 'true' || vars.INFINITO_TEST_MIGRATION == 'true' }}
    runs-on: ubuntu-latest
    timeout-minutes: 345
@@ -77,15 +83,10 @@ jobs:
        shell: bash
        run: make install

-      # Cheap first: seeds a Mailu-layout maildir stump, migrates it into a
-      # single pinned Stalwart container and verifies the data — no platform
-      # deploy involved (fails in ~1 minute when the migration script breaks).
      - name: 🔬 Standalone data-migration test (Mailu stump → Stalwart)
        shell: bash
        run: make test-migration

-      # Full platform cutover second: Mailu-active deploy → default-provider
-      # takeover over kept state.
      - name: 📧 Run the mail-provider migration scenario
        shell: bash
        run: make compose-deploy mode=migration
@@ -32,6 +32,11 @@ on:
        required: false
        type: string
        default: ""
+      filesystem:
+        description: "🗄️ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three."
+        required: false
+        type: string
+        default: ""

 permissions:
  contents: read
@@ -41,22 +46,24 @@ jobs:
  compose:
    name: 📦 Compose
    if: ${{ contains(inputs.modes, 'compose') }}
-    uses: ./.github/workflows/test-deploy-compose.yml
+    uses: ./.github/workflows/call-test-deploy-compose.yml
    with:
      distros: ${{ inputs.distros }}
      whitelist: ${{ inputs.whitelist }}
      lifecycles: ${{ inputs.lifecycles }}
      blacklist: ${{ inputs.blacklist }}
      marker: ${{ inputs.marker }}
+      filesystem: ${{ inputs.filesystem }}

  host:
    name: 💻 Host
    if: ${{ contains(inputs.modes, 'host') }}
-    uses: ./.github/workflows/test-deploy-host.yml
+    uses: ./.github/workflows/call-test-deploy-host.yml
    with:
      distros: ${{ inputs.distros }}
      whitelist: ${{ inputs.whitelist }}
      lifecycles: ${{ inputs.lifecycles }}
      blacklist: ${{ inputs.blacklist }}
      marker: ${{ inputs.marker }}
+      filesystem: ${{ inputs.filesystem }}
      exclude_compose_covered: ${{ contains(inputs.modes, 'compose') && 'true' || 'false' }}
@@ -5,9 +5,8 @@ on:
    inputs:
      distros:
        description: "Space-separated distro list (shown in the discover job name)"
-        required: false
+        required: true
        type: string
-        default: "arch debian ubuntu fedora centos"
      whitelist:
        description: "Optional space-separated whitelist of app ids"
        required: false
@@ -23,6 +22,11 @@ on:
        required: false
        type: string
        default: ""
+      filesystem:
+        description: "🗄️ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three."
+        required: false
+        type: string
+        default: ""
      marker:
        description: "Optional suffix appended to the discover job name (e.g. ' ⭐' for the priority line)"
        required: false
@@ -32,9 +36,8 @@ on:
    inputs:
      distros:
        description: "Space-separated distro list (shown in the discover job name)"
-        required: false
+        required: true
        type: string
-        default: "arch debian ubuntu fedora centos"
      whitelist:
        description: "Optional space-separated whitelist of app ids"
        required: false
@@ -50,18 +53,25 @@ on:
        required: false
        type: string
        default: ""
+      filesystem:
+        description: "🗄️ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three."
+        required: false
+        type: string
+        default: ""

 permissions:
  contents: read
  packages: read

 env:
+  INFINITO_DISTROS: ${{ inputs.distros }}
  NIX_CONFIG: |
    access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
  INFINITO_IMAGE_TAG: ci-${{ github.sha }}
  INFINITO_GHCR_MIRROR_PREFIX: mirror
  INFINITO_APP_DISCOVERY_RUNNER: host
  INFINITO_DEPLOY_MODE: swarm
+  INFINITO_DOCKER_FILESYSTEM_ALLOWED: ${{ inputs.filesystem }}
  INFINITO_LIFECYCLES: ${{ inputs.lifecycles }}
  INFINITO_BLACKLIST: ${{ inputs.blacklist }}

@@ -134,12 +144,17 @@ jobs:
            >> "$GITHUB_STEP_SUMMARY"

  swarm:
-    name: "🐝 ${{ matrix.apps }}${{ matrix.variant && format(' {0}',matrix.variant) || '' }}"
+    name: "🐝${{ matrix.label }}"
    if: needs.discover.outputs.apps != '' && needs.discover.outputs.apps != '[]'
    runs-on: ubuntu-latest
-    timeout-minutes: 345
+    timeout-minutes: 355
    needs: discover

+    concurrency:
+      group: "infinito-deploy-swarm-${{ matrix.apps }}#${{ matrix.variant_slug }}"
+      cancel-in-progress: false
+      queue: max
+
    strategy:
      fail-fast: false
      matrix:
@@ -204,24 +219,16 @@ jobs:
          source scripts/meta/env/load.sh
          echo "${VENV}/bin" >> "${GITHUB_PATH}"

-      - name: "🥾 Bring up + bootstrap the simulated cluster (compose up, IPs, lab DNS)"
-        run: ${{ env.SCRIPTS }}/01_bootstrap.sh
-
-      - name: "🔧 Project setup (.dockerignore + dotenv + scripts/setup.sh)"
-        run: make setup
-
-      - name: "🚀 Matrix-deploy ${{ matrix.apps }}: provision/deploy/e2e/verify per round"
-        timeout-minutes: 345
+      - name: "🚀 Drill ${{ matrix.apps }} per distro: ${{ inputs.distros }}"
        shell: bash
        env:
          variant: ${{ matrix.variant }}
-        run: |
-          source scripts/meta/env/load.sh
-          if [ "$(id -u)" -eq 0 ]; then
-            python3 -m utils.tests.swarm.matrix 2>&1 | tee "/tmp/ansible-${{ matrix.apps }}.log"
-          else
-            sudo -E env "PATH=$PATH" "HOME=$HOME" "SWARM_NAME=${SWARM_NAME:-}" "variant=${variant:-}" "disable=${disable:-}" python3 -m utils.tests.swarm.matrix 2>&1 | tee "/tmp/ansible-${{ matrix.apps }}.log"
-          fi
+        run: ./scripts/tests/deploy/distros.sh "${{ env.SCRIPTS }}/00_one.sh" 2>&1 | tee "/tmp/ansible-${{ matrix.apps }}.log"
+
+      - name: "🗄️ Effective node filesystems"
+        if: always()
+        shell: bash
+        run: ./scripts/tests/deploy/utils/filesystem/report.sh "/tmp/ansible-${{ matrix.apps }}.log"

      - name: "📊 Role runtimes"
        if: always()
@@ -246,24 +253,6 @@ jobs:
          path: /tmp/role-runtimes-${{ matrix.apps }}.csv
          if-no-files-found: ignore

-      - name: "🌱 Seed content (pre-drain marker on NFS volume)"
-        run: ${{ env.SCRIPTS }}/05_seed_content.sh
-
-      - name: "💪 Drain the worker running ${{ matrix.apps }} + force reschedule"
-        run: ${{ env.SCRIPTS }}/06_drain_worker.sh
-
-      - name: "✅ Assert state survived the reschedule"
-        run: ${{ env.SCRIPTS }}/07_assert_state.sh
-
-      - name: "🩺 Collect rescue diagnostics (recursive DiD snapshot)"
-        if: failure()
-        timeout-minutes: 25
-        shell: bash
-        run: |
-          INFINITO_RESCUE_DIAGNOSTICS_DIR="${INFINITO_RESCUE_DIAGNOSTICS_BASE}/${{ matrix.apps }}" \
-            python3 utils/diagnostics/container.py \
-            "${{ matrix.apps }}" "post-deploy failure" || true
-
      - name: "⬆️ Upload rescue diagnostics"
        if: ${{ always() && !env.ACT }}
        uses: actions/upload-artifact@v7
@@ -272,11 +261,6 @@ jobs:
          path: ${{ env.INFINITO_RESCUE_DIAGNOSTICS_BASE }}/
          if-no-files-found: ignore

-      - name: "📊 Collect diagnostics on failure"
-        if: failure()
-        timeout-minutes: 15
-        run: scripts/tests/deploy/swarm/utils/collect/diagnostics.sh
-
      - name: "🗜️ Zip generated inventory for debugging"
        if: always()
        shell: bash
@@ -292,19 +276,6 @@ jobs:
          path: /tmp/inventory-swarm-${{ matrix.apps }}.zip
          if-no-files-found: ignore

-      - name: "📊 Collect Playwright reports from the manager"
-        if: always()
-        timeout-minutes: 15
-        shell: bash
-        env:
-          APP_ID: ${{ matrix.apps }}
-        run: scripts/tests/deploy/swarm/utils/collect/playwright_reports.sh
-
-      - name: "🕸️ Swarm structure summary"
-        if: always()
-        shell: bash
-        run: scripts/tests/deploy/swarm/utils/collect/topology_summary.sh
-
      - name: "📋 Playwright step summary"
        if: always()
        shell: bash
@@ -1,8 +1,12 @@
-name: "📖 Test: Guide"
+name: "📖 Test: Instructions"

 on:
  workflow_call:
    inputs:
+      distros:
+        description: "Space-separated distro list"
+        required: true
+        type: string
      role:
        description: "Role to test; empty picks one at random"
        required: false
@@ -20,6 +24,10 @@ on:
        default: ""
  workflow_dispatch:
    inputs:
+      distros:
+        description: "Space-separated distro list"
+        required: true
+        type: string
      role:
        description: "Role to test; empty picks one at random"
        required: false
@@ -42,12 +50,16 @@ permissions:

 jobs:
  deploy:
-    name: 📖 Guide
+    name: 📖 Instructions
    runs-on: ubuntu-latest
-    timeout-minutes: 90
+    timeout-minutes: 345
+    concurrency:
+      group: infinito-instructions
+      cancel-in-progress: false
+      queue: max
    env:
      INFINITO_IMAGE_TAG: ci-${{ github.sha }}
-      INFINITO_DISTROS: "arch debian ubuntu fedora centos"
+      INFINITO_DISTROS: ${{ inputs.distros }}
    steps:
      - if: ${{ !env.ACT }}
        uses: actions/checkout@v7.0.0
@@ -56,14 +68,14 @@ jobs:
        shell: bash
        run: grep '^INFINITO_RESCUE_DIAGNOSTICS_BASE=' default.env >> "$GITHUB_ENV"

-      - name: 🐍 Install guide + compose deps (Act runner only)
+      - name: 🐍 Install instructions + compose deps (Act runner only)
        if: ${{ env.ACT }}
        shell: bash
        run: |
-          pip install --quiet --break-system-packages pyyaml
+          PYTHON=python3 PIP_BREAK_SYSTEM_PACKAGES=1 bash scripts/install/python.sh
          apt-get update -qq && apt-get install -y -qq gettext-base jq

-      - name: 🎲 Select the role, distro, and deploy mode
+      - name: 🎲 Select the role and deploy mode
        shell: bash
        run: |
          set -euo pipefail
@@ -80,39 +92,14 @@ jobs:
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

-      - name: 🖥️ Install a host role inside its pkgmgr distro container
-        if: ${{ env.GUIDE_MODE == 'host' }}
+      - name: 📖 Replay the instructions across ${{ inputs.distros }}
        shell: bash
-        run: bash scripts/github/guide/host_deploy.sh
-
-      - name: 🖥️ Boot the systemd server for a compose role
-        if: ${{ env.GUIDE_MODE == 'compose' }}
-        shell: bash
-        run: |
-          set -euo pipefail
-          INFINITO_IMAGE="ghcr.io/$(scripts/meta/resolve/repository/owner.sh)/$(scripts/meta/resolve/repository/name.sh)/${INFINITO_DISTRO}:ci-${{ github.sha }}"
-          echo "INFINITO_IMAGE=${INFINITO_IMAGE}" >> "$GITHUB_ENV"
-          export INFINITO_IMAGE
-          make compose-up
-
-      - name: 🚀 Deploy the compose role against the server
-        if: ${{ env.GUIDE_MODE == 'compose' }}
-        shell: bash
-        run: bash scripts/github/guide/compose_deploy.sh
-
-      - name: 🩺 Collect rescue diagnostics (recursive DiD snapshot)
-        if: failure()
-        timeout-minutes: 25
-        shell: bash
-        run: |
-          INFINITO_RESCUE_DIAGNOSTICS_DIR="${INFINITO_RESCUE_DIAGNOSTICS_BASE}/${GUIDE_ROLE}" \
-            python3 utils/diagnostics/container.py \
-            "${GUIDE_ROLE}" "guide ${GUIDE_MODE} post-deploy failure" || true
+        run: ./scripts/tests/deploy/distros.sh scripts/github/guide/one.sh

      - name: ⬆️ Upload rescue diagnostics
        if: ${{ always() && !env.ACT }}
        uses: actions/upload-artifact@v7
        with:
-          name: rescue-diagnostics-guide-${{ env.GUIDE_ROLE }}
+          name: rescue-diagnostics-instructions-${{ env.GUIDE_ROLE }}
          path: ${{ env.INFINITO_RESCUE_DIAGNOSTICS_BASE }}/
          if-no-files-found: ignore
@@ -0,0 +1,143 @@
+name: "💻 Test: Workspace"
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+permissions:
+  actions: write
+  contents: read
+  packages: read
+
+jobs:
+  prepare:
+    name: 📋 Workspace Matrix
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      images: ${{ steps.matrix.outputs.images }}
+
+    steps:
+      - name: ⬇️ Checkout repository
+        uses: actions/checkout@v7.0.0
+
+      - name: 🐍 Read the distro SPOT
+        id: matrix
+        shell: bash
+        run: |
+          set -euo pipefail
+          images="$(python3 -m cli.meta.ci.dev_runtime images)"
+          echo "images=${images}" | tee -a "$GITHUB_OUTPUT"
+
+  test-workspace:
+    name: 🧑‍💻 Workspace (${{ matrix.dev_runtime_image }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 345
+    needs: prepare
+
+    concurrency:
+      group: infinito-workspace-${{ matrix.dev_runtime_image }}
+      cancel-in-progress: false
+      queue: max
+
+    strategy:
+      fail-fast: false
+      matrix:
+        dev_runtime_image: ${{ fromJson(needs.prepare.outputs.images) }}
+
+    env:
+      DEV_RUNTIME_IMAGE: ${{ matrix.dev_runtime_image }}
+      NIX_CONFIG: |
+        access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+      INFINITO_CACHE_REGISTRY_MAX_SIZE: 16g
+      INFINITO_CACHE_REGISTRY_HOST_PATH: /mnt/cache/registry/mirror
+      INFINITO_CACHE_REGISTRY_CA_HOST_PATH: /mnt/cache/registry/ca
+      INFINITO_CACHE_PACKAGE_BLOBSTORE_MAX: 8g
+      INFINITO_CACHE_PACKAGE_HEAP: 1g
+      INFINITO_CACHE_PACKAGE_DIRECT_MEM: 1g
+      INFINITO_CACHE_PACKAGE_HOST_PATH: /mnt/cache/package/data
+
+    steps:
+      - name: 🧹 Free disk space
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+        with:
+          tool-cache: true
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: false
+
+      - name: ⬇️ Checkout repository
+        uses: actions/checkout@v7.0.0
+
+      - name: "📥 Load the rescue paths into job env"
+        shell: bash
+        run: |
+          set -euo pipefail
+          {
+            grep '^INFINITO_RESCUE_DIAGNOSTICS_BASE=' default.env
+            echo "INFINITO_WORKSPACE_SLUG=$(printf '%s' "${DEV_RUNTIME_IMAGE}" | tr '/:' '--')"
+          } >> "$GITHUB_ENV"
+
+      - name: "📥 Resolve the workspace distro"
+        shell: bash
+        run: |
+          set -euo pipefail
+          distro="$(python3 -m cli.meta.ci.dev_runtime distro --image "${DEV_RUNTIME_IMAGE}")"
+          echo "INFINITO_DISTRO=${distro}" >> "$GITHUB_ENV"
+
+      - name: ▶️ Run workspace routine
+        shell: bash
+        run: |
+          set -euo pipefail
+          HOST_WORKDIR="$(pwd -P)"
+
+          docker run --rm \
+            -e GITHUB_ACTIONS=true \
+            -e GITHUB_REPOSITORY="${{ github.repository }}" \
+            -e GITHUB_REPOSITORY_OWNER="${{ github.repository_owner }}" \
+            -e GITHUB_SHA="${{ github.sha }}" \
+            -e DEV_RUNTIME_IMAGE \
+            -e INFINITO_DISTRO \
+            -e INFINITO_IMAGE="infinito-${INFINITO_DISTRO}" \
+            -e INFINITO_PULL_POLICY=never \
+            -e INFINITO_BUILD=1 \
+            -e NIX_CONFIG \
+            -e INFINITO_CACHE_REGISTRY_MAX_SIZE \
+            -e INFINITO_CACHE_REGISTRY_HOST_PATH \
+            -e INFINITO_CACHE_REGISTRY_CA_HOST_PATH \
+            -e INFINITO_CACHE_PACKAGE_BLOBSTORE_MAX \
+            -e INFINITO_CACHE_PACKAGE_HEAP \
+            -e INFINITO_CACHE_PACKAGE_DIRECT_MEM \
+            -e INFINITO_CACHE_PACKAGE_HOST_PATH \
+            --network host \
+            -v "${HOST_WORKDIR}:${HOST_WORKDIR}" \
+            -v "/mnt/docker:/var/lib/docker" \
+            -v "/var/run/docker.sock:/var/run/docker.sock" \
+            -w "${HOST_WORKDIR}" \
+            "${DEV_RUNTIME_IMAGE}" \
+            bash -lc '
+              bash ./scripts/tests/environment/00_orchestrator.sh
+            '
+
+      - name: 🩺 Capture rescue diagnostics
+        if: failure()
+        timeout-minutes: 15
+        shell: bash
+        run: |
+          set -euo pipefail
+          rescue_dir="${INFINITO_RESCUE_DIAGNOSTICS_BASE:?}/${INFINITO_WORKSPACE_SLUG:?}"
+          INFINITO_RESCUE_DIAGNOSTICS_DIR="${rescue_dir}" \
+            python3 utils/diagnostics/container.py \
+            "${INFINITO_WORKSPACE_SLUG}" "workspace suite failure" || true
+          bash ./scripts/tests/deploy/utils/rescue_index.sh "${rescue_dir}"
+
+      - name: ⬆️ Upload rescue diagnostics
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: rescue-diagnostics-workspace-${{ env.INFINITO_WORKSPACE_SLUG }}
+          path: ${{ env.INFINITO_RESCUE_DIAGNOSTICS_BASE }}/
+          if-no-files-found: ignore
@@ -1,321 +0,0 @@
-name: "🎵 CI: Orchestrator"
-on:
-  workflow_call:
-    inputs:
-      distros:
-        description: "Space-separated distro list"
-        required: false
-        type: string
-        default: "arch debian ubuntu fedora centos"
-      whitelist:
-        description: "Optional space-separated whitelist of app ids"
-        required: false
-        type: string
-        default: ""
-      lifecycles:
-        description: "Lifecycle envelope for test-deploy discovery (comma/space-separated)."
-        required: false
-        type: string
-        default: ""
-      modes:
-        description: "Deploy modes to run (space-separated: 'swarm compose host'). Only listed modes run their deploy job."
-        required: false
-        type: string
-        default: "swarm compose host"
-      priority:
-        description: "Optional space-separated app ids for the priority line: deployed first in every mode; the regular line runs the remaining roles only after every priority job succeeded."
-        required: false
-        type: string
-        default: ""
-      image_wait_attempts:
-        description: "Polling attempts for waiting on prebuilt or mirrored images"
-        required: false
-        type: string
-        default: "1980"
-      image_wait_sleep_seconds:
-        description: "Polling interval in seconds for waiting on prebuilt or mirrored images"
-        required: false
-        type: string
-        default: "10"
-      roles_only:
-        description: "When 'true', skip the env-matrix test-development job."
-        required: false
-        type: string
-        default: "false"
-      trusted_pr:
-        description: "True when the PR carries the maintainer '🛡️ Trusted' label; skips the fork-prereq wait."
-        required: false
-        type: boolean
-        default: false
-      test_migration:
-        description: "'true' runs the opt-in mail-provider migration scenario (Mailu → Stalwart)."
-        required: false
-        type: string
-        default: "false"
-
-concurrency:
-  group: infinito-test-${{ github.ref }}
-  cancel-in-progress: false
-
-permissions:
-  actions: write
-  contents: read
-  packages: write
-  security-events: write
-  issues: write
-
-jobs:
-  wait-fork-prereq-run:
-    name: ⏳ Wait for fork run
-    runs-on: ubuntu-latest
-    env:
-      WAIT_REQUIRED: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && !inputs.trusted_pr }}
-    steps:
-      - name: 🏷️ Mark fork prerequisites as ready outside fork PRs or for trusted PRs
-        if: ${{ env.WAIT_REQUIRED != 'true' }}
-        shell: bash
-        run: |
-          set -euo pipefail
-          echo "Fork prerequisites are implicitly ready (non-fork PR or '🛡️ Trusted' label)."
-
-      - name: ⬇️ Checkout trusted workflow repo (helpers)
-        if: ${{ env.WAIT_REQUIRED == 'true' }}
-        uses: actions/checkout@v7.0.0
-        with:
-          ref: ${{ github.event.pull_request.base.sha }}
-          persist-credentials: false
-
-      - name: 🔑 Login to GHCR for CI image checks
-        if: ${{ env.WAIT_REQUIRED == 'true' }}
-        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: ⏳ Wait for privileged fork CI images
-        if: ${{ env.WAIT_REQUIRED == 'true' }}
-        shell: bash
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          WORKFLOW_FILE: entry-pull-request-change.yml
-          TARGET_EVENT: pull_request_target
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-          PR_UPDATED_AT: ${{ github.event.pull_request.updated_at }}
-          INFINITO_DISTROS: "arch debian ubuntu fedora centos"
-          IMAGE_TAG: ci-${{ github.sha }}
-          WAIT_ATTEMPTS: ${{ inputs.image_wait_attempts }}
-          WAIT_SLEEP_SECONDS: ${{ inputs.image_wait_sleep_seconds }}
-        run: ./scripts/meta/wait/images.sh
-
-  fork-prereqs-ready:
-    name: 🚦 Gate Dependencies
-    needs: wait-fork-prereq-run
-    if: ${{ always() }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v7.0.0
-      - name: ✅ Assert privileged prerequisites are ready
-        shell: bash
-        env:
-          RESULT: ${{ needs.wait-fork-prereq-run.result }}
-        run: ./scripts/github/gate/fork_prereqs.sh
-
-  security-codeql:
-    uses: ./.github/workflows/security-codeql.yml
-
-  lint:
-    uses: ./.github/workflows/lint.yml
-
-  test:
-    uses: ./.github/workflows/test.yml
-
-  build-ci-images:
-    uses: ./.github/workflows/images-build-ci.yml
-    with:
-      distros: "arch debian ubuntu fedora centos"
-      concurrency_channel: ci-orchestrator-build-consumer
-    secrets: inherit
-  
-  test-dns:
-    needs: 
-      - build-ci-images
-    uses: ./.github/workflows/test-dns.yml
-    with:
-      distros: "arch debian ubuntu fedora centos"
-  
-  images-mirror-missing:
-    uses: ./.github/workflows/images-mirror-missing.yml
-    with:
-      concurrency_channel: ci-orchestrator-mirror-consumer
-      image_wait_attempts: ${{ inputs.image_wait_attempts }}
-      image_wait_sleep_seconds: ${{ inputs.image_wait_sleep_seconds }}
-    secrets: inherit
-
-  test-deploy-single-node-priority:
-    name: ⭐ Priority · single-node
-    if: ${{ inputs.priority != '' && (contains(inputs.modes, 'compose') || contains(inputs.modes, 'host')) }}
-    needs:
-      - test-dns
-      - images-mirror-missing
-      - lint
-      - test
-    uses: ./.github/workflows/test-deploy-single-node.yml
-    with:
-      distros: ${{ inputs.distros }}
-      whitelist: ${{ inputs.priority }}
-      lifecycles: ${{ inputs.lifecycles }}
-      modes: ${{ inputs.modes }}
-      marker: " ⭐"
-
-  test-deploy-swarm-priority:
-    name: ⭐ Priority · swarm
-    if: ${{ inputs.priority != '' && contains(inputs.modes, 'swarm') }}
-    needs:
-      - lint
-      - test
-      - build-ci-images
-      - images-mirror-missing
-    uses: ./.github/workflows/test-deploy-swarm.yml
-    with:
-      distros: ${{ inputs.distros }}
-      whitelist: ${{ inputs.priority }}
-      lifecycles: ${{ inputs.lifecycles }}
-      marker: " ⭐"
-
-  test-deploy-single-node:
-    name: 🔁 Regular · single-node
-    if: >-
-      ${{ always()
-          && (contains(inputs.modes, 'compose') || contains(inputs.modes, 'host'))
-          && needs.lint.result == 'success'
-          && needs.test.result == 'success'
-          && needs.test-dns.result == 'success'
-          && needs.images-mirror-missing.result == 'success'
-          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
-          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) }}
-    needs:
-      - test-dns
-      - images-mirror-missing
-      - test-deploy-single-node-priority
-      - test-deploy-swarm-priority
-      - lint
-      - test
-    uses: ./.github/workflows/test-deploy-single-node.yml
-    with:
-      distros: ${{ inputs.distros }}
-      whitelist: ${{ inputs.whitelist }}
-      lifecycles: ${{ inputs.lifecycles }}
-      modes: ${{ inputs.modes }}
-      blacklist: ${{ inputs.priority }}
-
-  test-deploy-swarm:
-    name: 🔁 Regular · swarm
-    if: >-
-      ${{ always()
-          && contains(inputs.modes, 'swarm')
-          && needs.lint.result == 'success'
-          && needs.test.result == 'success'
-          && needs.build-ci-images.result == 'success'
-          && needs.images-mirror-missing.result == 'success'
-          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
-          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) }}
-    needs:
-      - build-ci-images
-      - images-mirror-missing
-      - test-deploy-single-node-priority
-      - test-deploy-swarm-priority
-      - lint
-      - test
-    uses: ./.github/workflows/test-deploy-swarm.yml
-    with:
-      distros: ${{ inputs.distros }}
-      whitelist: ${{ inputs.whitelist }}
-      lifecycles: ${{ inputs.lifecycles }}
-      blacklist: ${{ inputs.priority }}
-
-  # Exception: opt-in job — the called workflow gates itself on the forwarded
-  # test_migration input or the INFINITO_TEST_MIGRATION repository variable.
-  test-deploy-migration:
-    needs:
-      - test-dns
-      - images-mirror-missing
-    uses: ./.github/workflows/test-deploy-migration.yml
-    with:
-      test_migration: ${{ inputs.test_migration }}
-
-  test-runner-smoke:
-    needs:
-      - test-deploy-swarm
-    if: ${{ always() && vars.CI_SELF_HOSTED_RUNNER_COUNT != '0' && vars.CI_SELF_HOSTED_RUNNER_COUNT != '' && (needs.test-deploy-swarm.result == 'success' || needs.test-deploy-swarm.result == 'failure') }}
-    uses: ./.github/workflows/test-runner-smoke.yml
-    with:
-      ref: ${{ github.sha }}
-    secrets: inherit
-
-  test-install-make:
-    uses: ./.github/workflows/test-install-make.yml
-  test-install-pkgmgr:
-    uses: ./.github/workflows/test-install-pkgmgr.yml
-
-  test-development:
-    if: ${{ inputs.roles_only != 'true' }}
-    uses: ./.github/workflows/test-environment.yml
-    needs:
-      - images-mirror-missing
-    secrets: inherit
-
-  test-guide:
-    uses: ./.github/workflows/test-guide.yml
-    needs:
-      - images-mirror-missing
-      - build-ci-images
-    secrets: inherit
-
-  report-main-failures:
-    name: 🐛 Report main role failures
-    needs:
-      - test-deploy-single-node-priority
-      - test-deploy-swarm-priority
-      - test-deploy-single-node
-      - test-deploy-swarm
-    if: ${{ always() && github.ref == 'refs/heads/main' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      actions: read
-      contents: read
-    steps:
-      - uses: actions/checkout@v7.0.0
-      - name: 🐛 Open or append issues for failed roles
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: python3 -m cli.meta.ci.report_failures --run-id "${{ github.run_id }}" --repo "${{ github.repository }}"
-
-  done:
-    name: 🏁 Finish pipeline
-    needs:
-      - test-install-make
-      - test-install-pkgmgr
-      - test-deploy-single-node-priority
-      - test-deploy-swarm-priority
-      - test-deploy-single-node
-      - test-deploy-swarm
-      - test-deploy-migration
-      - test-runner-smoke
-      - test-development
-      - test-guide
-      - test
-      - lint
-      - security-codeql
-    if: ${{ always() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: 🛑 Fail if any upstream job failed or was cancelled
-        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
-        run: |
-          echo "Pipeline failed: at least one upstream job failed or was cancelled." >&2
-          exit 1
-      - name: ✅ Done
-        run: echo "All checks passed."
@@ -1,4 +1,4 @@
-name: "🧹 Cleanup: Stale Repository Data"
+name: "⏰ Cleanup: Stale Repository Data"

 on:
  schedule:
@@ -1,4 +1,4 @@
-name: "🧹 Images: Cleanup CI (GHCR)"
+name: "⏰ Images: Cleanup CI (GHCR)"

 on:
  schedule:
@@ -1,4 +1,4 @@
-name: "🪞 Mirror: Docker Hub → GHCR"
+name: "⏰ Mirror: Docker Hub → GHCR"

 on:
  schedule:
@@ -87,9 +87,7 @@ jobs:
          cache-dependency-path: pyproject.toml

      - name: 📦 Install Python deps
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install PyYAML
+        run: PYTHON=python bash scripts/install/python.sh

      - name: 📦 Install skopeo
        run: ./scripts/install/apt.sh skopeo
@@ -1,4 +1,4 @@
-name: "🚀 Release: Highest Missing Version"
+name: "⏰ Release: Highest Missing Version"

 on:
  schedule:
@@ -0,0 +1,42 @@
+name: "⏰ Rerun: Cancelled Branch Runs"
+
+concurrency:
+  group: rerun-cancelled
+  cancel-in-progress: true
+
+on:
+  schedule:
+    - cron: "0 * * * *"
+  workflow_dispatch:
+
+permissions:
+  actions: write
+  contents: read
+
+jobs:
+  rerun:
+    name: 🔁 Rerun the failed jobs of cancelled runs
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - name: ⬇️ Checkout repository
+        uses: actions/checkout@v7.0.0
+
+      - name: ⏱️ Decide whether this hour is due
+        id: due
+        if: ${{ github.event_name == 'schedule' }}
+        shell: bash
+        env:
+          INTERVAL_MINUTES: ${{ vars.CI_RERUN_INTERVAL_MINUTES || '60' }}
+        run: bash scripts/github/rerun/due.sh
+
+      - name: 🔁 Restart the failed jobs per branch
+        if: ${{ steps.due.outputs.due != 'false' }}
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPOSITORY: ${{ github.repository }}
+          MAX_ATTEMPTS: ${{ vars.CI_RERUN_MAX_ATTEMPTS || '0' }}
+          MAX_AGE_HOURS: ${{ vars.CI_RERUN_MAX_AGE_HOURS || '24' }}
+        run: bash scripts/github/rerun/cancelled_branch_runs.sh
@@ -1,3 +1,5 @@
+# nocheck: comments-valid  vendored CodeQL starter workflow; upstream comments kept verbatim
+#
 # For most projects, this workflow file will not need changing; you simply need
 # to commit it to your repository.
 #
@@ -9,7 +11,7 @@
 # the `language` matrix defined below to confirm you have the correct set of
 # supported CodeQL languages.
 #
-name: "🔒 Scan: CodeQL (Advanced)"
+name: "⏰ Scan: CodeQL (Advanced)"

 concurrency:
  group: codeql-${{ github.repository }}-${{ github.ref }}
@@ -3,7 +3,7 @@
 # policy, and support documentation.
 # nocheck: comments-valid  vendored OSSF scorecard workflow; upstream comments kept verbatim

-name: 🛡️ Scorecard supply-chain security
+name: "⏰ Scorecard supply-chain security"
 on:
  # For Branch-Protection check. Only the default branch is supported. See
  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
@@ -1,4 +1,4 @@
-name: "🔄 Update: Versions"
+name: "⏰ Update: Versions"

 on:
  push:
@@ -1,14 +1,15 @@
+# Enters the same concurrency group as entry-push-latest.yml for the deleted
+# branch, which cancels that run and all its child workflow_call runs.
+#
+# On `delete` events `github.ref_name` resolves to the default branch, not the
+# deleted ref, so the group keys on `github.event.ref`. Keying on `ref_name`
+# would collide with the push-to-main CI and cancel it.
+
 name: "🚫 Cancel: Runs on Branch Delete"

 on:
  delete:

-# Enters the same concurrency group as entry-push-latest.yml for the deleted
-# branch, which cancels that run and all its child workflow_call runs.
-# NOTE: on `delete` events, `github.ref_name` resolves to the default branch
-# (the workflow runs from the default branch context), not the deleted ref.
-# Use `github.event.ref` to get the actually deleted branch name — otherwise
-# this group would collide with the push-to-main CI and cancel it.
 concurrency:
  group: global-ci-${{ github.repository }}-${{ github.event.ref }}
  cancel-in-progress: true
@@ -18,7 +19,6 @@ permissions:
  contents: read

 jobs:
-  # Fallback: cancel any remaining active runs via the GitHub API.
  cancel-branch-runs:
    if: ${{ github.event.ref_type == 'branch' }}
    runs-on: ubuntu-latest
@@ -0,0 +1,118 @@
+name: "🕹️ Manual"
+run-name: "🕹️ ${{ inputs.mode_fail_fast && '🛑 ' || '' }}${{ inputs.test_migration && '📧迁 ' || '' }}${{ inputs.sequencing != 'auto' && format('🐛{0} ', inputs.sequencing == 'serial' && '序' || '并') || '' }}${{ inputs.workspace != 'auto' && format('🧑‍💻{0} ', inputs.workspace == 'true' && '是' || '否') || '' }}${{ inputs.instructions != 'auto' && format('📖{0} ', inputs.instructions == 'true' && '是' || '否') || '' }}${{ inputs.filesystem != '' && format('🗄️{0} ', inputs.filesystem) || '' }}${{ inputs.distros != '' && format('🐧{0} ', inputs.distros) || '' }}${{ inputs.lifecycles != '' && format('♻️{0} ', inputs.lifecycles) || '' }}${{ inputs.modes != 'swarm compose host' && format('🚀{0} ', inputs.modes) || '' }}${{ inputs.priority != '' && format('⭐{0} ', inputs.priority) || '' }}${{ inputs.whitelist != '' && format('🎯{0}', inputs.whitelist) || '🔀 diff (origin/main)' }}"
+
+concurrency:
+  group: global-ci-${{ github.repository }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+    inputs:
+      distros:
+        description: "🐧 Distros to sweep. Empty: one at random. Example: 'arch debian'"
+        required: false
+        default: ""
+        type: string
+      priority:
+        description: "⭐ Roles to deploy first. The rest waits until they are green."
+        required: false
+        default: ""
+        type: string
+      whitelist:
+        description: "🎯 Roles to deploy. Empty: diff vs origin/main. '__ALL__': every role."
+        required: false
+        default: ""
+        type: string
+      lifecycles:
+        description: "♻️ Lifecycle envelope for test-deploy discovery (comma/space-separated)."
+        required: false
+        default: ""
+        type: string
+      modes:
+        description: "🚀 Deploy modes to test. Only the listed ones run their deploy job."
+        required: false
+        default: "swarm compose host"
+        type: string
+      filesystem:
+        description: "🗄️ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three."
+        required: false
+        default: ""
+        type: string
+      sequencing:
+        description: "🐛 Deploy-mode order per line. 'serial': swarm, then compose, then host. 'parallel': all at once. 'auto': serial above 65 jobs per line."
+        required: false
+        default: "auto"
+        type: choice
+        options:
+          - "auto"
+          - "serial"
+          - "parallel"
+      mode_fail_fast:
+        description: "🛑 Stop a serial line at its first failed mode. Off: keep deploying the remaining modes."
+        required: false
+        default: true
+        type: boolean
+      workspace:
+        description: "🧑‍💻 Run the workspace test (dev-runtime image matrix). 'auto' runs it on a global scope only."
+        required: false
+        default: "auto"
+        type: choice
+        options:
+          - "auto"
+          - "true"
+          - "false"
+      instructions:
+        description: "📖 Run the instructions test (role README Production command). 'auto' runs it on a global scope only."
+        required: false
+        default: "auto"
+        type: choice
+        options:
+          - "auto"
+          - "true"
+          - "false"
+      test_migration:
+        description: "📧 Run the opt-in mail-provider migration scenario (Mailu → Stalwart)."
+        required: false
+        default: false
+        type: boolean
+
+permissions:
+  contents: read
+  packages: write
+  security-events: write
+  actions: write
+  issues: write
+
+jobs:
+  pick-distro:
+    name: 🎲 Pick distro(s)
+    runs-on: ubuntu-latest
+    outputs:
+      distros: ${{ steps.pick.outputs.distros }}
+    steps:
+      - uses: actions/checkout@v7.0.0
+
+      - id: pick
+        shell: bash
+        env:
+          INPUT_DISTROS: ${{ inputs.distros }}
+        run: ./scripts/github/resolve/manual_distros.sh
+
+  ci-orchestrator:
+    name: 🎛️ Orchestrate CI (manual)
+    needs: pick-distro
+    uses: ./.github/workflows/call-orchestrator.yml
+    with:
+      distros: ${{ needs.pick-distro.outputs.distros }}
+      whitelist: ${{ inputs.whitelist }}
+      lifecycles: ${{ inputs.lifecycles }}
+      modes: ${{ inputs.modes }}
+      priority: ${{ inputs.priority }}
+      sequencing: ${{ inputs.sequencing }}
+      mode_fail_fast: ${{ inputs.mode_fail_fast }}
+      filesystem: ${{ inputs.filesystem }}
+      workspace: ${{ inputs.workspace }}
+      instructions: ${{ inputs.instructions }}
+      test_migration: ${{ inputs.test_migration && 'true' || 'false' }}
+      force_cancel_in_progress: true
+    secrets: inherit
@@ -1,61 +0,0 @@
-name: "⚡ CI: Manual"
-run-name: "Manual CI triggered: ${{ inputs.distros }}; ${{ inputs.priority != '' && format('{0} ⭐; ', inputs.priority) || '' }}${{ inputs.whitelist != '' && inputs.whitelist || 'diff-derived (origin/main)' }}"
-
-concurrency:
-  group: global-ci-${{ github.repository }}-${{ github.ref_name }}
-  cancel-in-progress: true
-
-on:
-  workflow_dispatch:
-    inputs:
-      distros:
-        description: "Space-separated distro list (overrides default). Example: 'arch debian'"
-        required: false
-        default: "debian"
-        type: string
-      whitelist:
-        description: "Empty: diff vs origin/main. '__ALL__': force full deploy. Else: explicit space-separated app ids."
-        required: false
-        default: ""
-        type: string
-      test_migration:
-        description: "Run the opt-in mail-provider migration scenario (Mailu → Stalwart)."
-        required: false
-        default: false
-        type: boolean
-      lifecycles:
-        description: "Lifecycle envelope for test-deploy discovery (comma/space-separated)."
-        required: false
-        default: ""
-        type: string
-      modes:
-        description: "Deploy modes to test (space-separated: 'swarm compose host'). Only the listed modes run their deploy job."
-        required: false
-        default: "swarm compose host"
-        type: string
-      priority:
-        description: "Optional space-separated app ids for the priority line: deployed first in every mode; the regular line runs the remaining roles only after every priority job succeeded."
-        required: false
-        default: ""
-        type: string
-
-permissions:
-  contents: read
-  packages: write
-  security-events: write
-  actions: write
-  issues: write
-
-jobs:
-  ci-orchestrator:
-    name: 🎛️ Orchestrate CI (manual)
-    uses: ./.github/workflows/ci-orchestrator.yml
-    with:
-      distros: ${{ inputs.distros }}
-      whitelist: ${{ inputs.whitelist }}
-      lifecycles: ${{ inputs.lifecycles }}
-      modes: ${{ inputs.modes }}
-      priority: ${{ inputs.priority }}
-      roles_only: ${{ (inputs.priority != '' || (inputs.whitelist != '' && inputs.whitelist != '__ALL__')) && 'true' || 'false' }}
-      test_migration: ${{ inputs.test_migration && 'true' || 'false' }}
-    secrets: inherit
@@ -1,4 +1,4 @@
-name: "⚡ CI: Pull Request"
+name: "🔀 Pull Request"
 run-name: >-
  ${{ github.event_name }} / PR #${{ github.event.pull_request.number }} / ${{ github.event.action }} / ${{ github.event.pull_request.head.label }}

@@ -102,7 +102,6 @@ jobs:
    runs-on: ubuntu-latest
    outputs:
      whitelist: ${{ steps.resolve.outputs.whitelist }}
-      roles_only: ${{ steps.resolve.outputs.roles_only }}
    steps:
      - name: ⬇️ Checkout
        uses: actions/checkout@v7.0.0
@@ -134,11 +133,16 @@ jobs:
      - validate-pr-branch-prefix
      - detect-affected-roles
    if: ${{ github.event_name == 'pull_request' && needs.detect-pr-scope.outputs.run_ci_orchestrator == 'true' }}
-    uses: ./.github/workflows/ci-orchestrator.yml
+    permissions:
+      actions: write
+      contents: read
+      packages: write
+      security-events: write
+      issues: write
+    uses: ./.github/workflows/call-orchestrator.yml
    with:
      distros: "arch debian ubuntu fedora centos"
      whitelist: ${{ needs.detect-affected-roles.outputs.whitelist }}
-      roles_only: ${{ needs.detect-affected-roles.outputs.roles_only }}
      trusted_pr: ${{ contains(github.event.pull_request.labels.*.name, '🛡️ Trusted') }}

  prepare-fork-merge-ref:
@@ -164,7 +168,7 @@ jobs:

  build-ci-images-fork:
    needs: prepare-fork-merge-ref
-    uses: ./.github/workflows/images-build-ci.yml
+    uses: ./.github/workflows/call-images-build-ci.yml
    with:
      distros: "arch debian ubuntu fedora centos"
      checkout_ref: ${{ needs.prepare-fork-merge-ref.outputs.trusted == 'true' && github.event.pull_request.head.sha || needs.prepare-fork-merge-ref.outputs.checkout_ref }}
@@ -175,7 +179,7 @@ jobs:

  images-mirror-missing-fork:
    needs: prepare-fork-merge-ref
-    uses: ./.github/workflows/images-mirror-missing.yml
+    uses: ./.github/workflows/call-images-mirror-missing.yml
    with:
      repo_root: _pr_source
      source_repository: ${{ github.repository }}
@@ -1,5 +1,3 @@
-name: "🚫 Dependabot: Close while auto-updates disabled"
-
 # Closes Dependabot PRs while update PRs are gated off via the
 # CI_ENABLE_AUTO_UPDATES repository variable. Dependabot itself cannot read
 # repository variables, so the gate is enforced here.
@@ -8,6 +6,8 @@ name: "🚫 Dependabot: Close while auto-updates disabled"
 # access to repository vars/secrets and a write-scoped GITHUB_TOKEN
 # even for Dependabot's PRs. This workflow does NOT check out PR code.

+name: "🚫 Dependabot: Close while auto-updates disabled"
+
 on:
  pull_request_target:
    types: [opened, reopened]
@@ -25,4 +25,4 @@ jobs:
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_URL: ${{ github.event.pull_request.html_url }}
-        run: gh pr close "$PR_URL" --comment "Auto-closed by \`dependabot-close.yml\`. The repository variable \`CI_ENABLE_AUTO_UPDATES\` is not set to \`true\`, so update PRs are disabled. Set it to \`true\` under **Settings → Secrets and variables → Actions → Variables** to allow Dependabot PRs."
+        run: gh pr close "$PR_URL" --comment "Auto-closed by \`entry-pr-open-dependabot-close.yml\`. The repository variable \`CI_ENABLE_AUTO_UPDATES\` is not set to \`true\`, so update PRs are disabled. Set it to \`true\` under **Settings → Secrets and variables → Actions → Variables** to allow Dependabot PRs."
@@ -1,4 +1,4 @@
-name: "⚡ CI: Push"
+name: "📤 Push"

 concurrency:
  group: global-ci-${{ github.repository }}-${{ github.ref_name }}
@@ -129,7 +129,7 @@ jobs:
      - run-policy
      - pick-distro
      - resolve-priority
-    uses: ./.github/workflows/ci-orchestrator.yml
+    uses: ./.github/workflows/call-orchestrator.yml
    with:
      distros: ${{ needs.pick-distro.outputs.distros }}
      priority: ${{ needs.resolve-priority.outputs.priority }}
@@ -172,7 +172,7 @@ jobs:
      - pick-distro
      - release-guard
    if: needs.run-policy.outputs.should_run == 'true' && needs.release-guard.outputs.allowed == 'true'
-    uses: ./.github/workflows/release-version.yml
+    uses: ./.github/workflows/call-release-version.yml
    with:
      tag: ${{ needs.pick-distro.outputs.version_tag }}
    secrets: inherit
@@ -1,87 +0,0 @@
-name: "💻 Test: Development Environment"
-
-on:
-  workflow_dispatch:
-  workflow_call:
-
-permissions:
-  actions: write
-  contents: read
-  packages: read
-
-jobs:
-  test-environment:
-    name: 🧑‍💻 Environment (${{ matrix.dev_runtime_image }})
-    runs-on: ubuntu-latest
-    timeout-minutes: 345
-
-    strategy:
-      fail-fast: false
-      matrix:
-        dev_runtime_image:
-          - manjarolinux/base
-          - debian:bookworm
-          - ubuntu:latest
-          - fedora:latest
-          - quay.io/centos/centos:latest
-
-    env:
-      DEV_RUNTIME_IMAGE: ${{ matrix.dev_runtime_image }}
-      NIX_CONFIG: |
-        access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
-      INFINITO_CACHE_REGISTRY_MAX_SIZE: 16g
-      INFINITO_CACHE_REGISTRY_HOST_PATH: /mnt/cache/registry/mirror
-      INFINITO_CACHE_REGISTRY_CA_HOST_PATH: /mnt/cache/registry/ca
-      INFINITO_CACHE_PACKAGE_BLOBSTORE_MAX: 8g
-      INFINITO_CACHE_PACKAGE_HEAP: 1g
-      INFINITO_CACHE_PACKAGE_DIRECT_MEM: 1g
-      INFINITO_CACHE_PACKAGE_HOST_PATH: /mnt/cache/package/data
-
-    steps:
-      - name: 🧹 Free disk space
-        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
-        with:
-          tool-cache: true
-          android: true
-          dotnet: true
-          haskell: true
-          large-packages: true
-          docker-images: true
-          swap-storage: false
-
-      - name: ⬇️ Checkout repository
-        uses: actions/checkout@v7.0.0
-
-      - name: ▶️ Run dev environment routine
-        shell: bash
-        run: |
-          set -euo pipefail
-          HOST_WORKDIR="$(pwd -P)"
-
-          docker run --rm \
-            -e GITHUB_ACTIONS=true \
-            -e GITHUB_REPOSITORY="${{ github.repository }}" \
-            -e GITHUB_REPOSITORY_OWNER="${{ github.repository_owner }}" \
-            -e GITHUB_SHA="${{ github.sha }}" \
-            -e DEV_RUNTIME_IMAGE \
-            -e INFINITO_DISTRO=debian \
-            -e INFINITO_IMAGE=infinito-debian \
-            -e INFINITO_PULL_POLICY=never \
-            -e INFINITO_BUILD=1 \
-            -e NIX_CONFIG \
-            -e INFINITO_CACHE_REGISTRY_MAX_SIZE \
-            -e INFINITO_CACHE_REGISTRY_HOST_PATH \
-            -e INFINITO_CACHE_REGISTRY_CA_HOST_PATH \
-            -e INFINITO_CACHE_PACKAGE_BLOBSTORE_MAX \
-            -e INFINITO_CACHE_PACKAGE_HEAP \
-            -e INFINITO_CACHE_PACKAGE_DIRECT_MEM \
-            -e INFINITO_CACHE_PACKAGE_HOST_PATH \
-            --network host \
-            -v "${HOST_WORKDIR}:${HOST_WORKDIR}" \
-            -v "/mnt/docker:/var/lib/docker" \
-            -v "/var/run/docker.sock:/var/run/docker.sock" \
-            -w "${HOST_WORKDIR}" \
-            "${DEV_RUNTIME_IMAGE}" \
-            bash -lc '
-              bash ./scripts/tests/environment/00_orchestrator.sh
-            '