Merge upstream/main into feature/role-stalwart-mail-swarm

.claude/settings.json

@@ -39,6 +39,7 @@
       "WebFetch(domain:api.openai.com)",
       "WebFetch(domain:api.telegram.org)",
       "WebFetch(domain:apps.odoo.com)",
+      "WebFetch(domain:archive.archlinux.org)",
       "WebFetch(domain:archive.ubuntu.com)",
       "WebFetch(domain:archlinux.org)",
       "WebFetch(domain:askubuntu.com)",
@@ -142,6 +143,7 @@
       "WebFetch(domain:docs.moodle.org)",
       "WebFetch(domain:docs.n8n.io)",
       "WebFetch(domain:docs.nextcloud.com)",
+      "WebFetch(domain:docs.openclaw.ai)",
       "WebFetch(domain:docs.opencloud.eu)",
       "WebFetch(domain:docs.opentalk.eu)",
       "WebFetch(domain:docs.openwebui.com)",
@@ -188,6 +190,7 @@
       "WebFetch(domain:fvonline-db.bplaced.net)",
       "WebFetch(domain:gcc.gnu.org)",
       "WebFetch(domain:geminicli.com)",
+      "WebFetch(domain:geo.mirror.pkgbuild.com)",
       "WebFetch(domain:gergap.de)",
       "WebFetch(domain:gergap.de)",
       "WebFetch(domain:getcomposer.org)",
@@ -206,9 +209,11 @@
       "WebFetch(domain:gitlab.opencode.de)",
       "WebFetch(domain:gohugo.io)",
       "WebFetch(domain:grep.app)",
+      "WebFetch(domain:gvisor.dev)",
       "WebFetch(domain:help.penpot.app)",
       "WebFetch(domain:help.ubuntu.com)",
       "WebFetch(domain:helpcenter.onlyoffice.com)",
+      "WebFetch(domain:hermes-agent.nousresearch.com)",
       "WebFetch(domain:hexdocs.pm)",
       "WebFetch(domain:hluk.github.io)",
       "WebFetch(domain:hub.docker.com)",
@@ -246,6 +251,8 @@
       "WebFetch(domain:memcached.org)",
       "WebFetch(domain:meta.discourse.org)",
       "WebFetch(domain:micro-editor.github.io)",
+      "WebFetch(domain:mirror.manjaro.org)",
+      "WebFetch(domain:mirror.stream.centos.org)",
       "WebFetch(domain:mobilizon.org)",
       "WebFetch(domain:moodle.org)",
       "WebFetch(domain:moodledev.io)",
@@ -279,12 +286,15 @@
       "WebFetch(domain:openwebui.com)",
       "WebFetch(domain:owncloud.dev)",
       "WebFetch(domain:packages.debian.org)",
+      "WebFetch(domain:packages.ubuntu.com)",
       "WebFetch(domain:patchwork.kernel.org)",
       "WebFetch(domain:penpot.app)",
       "WebFetch(domain:pi-hole.net)",
       "WebFetch(domain:piehost.com)",
       "WebFetch(domain:pixelfed.org)",
       "WebFetch(domain:pkg.go.dev)",
+      "WebFetch(domain:pkgs.alpinelinux.org)",
+      "WebFetch(domain:pkgs.org)",
       "WebFetch(domain:platform.openai.com)",
       "WebFetch(domain:playwright.dev)",
       "WebFetch(domain:playwright.download.prss.microsoft.com)",
@@ -345,6 +355,7 @@
       "WebFetch(domain:snipe-it.readme.io)",
       "WebFetch(domain:socialhome.network)",
       "WebFetch(domain:spec.matrix.org)",
+      "WebFetch(domain:src.fedoraproject.org)",
       "WebFetch(domain:stackoverflow.com)",
       "WebFetch(domain:storage.googleapis.com)",
       "WebFetch(domain:store.shopware.com)",
@@ -398,6 +409,7 @@
       "WebFetch(domain:www.gnu.org)",
       "WebFetch(domain:www.google.com)",
       "WebFetch(domain:www.googletagmanager.com)",
+      "WebFetch(domain:www.home-assistant.io)",
       "WebFetch(domain:www.hvs-consulting.de)",
       "WebFetch(domain:www.jenkins.io)",
       "WebFetch(domain:www.joomla.org)",
@@ -479,6 +491,8 @@
       "Bash(gh variable *)",
       "Bash(gh workflow disable*)",
       "Bash(gh workflow enable*)",
+      "Bash(git --git-dir*)",
+      "Bash(git --work-tree*)",
       "Bash(git -C *)",
       "Bash(git -c *)",
       "Bash(git branch -D*)",
@@ -629,6 +643,7 @@
         "~/.cache/pip",
         "~/.cache/pre-commit",
         "~/.docker/buildx",
+        "~/.local/share/worktrees",
         "~/.venvs"
       ],
       "denyRead": [

.github/workflows/README.md

@@ -10,8 +10,8 @@ inputs and descriptions: [workflows.md](../../docs/contributing/tools/github/act
 ```mermaid
 flowchart TB
     push["push: main, feature/**, hotfix/**, fix/**"] --> epl["entry-push-latest.yml"]
-    pr["pull_request: opened, synchronize, reopened, ready_for_review"] --> eprc["entry-pull-request-change.yml"]
-    dispatch["workflow_dispatch"] --> eman["entry-manual.yml"]
+    pr["pull_request: opened, synchronize, reopened, ready_for_review"] --> eprc["entry-pr-change-orchestrate.yml"]
+    dispatch["workflow_dispatch"] --> eman["entry-manual-steer.yml"]
 
     epl --> orch["ci-orchestrator.yml"]
     epl -->|"version tag on main"| relv["release-version.yml"]
@@ -21,50 +21,52 @@ flowchart TB
     eman --> orch
 
     subgraph orchestrator["ci-orchestrator.yml jobs"]
-        plan["plan: run-summary table"]
         waitfork["wait-fork-prereq-run"] --> forkready["fork-prereqs-ready"]
 
-        lintwf["lint.yml: make lint + hadolint"] --> qualitygate["code-quality-gate"]
-        testwf["test.yml: make test"] --> qualitygate
-        codeql["security-codeql.yml"] --> qualitygate
-
-        qualitygate --> buildci["build-ci-images: images-build-ci.yml"]
-        buildci --> dns["test-dns.yml"]
+        lintwf["lint.yml: make lint + hadolint"]
+        testwf["test.yml: make test"]
+        codeql["cron-security-codeql.yml"]
+        buildci["build-ci-images: images-build-ci.yml"] --> dns["test-dns.yml"]
         mirror["images-mirror-missing.yml"]
+        seq["sequencing: serial or parallel, per line"]
 
-        dns --> snprio["test-deploy-single-node-priority"]
-        mirror --> snprio
-        buildci --> swarmprio["test-deploy-swarm-priority"]
-        mirror --> swarmprio
+        subgraph prio["priority line, skipped without a priority input"]
+            swarmprio["test-deploy-swarm-priority"]
+            swarmprio -->|"serial"| composeprio["test-deploy-compose-priority"]
+            composeprio -->|"serial"| hostprio["test-deploy-host-priority"]
+            snprio["test-deploy-single-node-priority: parallel"]
+        end
 
-        snprio -->|"all priority jobs green"| snreg["test-deploy-single-node"]
-        swarmprio -->|"all priority jobs green"| snreg
-        snprio -->|"all priority jobs green"| swarmreg["test-deploy-swarm"]
-        swarmprio -->|"all priority jobs green"| swarmreg
-        dns --> snreg
-        mirror --> snreg
-        buildci --> swarmreg
-        mirror --> swarmreg
+        subgraph reg["regular line"]
+            swarmreg["test-deploy-swarm"]
+            swarmreg -->|"serial"| composereg["test-deploy-compose"]
+            composereg -->|"serial"| hostreg["test-deploy-host"]
+            snreg["test-deploy-single-node: parallel"]
+        end
+
+        lintwf --> prio
+        testwf --> prio
+        dns --> prio
+        mirror --> prio
+        seq --> prio
+        prio -->|"all priority jobs green"| reg
 
         swarmreg --> smoke["test-runner-smoke.yml"]
+        prio --> report["report-main-failures"]
+        reg --> report
 
-        qualitygate --> instmake["test-install-make.yml"]
-        qualitygate --> instpkgmgr["test-install-pkgmgr.yml"]
-        instmake --> instgate["test-install-gate"]
-        instpkgmgr --> instgate
-
-        qualitygate --> devenv["test-development: test-environment.yml"]
-        mirror --> devenv
-
-        buildci --> testguide["test-guide.yml"]
+        instmake["test-install-make.yml"]
+        instpkgmgr["test-install-pkgmgr.yml"]
+        mirror --> devenv["test-workspace: test-workspace.yml"]
+        buildci --> testguide["test-instructions.yml"]
         mirror --> testguide
 
-        snprio --> donegate["done"]
-        swarmprio --> donegate
-        snreg --> donegate
-        swarmreg --> donegate
+        prio --> donegate["done"]
+        reg --> donegate
+        seq --> donegate
         smoke --> donegate
-        instgate --> donegate
+        instmake --> donegate
+        instpkgmgr --> donegate
         devenv --> donegate
         testguide --> donegate
     end
@@ -73,21 +75,114 @@ flowchart TB
     snreg --> singlenode
     singlenode --> deploycompose["test-deploy-compose.yml"]
     singlenode --> deployhost["test-deploy-host.yml"]
+    composeprio --> deploycompose
+    composereg --> deploycompose
+    hostprio --> deployhost
+    hostreg --> deployhost
     swarmprio --> deployswarm["test-deploy-swarm.yml"]
     swarmreg --> deployswarm
 ```
 
-`test-deploy-single-node-priority` and `test-deploy-swarm-priority` run only
-when the orchestrator's `priority` input is set; with it empty they are
-skipped and the regular deploy jobs start directly. The regular jobs receive
-the priority ids as `blacklist`, so each role deploys in exactly one line.
+The priority line runs only when the orchestrator's `priority` input is set;
+with it empty every priority job is skipped and the regular line starts
+directly. The regular jobs receive the priority ids as `blacklist`, so each
+role deploys in exactly one line.
+
+## Deploy-line sequencing
+
+A **line** is one half of a run: the priority line (`⭐`, the roles named in
+`priority`) or the regular line (`🔁`, everything else). Each line deploys up
+to three **modes**: swarm, compose, host. The lines are always sequential
+relative to each other; this section is about the order *inside* one line.
+
+### Why the order exists
+
+GitHub cancels a job that has sat queued for 24 hours. That clock starts when
+the job is queued, not when the run starts, and a `needs:` edge delays queueing
+until the dependency completes. A line that starts all three modes at once
+queues its whole matrix in one go, so with more roles than runner slots the
+tail of that matrix waits past the cut and dies unrun. Deploying the modes one
+after the other restarts the clock per mode.
+
+Serialising costs wall-clock, so it is only worth it above a certain size. The
+run does not pay it on a small diff.
+
+### The decision
+
+The `sequencing` job resolves the effective whitelist (the same
+`scripts/github/resolve/effective_whitelist.sh` the discover steps use) and
+calls `cli.meta.ci.sequencing` once per line, emitting `serial` or `parallel`
+as a job output.
+
+Job counts are taken on the row basis each mode actually runs on: swarm
+selections are `role#variant` tokens that map 1:1 onto jobs, compose and host
+selections are whole roles whose variants pack into bundles
+(`utils.github.variant.bundles`). The count deliberately omits the
+runner-storage filter that `scripts/meta/resolve/apps.sh` applies inside
+GitHub Actions, so it can only overestimate — erring towards the sequential
+layout, which cannot be cancelled.
+
+| `sequencing` input | Behaviour |
+|---|---|
+| `auto` (default) | `serial` above `INFINITO_CI_SEQUENTIAL_THRESHOLD` jobs in that line, `parallel` at or below it |
+| `serial` | forced; skips the count entirely |
+| `parallel` | forced; skips the count entirely |
+
+The threshold lives in [`default.env`](../../default.env) as
+`INFINITO_CI_SEQUENTIAL_THRESHOLD`. Both lines are decided independently: a
+small priority line can run parallel while the regular line behind it runs
+serial.
+
+### The two layouts
+
+Serial order is **swarm, then compose, then host** — heaviest mode first, so
+the longest queue drains while the run is youngest.
+
+- **parallel** — `test-deploy-swarm-priority` plus
+  `test-deploy-single-node-priority`, which fans compose and host out
+  together. This is the layout the pipeline had before sequencing existed.
+- **serial** — `test-deploy-swarm-priority` →
+  `test-deploy-compose-priority` → `test-deploy-host-priority`, each chained
+  on the previous through `needs:`.
+
+The regular line mirrors this with the unsuffixed job names.
+
+A `needs:` edge is static, so a workflow cannot make one conditional. Both
+layouts therefore exist as separate jobs and the `sequencing` output skips
+one of them. Because a skipped dependency otherwise skips its dependents, the
+serial jobs carry `always()` plus explicit `needs.<job>.result` checks on the
+real gates. The swarm job depends on no sequencing decision at all: it leads
+in either layout.
+
+### Stopping on failure
+
+`mode_fail_fast` (default `true`) decides whether a serial line stops at its
+first failed mode:
+
+| Value | Behaviour |
+|---|---|
+| `true` | swarm fails → compose and host of that line are skipped |
+| `false` | every mode deploys and reports; the run still ends red |
+
+`skipped` counts as passed, so a mode absent from `modes` never blocks the
+chain. The parallel layout ignores the switch — with no order there is nothing
+to stop. The setting is a checkbox on `entry-manual-steer.yml`; the other entry
+points take the default.
+
+### Job budget
+
+`cli.meta.ci.slots` divides the run's 256-job cap between the deploy
+matrices and reads the orchestrator to do it, so every deploy caller —
+including the serial twins — must be listed in its `_DEPLOY_CALLERS`.
+A caller missing there is charged a guessed dynamic-matrix estimate instead
+of its single discover job.
 
 ## Cancellation
 
 ```mermaid
 flowchart TB
-    prclose["pull_request_target: closed, converted_to_draft"] --> eprcancel["entry-pull-request-cancel.yml"]
-    branchdelete["delete: branch"] --> delbranch["delete-branch.yml"]
+    prclose["pull_request_target: closed, converted_to_draft"] --> eprcancel["entry-pr-closed-cancel-workflows.yml"]
+    branchdelete["delete: branch"] --> delbranch["entry-delete-branch.yml"]
     eprcancel -.->|"cancels concurrency group"| runningci["running entry + child workflow runs"]
     delbranch -.->|"cancels concurrency group"| runningci
 ```
@@ -96,23 +191,23 @@ flowchart TB
 
 ```mermaid
 flowchart TB
-    daily["schedule: daily 00:00 UTC"] --> mirrorall["images-mirror-all.yml"]
-    daily --> stale["cleanup-stale.yml"]
-    daily --> relhighest["release-highest.yml"]
-    weekly["schedule: weekly Sat 00:00 UTC"] --> updatewf["update.yml"]
-    weekly --> cleanupci["images-cleanup-ci.yml"]
-    weeklymon["schedule: weekly Mon 00:00 UTC"] --> scorecard["security-scorecard.yml"]
+    daily["schedule: daily 00:00 UTC"] --> mirrorall["cron-images-mirror-all.yml"]
+    daily --> stale["cron-cleanup-stale.yml"]
+    daily --> relhighest["cron-release-highest.yml"]
+    weekly["schedule: weekly Sat 00:00 UTC"] --> updatewf["cron-update.yml"]
+    weekly --> cleanupci["cron-images-cleanup-ci.yml"]
+    weeklymon["schedule: weekly Mon 00:00 UTC"] --> scorecard["cron-security-scorecard.yml"]
     branchprot["branch_protection_rule"] --> scorecard
     pushmain["push: main"] --> updatewf
-    prtarget["pull_request_target: opened, reopened"] --> depclose["dependabot-close.yml"]
+    prtarget["pull_request_target: opened, reopened"] --> depclose["entry-pr-open-dependabot-close.yml"]
 
     relhighest -.->|"gh workflow run"| relver["release-version.yml"]
     relver --> imgbuildci["images-build-ci.yml"]
     manual["workflow_dispatch"] --> mirrorcleanup["images-mirror-cleanup.yml"]
-    manual --> deploywf["test-guide.yml: run a role README Production command"]
+    manual --> deploywf["test-instructions.yml: run a role README Production command"]
 ```
 
-Also manually dispatchable: `images-mirror-all.yml`, `images-cleanup-ci.yml`,
-`cleanup-stale.yml`, `update.yml`, `release-highest.yml`, `release-version.yml`,
-`lint.yml`, `test.yml`, `test-deploy-swarm.yml`, `test-dns.yml`,
-`test-environment.yml`, `test-runner-smoke.yml`.
+Also manually dispatchable: `cron-images-mirror-all.yml`, `cron-images-cleanup-ci.yml`,
+`cron-cleanup-stale.yml`, `cron-update.yml`, `cron-release-highest.yml`, `call-release-version.yml`,
+`call-lint.yml`, `call-test.yml`, `call-test-deploy-swarm.yml`, `call-test-dns.yml`,
+`test-workspace.yml`, `test-runner-smoke.yml`.

.github/workflows/call-images-build-ci.yml

@@ -43,7 +43,7 @@ concurrency:
 
 jobs:
   prepare:
-    name: 🏷️ Label images
+    name: 📋 Build Matrix
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.out.outputs.matrix }}
@@ -88,17 +88,25 @@ jobs:
         shell: bash
         run: bash scripts/github/runner/disk_pressure_report.sh START
 
+      - name: 🔍 Resolve buildkit image ref
+        id: buildkit
+        shell: bash
+        run: |
+          set -euo pipefail
+          ref="$(bash scripts/meta/resolve/image/service_ref.sh svc-runner buildkit mirror)"
+          echo "ref=${ref}" >> "$GITHUB_OUTPUT"
+
       - name: ⬇️ Pre-pull buildkit image (retry 7x, 20s)
         shell: bash
         env:
           MAX_ATTEMPTS: "7"
           RETRY_DELAY_SECONDS: "20"
-        run: bash scripts/github/runner/pull_with_retry.sh moby/buildkit:buildx-stable-1
+        run: bash scripts/github/runner/pull_with_retry.sh "${{ steps.buildkit.outputs.ref }}"
 
       - name: 🔧 Set up Docker Buildx
         uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
         with:
-          driver-opts: image=moby/buildkit:buildx-stable-1
+          driver-opts: image=${{ steps.buildkit.outputs.ref }}
 
       - name: 🔑 Login to GHCR
         uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0

.github/workflows/call-images-mirror-missing.yml

@@ -116,9 +116,7 @@ jobs:
           cache-dependency-path: pyproject.toml
 
       - name: 📦 Install Python deps
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install PyYAML
+        run: PYTHON=python bash scripts/install/python.sh
 
       - name: 📦 Install skopeo
         run: ./scripts/install/apt.sh skopeo

.github/workflows/call-orchestrator.yml

@@ -0,0 +1,572 @@
+name: "🎵 CI: Orchestrator"
+on:
+  workflow_call:
+    inputs:
+      distros:
+        description: "Space-separated distro list"
+        required: true
+        type: string
+      whitelist:
+        description: "Optional space-separated whitelist of app ids"
+        required: false
+        type: string
+        default: ""
+      lifecycles:
+        description: "Lifecycle envelope for test-deploy discovery (comma/space-separated)."
+        required: false
+        type: string
+        default: ""
+      modes:
+        description: "Deploy modes to run (space-separated: 'swarm compose host'). Only listed modes run their deploy job."
+        required: false
+        type: string
+        default: "swarm compose host"
+      priority:
+        description: "Optional space-separated app ids for the priority line: deployed first in every mode; the regular line runs the remaining roles only after every priority job succeeded."
+        required: false
+        type: string
+        default: ""
+      sequencing:
+        description: "🐛 Deploy-mode order per line. 'serial': swarm, then compose, then host. 'parallel': all at once. 'auto': serial above the job threshold."
+        required: false
+        type: string
+        default: "auto"
+      mode_fail_fast:
+        description: "🛑 Stop a serial line at its first failed mode. Off: keep deploying the remaining modes."
+        required: false
+        type: boolean
+        default: true
+      filesystem:
+        description: "🗄️ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three."
+        required: false
+        type: string
+        default: ""
+      image_wait_attempts:
+        description: "Polling attempts for waiting on prebuilt or mirrored images"
+        required: false
+        type: string
+        default: "1980"
+      image_wait_sleep_seconds:
+        description: "Polling interval in seconds for waiting on prebuilt or mirrored images"
+        required: false
+        type: string
+        default: "10"
+      workspace:
+        description: "Run the workspace test (dev-runtime image matrix). 'auto' runs it on a global scope only, 'true'/'false' force it."
+        required: false
+        type: string
+        default: "auto"
+      instructions:
+        description: "Run the instructions test (role README Production command). 'auto' runs it on a global scope only, 'true'/'false' force it."
+        required: false
+        type: string
+        default: "auto"
+      test_migration:
+        description: "'true' runs the opt-in mail-provider migration scenario (Mailu → Stalwart)."
+        required: false
+        type: string
+        default: "false"
+      trusted_pr:
+        description: "True when the PR carries the maintainer '🛡️ Trusted' label; skips the fork-prereq wait."
+        required: false
+        type: boolean
+        default: false
+      force_cancel_in_progress:
+        description: "When true, a new run always cancels the running orchestrator on the same ref, ignoring the CI_CANCEL_IN_PROGRESS repository variable."
+        required: false
+        type: boolean
+        default: false
+
+concurrency:
+  group: infinito-test-${{ github.ref }}
+  cancel-in-progress: ${{ inputs.force_cancel_in_progress || vars.CI_CANCEL_IN_PROGRESS != 'false' }}
+
+permissions:
+  actions: write
+  contents: read
+  packages: write
+  security-events: write
+  issues: write
+
+jobs:
+  wait-fork-prereq-run:
+    name: ⏳ Wait for fork run
+    runs-on: ubuntu-latest
+    env:
+      WAIT_REQUIRED: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && !inputs.trusted_pr }}
+    steps:
+      - name: 🏷️ Mark fork prerequisites as ready outside fork PRs or for trusted PRs
+        if: ${{ env.WAIT_REQUIRED != 'true' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          echo "Fork prerequisites are implicitly ready (non-fork PR or '🛡️ Trusted' label)."
+
+      - name: ⬇️ Checkout trusted workflow repo (helpers)
+        if: ${{ env.WAIT_REQUIRED == 'true' }}
+        uses: actions/checkout@v7.0.0
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          persist-credentials: false
+
+      - name: 🔑 Login to GHCR for CI image checks
+        if: ${{ env.WAIT_REQUIRED == 'true' }}
+        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: ⏳ Wait for privileged fork CI images
+        if: ${{ env.WAIT_REQUIRED == 'true' }}
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          WORKFLOW_FILE: entry-pr-change-orchestrate.yml
+          TARGET_EVENT: pull_request_target
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          PR_UPDATED_AT: ${{ github.event.pull_request.updated_at }}
+          INFINITO_DISTROS: "arch debian ubuntu fedora centos"
+          IMAGE_TAG: ci-${{ github.sha }}
+          WAIT_ATTEMPTS: ${{ inputs.image_wait_attempts }}
+          WAIT_SLEEP_SECONDS: ${{ inputs.image_wait_sleep_seconds }}
+        run: ./scripts/meta/wait/images.sh
+
+  fork-prereqs-ready:
+    name: 🚦 Gate Dependencies
+    needs: wait-fork-prereq-run
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7.0.0
+      - name: ✅ Assert privileged prerequisites are ready
+        shell: bash
+        env:
+          RESULT: ${{ needs.wait-fork-prereq-run.result }}
+        run: ./scripts/github/gate/fork_prereqs.sh
+
+  security-codeql:
+    uses: ./.github/workflows/cron-security-codeql.yml
+
+  lint:
+    uses: ./.github/workflows/call-lint.yml
+
+  test:
+    uses: ./.github/workflows/call-test.yml
+
+  build-ci-images:
+    uses: ./.github/workflows/call-images-build-ci.yml
+    with:
+      distros: "arch debian ubuntu fedora centos"
+      concurrency_channel: ci-orchestrator-build-consumer
+    secrets: inherit
+  
+  test-dns:
+    needs: 
+      - build-ci-images
+    uses: ./.github/workflows/call-test-dns.yml
+    with:
+      distros: "arch debian ubuntu fedora centos"
+  
+  images-mirror-missing:
+    uses: ./.github/workflows/call-images-mirror-missing.yml
+    with:
+      concurrency_channel: ci-orchestrator-mirror-consumer
+      image_wait_attempts: ${{ inputs.image_wait_attempts }}
+      image_wait_sleep_seconds: ${{ inputs.image_wait_sleep_seconds }}
+    secrets: inherit
+
+  sequencing:
+    name: 🐛 Sequencing
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    outputs:
+      priority: ${{ steps.priority.outputs.sequencing }}
+      regular: ${{ steps.regular.outputs.sequencing }}
+    env:
+      INFINITO_APP_DISCOVERY_RUNNER: host
+      INFINITO_LIFECYCLES: ${{ inputs.lifecycles }}
+      INFINITO_HOST_EXCLUDE_COMPOSE: ${{ contains(inputs.modes, 'compose') && 'true' || 'false' }}
+    steps:
+      - uses: actions/checkout@v7.0.0
+        with:
+          fetch-depth: 0
+
+      - name: 🔧 Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: 📦 Install project (host venv)
+        shell: bash
+        run: make install
+
+      - name: 🔍 Resolve effective whitelist (diff vs origin/main)
+        id: resolve_wl
+        shell: bash
+        env:
+          INPUT_WHITELIST: ${{ inputs.whitelist }}
+        run: ./scripts/github/resolve/effective_whitelist.sh
+
+      - name: ⭐ Decide the priority line
+        id: priority
+        if: ${{ inputs.priority != '' }}
+        shell: bash
+        run: |
+          source scripts/meta/env/load.sh
+          "${PYTHON}" -m cli.meta.ci.sequencing \
+            --modes "${{ inputs.modes }}" \
+            --whitelist "${{ inputs.priority }}" \
+            --lifecycles "${{ inputs.lifecycles }}" \
+            --choice "${{ inputs.sequencing }}" \
+            | tee -a "$GITHUB_OUTPUT"
+
+      - name: 🔁 Decide the regular line
+        id: regular
+        shell: bash
+        run: |
+          source scripts/meta/env/load.sh
+          "${PYTHON}" -m cli.meta.ci.sequencing \
+            --modes "${{ inputs.modes }}" \
+            --whitelist "${{ steps.resolve_wl.outputs.whitelist }}" \
+            --blacklist "${{ inputs.priority }}" \
+            --lifecycles "${{ inputs.lifecycles }}" \
+            --choice "${{ inputs.sequencing }}" \
+            | tee -a "$GITHUB_OUTPUT"
+
+  test-deploy-swarm-priority:
+    name: ⭐ Priority · swarm
+    if: ${{ inputs.priority != '' && contains(inputs.modes, 'swarm') }}
+    needs:
+      - lint
+      - test
+      - build-ci-images
+      - images-mirror-missing
+    uses: ./.github/workflows/call-test-deploy-swarm.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.priority }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      marker: " ⭐"
+
+  test-deploy-single-node-priority:
+    name: ⭐ Priority · single-node
+    if: >-
+      ${{ inputs.priority != ''
+          && needs.sequencing.outputs.priority == 'parallel'
+          && (contains(inputs.modes, 'compose') || contains(inputs.modes, 'host')) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - lint
+      - test
+      - sequencing
+    uses: ./.github/workflows/call-test-deploy-single-node.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.priority }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      modes: ${{ inputs.modes }}
+      marker: " ⭐"
+
+  test-deploy-compose-priority:
+    name: ⭐ Priority · compose
+    if: >-
+      ${{ always()
+          && inputs.priority != ''
+          && needs.sequencing.outputs.priority == 'serial'
+          && contains(inputs.modes, 'compose')
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.test-dns.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && (!inputs.mode_fail_fast
+              || contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result)) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - lint
+      - test
+      - sequencing
+      - test-deploy-swarm-priority
+    uses: ./.github/workflows/call-test-deploy-compose.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.priority }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      marker: " ⭐"
+
+  test-deploy-host-priority:
+    name: ⭐ Priority · host
+    if: >-
+      ${{ always()
+          && inputs.priority != ''
+          && needs.sequencing.outputs.priority == 'serial'
+          && contains(inputs.modes, 'host')
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.test-dns.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && (!inputs.mode_fail_fast
+              || contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - lint
+      - test
+      - sequencing
+      - test-deploy-compose-priority
+    uses: ./.github/workflows/call-test-deploy-host.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.priority }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      marker: " ⭐"
+      exclude_compose_covered: ${{ contains(inputs.modes, 'compose') && 'true' || 'false' }}
+
+  test-deploy-swarm:
+    name: 🔁 Regular · swarm
+    if: >-
+      ${{ always()
+          && contains(inputs.modes, 'swarm')
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.build-ci-images.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) }}
+    needs:
+      - build-ci-images
+      - images-mirror-missing
+      - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
+      - test-deploy-swarm-priority
+      - lint
+      - test
+    uses: ./.github/workflows/call-test-deploy-swarm.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.whitelist }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      blacklist: ${{ inputs.priority }}
+
+  test-deploy-single-node:
+    name: 🔁 Regular · single-node
+    if: >-
+      ${{ always()
+          && needs.sequencing.outputs.regular == 'parallel'
+          && (contains(inputs.modes, 'compose') || contains(inputs.modes, 'host'))
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.test-dns.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - sequencing
+      - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
+      - test-deploy-swarm-priority
+      - lint
+      - test
+    uses: ./.github/workflows/call-test-deploy-single-node.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.whitelist }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      modes: ${{ inputs.modes }}
+      blacklist: ${{ inputs.priority }}
+
+  test-deploy-compose:
+    name: 🔁 Regular · compose
+    if: >-
+      ${{ always()
+          && needs.sequencing.outputs.regular == 'serial'
+          && contains(inputs.modes, 'compose')
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.test-dns.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result)
+          && (!inputs.mode_fail_fast
+              || contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm.result)) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - sequencing
+      - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
+      - test-deploy-swarm-priority
+      - test-deploy-swarm
+      - lint
+      - test
+    uses: ./.github/workflows/call-test-deploy-compose.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.whitelist }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      blacklist: ${{ inputs.priority }}
+
+  test-deploy-host:
+    name: 🔁 Regular · host
+    if: >-
+      ${{ always()
+          && needs.sequencing.outputs.regular == 'serial'
+          && contains(inputs.modes, 'host')
+          && needs.lint.result == 'success'
+          && needs.test.result == 'success'
+          && needs.test-dns.result == 'success'
+          && needs.images-mirror-missing.result == 'success'
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-host-priority.result)
+          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result)
+          && (!inputs.mode_fail_fast
+              || contains(fromJSON('["success","skipped"]'), needs.test-deploy-compose.result)) }}
+    needs:
+      - test-dns
+      - images-mirror-missing
+      - sequencing
+      - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
+      - test-deploy-swarm-priority
+      - test-deploy-compose
+      - lint
+      - test
+    uses: ./.github/workflows/call-test-deploy-host.yml
+    with:
+      distros: ${{ inputs.distros }}
+      whitelist: ${{ inputs.whitelist }}
+      lifecycles: ${{ inputs.lifecycles }}
+      filesystem: ${{ inputs.filesystem }}
+      blacklist: ${{ inputs.priority }}
+      exclude_compose_covered: ${{ contains(inputs.modes, 'compose') && 'true' || 'false' }}
+
+  # Exception: opt-in job — the called workflow gates itself on the forwarded
+  # test_migration input or the INFINITO_TEST_MIGRATION repository variable.
+  test-deploy-migration:
+    needs:
+      - test-dns
+      - images-mirror-missing
+    uses: ./.github/workflows/call-test-deploy-migration.yml
+    with:
+      test_migration: ${{ inputs.test_migration }}
+
+  test-runner-smoke:
+    needs:
+      - test-deploy-swarm
+    if: ${{ always() && vars.CI_SELF_HOSTED_RUNNER_COUNT != '0' && vars.CI_SELF_HOSTED_RUNNER_COUNT != '' && (needs.test-deploy-swarm.result == 'success' || needs.test-deploy-swarm.result == 'failure') }}
+    uses: ./.github/workflows/call-test-runner-smoke.yml
+    with:
+      ref: ${{ github.sha }}
+    secrets: inherit
+
+  test-install-make:
+    uses: ./.github/workflows/call-test-install-make.yml
+  test-install-pkgmgr:
+    uses: ./.github/workflows/call-test-install-pkgmgr.yml
+
+  test-workspace:
+    if: ${{ inputs.workspace == 'true' || (inputs.workspace == 'auto' && (inputs.whitelist == '' || inputs.whitelist == '__ALL__')) }}
+    uses: ./.github/workflows/call-test-workspace.yml
+    needs:
+      - images-mirror-missing
+      - lint
+      - test
+    secrets: inherit
+
+  test-instructions:
+    if: ${{ inputs.instructions == 'true' || (inputs.instructions == 'auto' && (inputs.whitelist == '' || inputs.whitelist == '__ALL__')) }}
+    uses: ./.github/workflows/call-test-instructions.yml
+    needs:
+      - images-mirror-missing
+      - build-ci-images
+      - lint
+      - test
+    with:
+      distros: ${{ inputs.distros }}
+    secrets: inherit
+
+  report-main-failures:
+    name: 🐞 Report main role failures
+    needs:
+      - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
+      - test-deploy-swarm-priority
+      - test-deploy-single-node
+      - test-deploy-compose
+      - test-deploy-host
+      - test-deploy-swarm
+    if: ${{ always() && github.ref == 'refs/heads/main' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      actions: read
+      contents: read
+    steps:
+      - uses: actions/checkout@v7.0.0
+      - name: 🐞 Open or append issues for failed roles
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: python3 -m cli.meta.ci.report_failures --run-id "${{ github.run_id }}" --repo "${{ github.repository }}"
+
+  done:
+    name: 🏁 Finish pipeline
+    needs:
+      - wait-fork-prereq-run
+      - fork-prereqs-ready
+      - build-ci-images
+      - images-mirror-missing
+      - test-dns
+      - test-install-make
+      - test-install-pkgmgr
+      - sequencing
+      - test-deploy-single-node-priority
+      - test-deploy-compose-priority
+      - test-deploy-host-priority
+      - test-deploy-swarm-priority
+      - test-deploy-single-node
+      - test-deploy-compose
+      - test-deploy-host
+      - test-deploy-swarm
+      - test-deploy-migration
+      - test-runner-smoke
+      - test-workspace
+      - test-instructions
+      - test
+      - lint
+      - security-codeql
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: 🛑 Fail if any upstream job failed or was cancelled
+        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
+        run: |
+          echo "Pipeline failed: at least one upstream job failed or was cancelled." >&2
+          exit 1
+      - name: ✅ Done
+        run: echo "All checks passed."

.github/workflows/call-release-version.yml

@@ -102,7 +102,7 @@ jobs:
     name: 🏗️ Build missing CI images
     needs: check-ci-images
     if: needs.check-ci-images.outputs.missing == 'true'
-    uses: ./.github/workflows/images-build-ci.yml
+    uses: ./.github/workflows/call-images-build-ci.yml
     with:
       distros: ${{ needs.check-ci-images.outputs.distros }}
       checkout_ref: ${{ needs.check-ci-images.outputs.version_tag }}
@@ -138,17 +138,25 @@ jobs:
           ref: ${{ github.sha }}
           persist-credentials: false
 
+      - name: 🔍 Resolve buildkit image ref
+        id: buildkit
+        shell: bash
+        run: |
+          set -euo pipefail
+          ref="$(bash scripts/meta/resolve/image/service_ref.sh svc-runner buildkit mirror)"
+          echo "ref=${ref}" >> "$GITHUB_OUTPUT"
+
       - name: ⬇️ Pre-pull buildkit image (retry 7x, 20s)
         shell: bash
         env:
           MAX_ATTEMPTS: "7"
           RETRY_DELAY_SECONDS: "20"
-        run: bash scripts/github/runner/pull_with_retry.sh moby/buildkit:buildx-stable-1
+        run: bash scripts/github/runner/pull_with_retry.sh "${{ steps.buildkit.outputs.ref }}"
 
       - name: 🔧 Set up Docker Buildx
         uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
         with:
-          driver-opts: image=moby/buildkit:buildx-stable-1
+          driver-opts: image=${{ steps.buildkit.outputs.ref }}
 
       - name: 🔑 Login to GHCR
         uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0

.github/workflows/call-test-deploy-compose.yml

@@ -27,6 +27,11 @@ on:
         required: false
         type: string
         default: ""
+      filesystem:
+        description: "🗄️ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three."
+        required: false
+        type: string
+        default: ""
 
 permissions:
   contents: read
@@ -41,6 +46,7 @@ env:
   INFINITO_PLAYWRIGHT_KEEP: ${{ vars.INFINITO_PLAYWRIGHT_KEEP }}
   INFINITO_APP_DISCOVERY_RUNNER: host
   INFINITO_DEPLOY_MODE: compose
+  INFINITO_DOCKER_FILESYSTEM_ALLOWED: ${{ inputs.filesystem }}
   INFINITO_LIFECYCLES: ${{ inputs.lifecycles }}
   INFINITO_BLACKLIST: ${{ inputs.blacklist }}
 jobs:
@@ -108,12 +114,17 @@ jobs:
             >> "$GITHUB_STEP_SUMMARY"
 
   compose:
-    name: 🐳 ${{ matrix.apps }}${{ matrix.variant && format(' {0}',matrix.variant) || '' }}
+    name: 🐳${{ matrix.label }}
     if: needs.discover.outputs.apps != '' && needs.discover.outputs.apps != '[]'
     runs-on: ubuntu-latest
-    timeout-minutes: 345
+    timeout-minutes: 355
     needs: discover
 
+    concurrency:
+      group: "infinito-deploy-compose-${{ matrix.apps }}#${{ matrix.variant_slug }}"
+      cancel-in-progress: false
+      queue: max
+
     strategy:
       fail-fast: false
       matrix:
@@ -157,7 +168,7 @@ jobs:
         run: ./scripts/install/apt.sh jq
 
       - name: 🚀 Deploy app
-        timeout-minutes: 300
+        timeout-minutes: 350
         shell: bash
         env:
           apps: ${{ matrix.apps }}
@@ -190,7 +201,7 @@ jobs:
 
       - name: 🩺 Dump runner state (on failure)
         if: failure()
-        timeout-minutes: 10
+        timeout-minutes: 5
         shell: bash
         run: ./scripts/github/runner/dump_state.sh "post-deploy failure (${{ matrix.apps }})"
 

.github/workflows/call-test-deploy-host.yml

@@ -27,6 +27,11 @@ on:
         required: false
         type: string
         default: ""
+      filesystem:
+        description: "🗄️ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three."
+        required: false
+        type: string
+        default: ""
       exclude_compose_covered:
         description: "Drop roles the same run's compose line already deploys"
         required: false
@@ -46,6 +51,7 @@ env:
   INFINITO_PLAYWRIGHT_KEEP: ${{ vars.INFINITO_PLAYWRIGHT_KEEP }}
   INFINITO_APP_DISCOVERY_RUNNER: host
   INFINITO_DEPLOY_MODE: host
+  INFINITO_DOCKER_FILESYSTEM_ALLOWED: ${{ inputs.filesystem }}
   INFINITO_LIFECYCLES: ${{ inputs.lifecycles }}
   INFINITO_BLACKLIST: ${{ inputs.blacklist }}
   INFINITO_HOST_EXCLUDE_COMPOSE: ${{ inputs.exclude_compose_covered }}
@@ -114,12 +120,17 @@ jobs:
             >> "$GITHUB_STEP_SUMMARY"
 
   host:
-    name: 💻 ${{ matrix.apps }}${{ matrix.variant && format(' {0}',matrix.variant) || '' }}
+    name: 💻${{ matrix.label }}
     if: needs.discover.outputs.apps != '' && needs.discover.outputs.apps != '[]'
     runs-on: ubuntu-latest
-    timeout-minutes: 345
+    timeout-minutes: 355
     needs: discover
 
+    concurrency:
+      group: "infinito-deploy-host-${{ matrix.apps }}#${{ matrix.variant_slug }}"
+      cancel-in-progress: false
+      queue: max
+
     strategy:
       fail-fast: false
       matrix:
@@ -163,7 +174,7 @@ jobs:
         run: ./scripts/install/apt.sh jq
 
       - name: 🚀 Deploy host role
-        timeout-minutes: 300
+        timeout-minutes: 350
         shell: bash
         env:
           apps: ${{ matrix.apps }}
@@ -196,7 +207,7 @@ jobs:
 
       - name: 🩺 Dump runner state (on failure)
         if: failure()
-        timeout-minutes: 10
+        timeout-minutes: 5
         shell: bash
         run: ./scripts/github/runner/dump_state.sh "post-deploy failure (${{ matrix.apps }})"
 

.github/workflows/call-test-deploy-migration.yml

@@ -1,5 +1,3 @@
-name: "📧 Test: Mail-Provider Migration (Mailu → Stalwart)"
-
 # Opt-in migration scenario: deploys Mailu as the ACTIVE provider, then
 # redeploys with the default provider (Stalwart) over kept state — the exact
 # upgrade path of an existing Mailu installation. See
@@ -8,7 +6,16 @@ name: "📧 Test: Mail-Provider Migration (Mailu → Stalwart)"
 # Gated OFF by default (INFINITO_TEST_MIGRATION=false in default.env):
 # enable per run via the manual CI workflow's `test_migration` field, per
 # dispatch of this workflow, or repo-wide via the INFINITO_TEST_MIGRATION
-# GitHub repository variable.
+# GitHub repository variable. The job-level gate lets the dispatch/call input
+# win; the repository variable enables it fleet-wide.
+#
+# Step order: the standalone data-migration test runs first — it seeds a
+# Mailu-layout maildir stump, migrates it into a single pinned Stalwart
+# container and verifies the data with no platform deploy involved, so it
+# fails in ~1 minute when the migration script breaks. The full platform
+# cutover (Mailu-active deploy → default-provider takeover over kept state)
+# runs second.
+name: "📧 Test: Mail-Provider Migration (Mailu → Stalwart)"
 
 on:
   workflow_dispatch:
@@ -39,7 +46,6 @@ env:
 jobs:
   mail-migration:
     name: 🚀 Mailu → Stalwart migration
-    # The gate: dispatch/call input wins; the repo variable enables it fleet-wide.
     if: ${{ inputs.test_migration == true || inputs.test_migration == 'true' || vars.INFINITO_TEST_MIGRATION == 'true' }}
     runs-on: ubuntu-latest
     timeout-minutes: 345
@@ -77,15 +83,10 @@ jobs:
         shell: bash
         run: make install
 
-      # Cheap first: seeds a Mailu-layout maildir stump, migrates it into a
-      # single pinned Stalwart container and verifies the data — no platform
-      # deploy involved (fails in ~1 minute when the migration script breaks).
       - name: 🔬 Standalone data-migration test (Mailu stump → Stalwart)
         shell: bash
         run: make test-migration
 
-      # Full platform cutover second: Mailu-active deploy → default-provider
-      # takeover over kept state.
       - name: 📧 Run the mail-provider migration scenario
         shell: bash
         run: make compose-deploy mode=migration

.github/workflows/call-test-deploy-single-node.yml

@@ -32,6 +32,11 @@ on:
         required: false
         type: string
         default: ""
+      filesystem:
+        description: "🗄️ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three."
+        required: false
+        type: string
+        default: ""
 
 permissions:
   contents: read
@@ -41,22 +46,24 @@ jobs:
   compose:
     name: 📦 Compose
     if: ${{ contains(inputs.modes, 'compose') }}
-    uses: ./.github/workflows/test-deploy-compose.yml
+    uses: ./.github/workflows/call-test-deploy-compose.yml
     with:
       distros: ${{ inputs.distros }}
       whitelist: ${{ inputs.whitelist }}
       lifecycles: ${{ inputs.lifecycles }}
       blacklist: ${{ inputs.blacklist }}
       marker: ${{ inputs.marker }}
+      filesystem: ${{ inputs.filesystem }}
 
   host:
     name: 💻 Host
     if: ${{ contains(inputs.modes, 'host') }}
-    uses: ./.github/workflows/test-deploy-host.yml
+    uses: ./.github/workflows/call-test-deploy-host.yml
     with:
       distros: ${{ inputs.distros }}
       whitelist: ${{ inputs.whitelist }}
       lifecycles: ${{ inputs.lifecycles }}
       blacklist: ${{ inputs.blacklist }}
       marker: ${{ inputs.marker }}
+      filesystem: ${{ inputs.filesystem }}
       exclude_compose_covered: ${{ contains(inputs.modes, 'compose') && 'true' || 'false' }}

.github/workflows/call-test-deploy-swarm.yml

@@ -5,9 +5,8 @@ on:
     inputs:
       distros:
         description: "Space-separated distro list (shown in the discover job name)"
-        required: false
+        required: true
         type: string
-        default: "arch debian ubuntu fedora centos"
       whitelist:
         description: "Optional space-separated whitelist of app ids"
         required: false
@@ -23,6 +22,11 @@ on:
         required: false
         type: string
         default: ""
+      filesystem:
+        description: "🗄️ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three."
+        required: false
+        type: string
+        default: ""
       marker:
         description: "Optional suffix appended to the discover job name (e.g. ' ⭐' for the priority line)"
         required: false
@@ -32,9 +36,8 @@ on:
     inputs:
       distros:
         description: "Space-separated distro list (shown in the discover job name)"
-        required: false
+        required: true
         type: string
-        default: "arch debian ubuntu fedora centos"
       whitelist:
         description: "Optional space-separated whitelist of app ids"
         required: false
@@ -50,18 +53,25 @@ on:
         required: false
         type: string
         default: ""
+      filesystem:
+        description: "🗄️ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three."
+        required: false
+        type: string
+        default: ""
 
 permissions:
   contents: read
   packages: read
 
 env:
+  INFINITO_DISTROS: ${{ inputs.distros }}
   NIX_CONFIG: |
     access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
   INFINITO_IMAGE_TAG: ci-${{ github.sha }}
   INFINITO_GHCR_MIRROR_PREFIX: mirror
   INFINITO_APP_DISCOVERY_RUNNER: host
   INFINITO_DEPLOY_MODE: swarm
+  INFINITO_DOCKER_FILESYSTEM_ALLOWED: ${{ inputs.filesystem }}
   INFINITO_LIFECYCLES: ${{ inputs.lifecycles }}
   INFINITO_BLACKLIST: ${{ inputs.blacklist }}
 
@@ -134,12 +144,17 @@ jobs:
             >> "$GITHUB_STEP_SUMMARY"
 
   swarm:
-    name: "🐝 ${{ matrix.apps }}${{ matrix.variant && format(' {0}',matrix.variant) || '' }}"
+    name: "🐝${{ matrix.label }}"
     if: needs.discover.outputs.apps != '' && needs.discover.outputs.apps != '[]'
     runs-on: ubuntu-latest
-    timeout-minutes: 345
+    timeout-minutes: 355
     needs: discover
 
+    concurrency:
+      group: "infinito-deploy-swarm-${{ matrix.apps }}#${{ matrix.variant_slug }}"
+      cancel-in-progress: false
+      queue: max
+
     strategy:
       fail-fast: false
       matrix:
@@ -204,24 +219,16 @@ jobs:
           source scripts/meta/env/load.sh
           echo "${VENV}/bin" >> "${GITHUB_PATH}"
 
-      - name: "🥾 Bring up + bootstrap the simulated cluster (compose up, IPs, lab DNS)"
-        run: ${{ env.SCRIPTS }}/01_bootstrap.sh
-
-      - name: "🔧 Project setup (.dockerignore + dotenv + scripts/setup.sh)"
-        run: make setup
-
-      - name: "🚀 Matrix-deploy ${{ matrix.apps }}: provision/deploy/e2e/verify per round"
-        timeout-minutes: 345
+      - name: "🚀 Drill ${{ matrix.apps }} per distro: ${{ inputs.distros }}"
         shell: bash
         env:
           variant: ${{ matrix.variant }}
-        run: |
-          source scripts/meta/env/load.sh
-          if [ "$(id -u)" -eq 0 ]; then
-            python3 -m utils.tests.swarm.matrix 2>&1 | tee "/tmp/ansible-${{ matrix.apps }}.log"
-          else
-            sudo -E env "PATH=$PATH" "HOME=$HOME" "SWARM_NAME=${SWARM_NAME:-}" "variant=${variant:-}" "disable=${disable:-}" python3 -m utils.tests.swarm.matrix 2>&1 | tee "/tmp/ansible-${{ matrix.apps }}.log"
-          fi
+        run: ./scripts/tests/deploy/distros.sh "${{ env.SCRIPTS }}/00_one.sh" 2>&1 | tee "/tmp/ansible-${{ matrix.apps }}.log"
+
+      - name: "🗄️ Effective node filesystems"
+        if: always()
+        shell: bash
+        run: ./scripts/tests/deploy/utils/filesystem/report.sh "/tmp/ansible-${{ matrix.apps }}.log"
 
       - name: "📊 Role runtimes"
         if: always()
@@ -246,24 +253,6 @@ jobs:
           path: /tmp/role-runtimes-${{ matrix.apps }}.csv
           if-no-files-found: ignore
 
-      - name: "🌱 Seed content (pre-drain marker on NFS volume)"
-        run: ${{ env.SCRIPTS }}/05_seed_content.sh
-
-      - name: "💪 Drain the worker running ${{ matrix.apps }} + force reschedule"
-        run: ${{ env.SCRIPTS }}/06_drain_worker.sh
-
-      - name: "✅ Assert state survived the reschedule"
-        run: ${{ env.SCRIPTS }}/07_assert_state.sh
-
-      - name: "🩺 Collect rescue diagnostics (recursive DiD snapshot)"
-        if: failure()
-        timeout-minutes: 25
-        shell: bash
-        run: |
-          INFINITO_RESCUE_DIAGNOSTICS_DIR="${INFINITO_RESCUE_DIAGNOSTICS_BASE}/${{ matrix.apps }}" \
-            python3 utils/diagnostics/container.py \
-            "${{ matrix.apps }}" "post-deploy failure" || true
-
       - name: "⬆️ Upload rescue diagnostics"
         if: ${{ always() && !env.ACT }}
         uses: actions/upload-artifact@v7
@@ -272,11 +261,6 @@ jobs:
           path: ${{ env.INFINITO_RESCUE_DIAGNOSTICS_BASE }}/
           if-no-files-found: ignore
 
-      - name: "📊 Collect diagnostics on failure"
-        if: failure()
-        timeout-minutes: 15
-        run: scripts/tests/deploy/swarm/utils/collect/diagnostics.sh
-
       - name: "🗜️ Zip generated inventory for debugging"
         if: always()
         shell: bash
@@ -292,19 +276,6 @@ jobs:
           path: /tmp/inventory-swarm-${{ matrix.apps }}.zip
           if-no-files-found: ignore
 
-      - name: "📊 Collect Playwright reports from the manager"
-        if: always()
-        timeout-minutes: 15
-        shell: bash
-        env:
-          APP_ID: ${{ matrix.apps }}
-        run: scripts/tests/deploy/swarm/utils/collect/playwright_reports.sh
-
-      - name: "🕸️ Swarm structure summary"
-        if: always()
-        shell: bash
-        run: scripts/tests/deploy/swarm/utils/collect/topology_summary.sh
-
       - name: "📋 Playwright step summary"
         if: always()
         shell: bash

.github/workflows/call-test-instructions.yml

@@ -1,8 +1,12 @@
-name: "📖 Test: Guide"
+name: "📖 Test: Instructions"
 
 on:
   workflow_call:
     inputs:
+      distros:
+        description: "Space-separated distro list"
+        required: true
+        type: string
       role:
         description: "Role to test; empty picks one at random"
         required: false
@@ -20,6 +24,10 @@ on:
         default: ""
   workflow_dispatch:
     inputs:
+      distros:
+        description: "Space-separated distro list"
+        required: true
+        type: string
       role:
         description: "Role to test; empty picks one at random"
         required: false
@@ -42,12 +50,16 @@ permissions:
 
 jobs:
   deploy:
-    name: 📖 Guide
+    name: 📖 Instructions
     runs-on: ubuntu-latest
-    timeout-minutes: 90
+    timeout-minutes: 345
+    concurrency:
+      group: infinito-instructions
+      cancel-in-progress: false
+      queue: max
     env:
       INFINITO_IMAGE_TAG: ci-${{ github.sha }}
-      INFINITO_DISTROS: "arch debian ubuntu fedora centos"
+      INFINITO_DISTROS: ${{ inputs.distros }}
     steps:
       - if: ${{ !env.ACT }}
         uses: actions/checkout@v7.0.0
@@ -56,14 +68,14 @@ jobs:
         shell: bash
         run: grep '^INFINITO_RESCUE_DIAGNOSTICS_BASE=' default.env >> "$GITHUB_ENV"
 
-      - name: 🐍 Install guide + compose deps (Act runner only)
+      - name: 🐍 Install instructions + compose deps (Act runner only)
         if: ${{ env.ACT }}
         shell: bash
         run: |
-          pip install --quiet --break-system-packages pyyaml
+          PYTHON=python3 PIP_BREAK_SYSTEM_PACKAGES=1 bash scripts/install/python.sh
           apt-get update -qq && apt-get install -y -qq gettext-base jq
 
-      - name: 🎲 Select the role, distro, and deploy mode
+      - name: 🎲 Select the role and deploy mode
         shell: bash
         run: |
           set -euo pipefail
@@ -80,39 +92,14 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: 🖥️ Install a host role inside its pkgmgr distro container
-        if: ${{ env.GUIDE_MODE == 'host' }}
+      - name: 📖 Replay the instructions across ${{ inputs.distros }}
         shell: bash
-        run: bash scripts/github/guide/host_deploy.sh
-
-      - name: 🖥️ Boot the systemd server for a compose role
-        if: ${{ env.GUIDE_MODE == 'compose' }}
-        shell: bash
-        run: |
-          set -euo pipefail
-          INFINITO_IMAGE="ghcr.io/$(scripts/meta/resolve/repository/owner.sh)/$(scripts/meta/resolve/repository/name.sh)/${INFINITO_DISTRO}:ci-${{ github.sha }}"
-          echo "INFINITO_IMAGE=${INFINITO_IMAGE}" >> "$GITHUB_ENV"
-          export INFINITO_IMAGE
-          make compose-up
-
-      - name: 🚀 Deploy the compose role against the server
-        if: ${{ env.GUIDE_MODE == 'compose' }}
-        shell: bash
-        run: bash scripts/github/guide/compose_deploy.sh
-
-      - name: 🩺 Collect rescue diagnostics (recursive DiD snapshot)
-        if: failure()
-        timeout-minutes: 25
-        shell: bash
-        run: |
-          INFINITO_RESCUE_DIAGNOSTICS_DIR="${INFINITO_RESCUE_DIAGNOSTICS_BASE}/${GUIDE_ROLE}" \
-            python3 utils/diagnostics/container.py \
-            "${GUIDE_ROLE}" "guide ${GUIDE_MODE} post-deploy failure" || true
+        run: ./scripts/tests/deploy/distros.sh scripts/github/guide/one.sh
 
       - name: ⬆️ Upload rescue diagnostics
         if: ${{ always() && !env.ACT }}
         uses: actions/upload-artifact@v7
         with:
-          name: rescue-diagnostics-guide-${{ env.GUIDE_ROLE }}
+          name: rescue-diagnostics-instructions-${{ env.GUIDE_ROLE }}
           path: ${{ env.INFINITO_RESCUE_DIAGNOSTICS_BASE }}/
           if-no-files-found: ignore

.github/workflows/call-test-workspace.yml

@@ -0,0 +1,143 @@
+name: "💻 Test: Workspace"
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+permissions:
+  actions: write
+  contents: read
+  packages: read
+
+jobs:
+  prepare:
+    name: 📋 Workspace Matrix
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      images: ${{ steps.matrix.outputs.images }}
+
+    steps:
+      - name: ⬇️ Checkout repository
+        uses: actions/checkout@v7.0.0
+
+      - name: 🐍 Read the distro SPOT
+        id: matrix
+        shell: bash
+        run: |
+          set -euo pipefail
+          images="$(python3 -m cli.meta.ci.dev_runtime images)"
+          echo "images=${images}" | tee -a "$GITHUB_OUTPUT"
+
+  test-workspace:
+    name: 🧑‍💻 Workspace (${{ matrix.dev_runtime_image }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 345
+    needs: prepare
+
+    concurrency:
+      group: infinito-workspace-${{ matrix.dev_runtime_image }}
+      cancel-in-progress: false
+      queue: max
+
+    strategy:
+      fail-fast: false
+      matrix:
+        dev_runtime_image: ${{ fromJson(needs.prepare.outputs.images) }}
+
+    env:
+      DEV_RUNTIME_IMAGE: ${{ matrix.dev_runtime_image }}
+      NIX_CONFIG: |
+        access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+      INFINITO_CACHE_REGISTRY_MAX_SIZE: 16g
+      INFINITO_CACHE_REGISTRY_HOST_PATH: /mnt/cache/registry/mirror
+      INFINITO_CACHE_REGISTRY_CA_HOST_PATH: /mnt/cache/registry/ca
+      INFINITO_CACHE_PACKAGE_BLOBSTORE_MAX: 8g
+      INFINITO_CACHE_PACKAGE_HEAP: 1g
+      INFINITO_CACHE_PACKAGE_DIRECT_MEM: 1g
+      INFINITO_CACHE_PACKAGE_HOST_PATH: /mnt/cache/package/data
+
+    steps:
+      - name: 🧹 Free disk space
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+        with:
+          tool-cache: true
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: false
+
+      - name: ⬇️ Checkout repository
+        uses: actions/checkout@v7.0.0
+
+      - name: "📥 Load the rescue paths into job env"
+        shell: bash
+        run: |
+          set -euo pipefail
+          {
+            grep '^INFINITO_RESCUE_DIAGNOSTICS_BASE=' default.env
+            echo "INFINITO_WORKSPACE_SLUG=$(printf '%s' "${DEV_RUNTIME_IMAGE}" | tr '/:' '--')"
+          } >> "$GITHUB_ENV"
+
+      - name: "📥 Resolve the workspace distro"
+        shell: bash
+        run: |
+          set -euo pipefail
+          distro="$(python3 -m cli.meta.ci.dev_runtime distro --image "${DEV_RUNTIME_IMAGE}")"
+          echo "INFINITO_DISTRO=${distro}" >> "$GITHUB_ENV"
+
+      - name: ▶️ Run workspace routine
+        shell: bash
+        run: |
+          set -euo pipefail
+          HOST_WORKDIR="$(pwd -P)"
+
+          docker run --rm \
+            -e GITHUB_ACTIONS=true \
+            -e GITHUB_REPOSITORY="${{ github.repository }}" \
+            -e GITHUB_REPOSITORY_OWNER="${{ github.repository_owner }}" \
+            -e GITHUB_SHA="${{ github.sha }}" \
+            -e DEV_RUNTIME_IMAGE \
+            -e INFINITO_DISTRO \
+            -e INFINITO_IMAGE="infinito-${INFINITO_DISTRO}" \
+            -e INFINITO_PULL_POLICY=never \
+            -e INFINITO_BUILD=1 \
+            -e NIX_CONFIG \
+            -e INFINITO_CACHE_REGISTRY_MAX_SIZE \
+            -e INFINITO_CACHE_REGISTRY_HOST_PATH \
+            -e INFINITO_CACHE_REGISTRY_CA_HOST_PATH \
+            -e INFINITO_CACHE_PACKAGE_BLOBSTORE_MAX \
+            -e INFINITO_CACHE_PACKAGE_HEAP \
+            -e INFINITO_CACHE_PACKAGE_DIRECT_MEM \
+            -e INFINITO_CACHE_PACKAGE_HOST_PATH \
+            --network host \
+            -v "${HOST_WORKDIR}:${HOST_WORKDIR}" \
+            -v "/mnt/docker:/var/lib/docker" \
+            -v "/var/run/docker.sock:/var/run/docker.sock" \
+            -w "${HOST_WORKDIR}" \
+            "${DEV_RUNTIME_IMAGE}" \
+            bash -lc '
+              bash ./scripts/tests/environment/00_orchestrator.sh
+            '
+
+      - name: 🩺 Capture rescue diagnostics
+        if: failure()
+        timeout-minutes: 15
+        shell: bash
+        run: |
+          set -euo pipefail
+          rescue_dir="${INFINITO_RESCUE_DIAGNOSTICS_BASE:?}/${INFINITO_WORKSPACE_SLUG:?}"
+          INFINITO_RESCUE_DIAGNOSTICS_DIR="${rescue_dir}" \
+            python3 utils/diagnostics/container.py \
+            "${INFINITO_WORKSPACE_SLUG}" "workspace suite failure" || true
+          bash ./scripts/tests/deploy/utils/rescue_index.sh "${rescue_dir}"
+
+      - name: ⬆️ Upload rescue diagnostics
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: rescue-diagnostics-workspace-${{ env.INFINITO_WORKSPACE_SLUG }}
+          path: ${{ env.INFINITO_RESCUE_DIAGNOSTICS_BASE }}/
+          if-no-files-found: ignore

.github/workflows/ci-orchestrator.yml

@@ -1,321 +0,0 @@
-name: "🎵 CI: Orchestrator"
-on:
-  workflow_call:
-    inputs:
-      distros:
-        description: "Space-separated distro list"
-        required: false
-        type: string
-        default: "arch debian ubuntu fedora centos"
-      whitelist:
-        description: "Optional space-separated whitelist of app ids"
-        required: false
-        type: string
-        default: ""
-      lifecycles:
-        description: "Lifecycle envelope for test-deploy discovery (comma/space-separated)."
-        required: false
-        type: string
-        default: ""
-      modes:
-        description: "Deploy modes to run (space-separated: 'swarm compose host'). Only listed modes run their deploy job."
-        required: false
-        type: string
-        default: "swarm compose host"
-      priority:
-        description: "Optional space-separated app ids for the priority line: deployed first in every mode; the regular line runs the remaining roles only after every priority job succeeded."
-        required: false
-        type: string
-        default: ""
-      image_wait_attempts:
-        description: "Polling attempts for waiting on prebuilt or mirrored images"
-        required: false
-        type: string
-        default: "1980"
-      image_wait_sleep_seconds:
-        description: "Polling interval in seconds for waiting on prebuilt or mirrored images"
-        required: false
-        type: string
-        default: "10"
-      roles_only:
-        description: "When 'true', skip the env-matrix test-development job."
-        required: false
-        type: string
-        default: "false"
-      trusted_pr:
-        description: "True when the PR carries the maintainer '🛡️ Trusted' label; skips the fork-prereq wait."
-        required: false
-        type: boolean
-        default: false
-      test_migration:
-        description: "'true' runs the opt-in mail-provider migration scenario (Mailu → Stalwart)."
-        required: false
-        type: string
-        default: "false"
-
-concurrency:
-  group: infinito-test-${{ github.ref }}
-  cancel-in-progress: false
-
-permissions:
-  actions: write
-  contents: read
-  packages: write
-  security-events: write
-  issues: write
-
-jobs:
-  wait-fork-prereq-run:
-    name: ⏳ Wait for fork run
-    runs-on: ubuntu-latest
-    env:
-      WAIT_REQUIRED: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && !inputs.trusted_pr }}
-    steps:
-      - name: 🏷️ Mark fork prerequisites as ready outside fork PRs or for trusted PRs
-        if: ${{ env.WAIT_REQUIRED != 'true' }}
-        shell: bash
-        run: |
-          set -euo pipefail
-          echo "Fork prerequisites are implicitly ready (non-fork PR or '🛡️ Trusted' label)."
-
-      - name: ⬇️ Checkout trusted workflow repo (helpers)
-        if: ${{ env.WAIT_REQUIRED == 'true' }}
-        uses: actions/checkout@v7.0.0
-        with:
-          ref: ${{ github.event.pull_request.base.sha }}
-          persist-credentials: false
-
-      - name: 🔑 Login to GHCR for CI image checks
-        if: ${{ env.WAIT_REQUIRED == 'true' }}
-        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: ⏳ Wait for privileged fork CI images
-        if: ${{ env.WAIT_REQUIRED == 'true' }}
-        shell: bash
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          WORKFLOW_FILE: entry-pull-request-change.yml
-          TARGET_EVENT: pull_request_target
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-          PR_UPDATED_AT: ${{ github.event.pull_request.updated_at }}
-          INFINITO_DISTROS: "arch debian ubuntu fedora centos"
-          IMAGE_TAG: ci-${{ github.sha }}
-          WAIT_ATTEMPTS: ${{ inputs.image_wait_attempts }}
-          WAIT_SLEEP_SECONDS: ${{ inputs.image_wait_sleep_seconds }}
-        run: ./scripts/meta/wait/images.sh
-
-  fork-prereqs-ready:
-    name: 🚦 Gate Dependencies
-    needs: wait-fork-prereq-run
-    if: ${{ always() }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v7.0.0
-      - name: ✅ Assert privileged prerequisites are ready
-        shell: bash
-        env:
-          RESULT: ${{ needs.wait-fork-prereq-run.result }}
-        run: ./scripts/github/gate/fork_prereqs.sh
-
-  security-codeql:
-    uses: ./.github/workflows/security-codeql.yml
-
-  lint:
-    uses: ./.github/workflows/lint.yml
-
-  test:
-    uses: ./.github/workflows/test.yml
-
-  build-ci-images:
-    uses: ./.github/workflows/images-build-ci.yml
-    with:
-      distros: "arch debian ubuntu fedora centos"
-      concurrency_channel: ci-orchestrator-build-consumer
-    secrets: inherit
-  
-  test-dns:
-    needs: 
-      - build-ci-images
-    uses: ./.github/workflows/test-dns.yml
-    with:
-      distros: "arch debian ubuntu fedora centos"
-  
-  images-mirror-missing:
-    uses: ./.github/workflows/images-mirror-missing.yml
-    with:
-      concurrency_channel: ci-orchestrator-mirror-consumer
-      image_wait_attempts: ${{ inputs.image_wait_attempts }}
-      image_wait_sleep_seconds: ${{ inputs.image_wait_sleep_seconds }}
-    secrets: inherit
-
-  test-deploy-single-node-priority:
-    name: ⭐ Priority · single-node
-    if: ${{ inputs.priority != '' && (contains(inputs.modes, 'compose') || contains(inputs.modes, 'host')) }}
-    needs:
-      - test-dns
-      - images-mirror-missing
-      - lint
-      - test
-    uses: ./.github/workflows/test-deploy-single-node.yml
-    with:
-      distros: ${{ inputs.distros }}
-      whitelist: ${{ inputs.priority }}
-      lifecycles: ${{ inputs.lifecycles }}
-      modes: ${{ inputs.modes }}
-      marker: " ⭐"
-
-  test-deploy-swarm-priority:
-    name: ⭐ Priority · swarm
-    if: ${{ inputs.priority != '' && contains(inputs.modes, 'swarm') }}
-    needs:
-      - lint
-      - test
-      - build-ci-images
-      - images-mirror-missing
-    uses: ./.github/workflows/test-deploy-swarm.yml
-    with:
-      distros: ${{ inputs.distros }}
-      whitelist: ${{ inputs.priority }}
-      lifecycles: ${{ inputs.lifecycles }}
-      marker: " ⭐"
-
-  test-deploy-single-node:
-    name: 🔁 Regular · single-node
-    if: >-
-      ${{ always()
-          && (contains(inputs.modes, 'compose') || contains(inputs.modes, 'host'))
-          && needs.lint.result == 'success'
-          && needs.test.result == 'success'
-          && needs.test-dns.result == 'success'
-          && needs.images-mirror-missing.result == 'success'
-          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
-          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) }}
-    needs:
-      - test-dns
-      - images-mirror-missing
-      - test-deploy-single-node-priority
-      - test-deploy-swarm-priority
-      - lint
-      - test
-    uses: ./.github/workflows/test-deploy-single-node.yml
-    with:
-      distros: ${{ inputs.distros }}
-      whitelist: ${{ inputs.whitelist }}
-      lifecycles: ${{ inputs.lifecycles }}
-      modes: ${{ inputs.modes }}
-      blacklist: ${{ inputs.priority }}
-
-  test-deploy-swarm:
-    name: 🔁 Regular · swarm
-    if: >-
-      ${{ always()
-          && contains(inputs.modes, 'swarm')
-          && needs.lint.result == 'success'
-          && needs.test.result == 'success'
-          && needs.build-ci-images.result == 'success'
-          && needs.images-mirror-missing.result == 'success'
-          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-single-node-priority.result)
-          && contains(fromJSON('["success","skipped"]'), needs.test-deploy-swarm-priority.result) }}
-    needs:
-      - build-ci-images
-      - images-mirror-missing
-      - test-deploy-single-node-priority
-      - test-deploy-swarm-priority
-      - lint
-      - test
-    uses: ./.github/workflows/test-deploy-swarm.yml
-    with:
-      distros: ${{ inputs.distros }}
-      whitelist: ${{ inputs.whitelist }}
-      lifecycles: ${{ inputs.lifecycles }}
-      blacklist: ${{ inputs.priority }}
-
-  # Exception: opt-in job — the called workflow gates itself on the forwarded
-  # test_migration input or the INFINITO_TEST_MIGRATION repository variable.
-  test-deploy-migration:
-    needs:
-      - test-dns
-      - images-mirror-missing
-    uses: ./.github/workflows/test-deploy-migration.yml
-    with:
-      test_migration: ${{ inputs.test_migration }}
-
-  test-runner-smoke:
-    needs:
-      - test-deploy-swarm
-    if: ${{ always() && vars.CI_SELF_HOSTED_RUNNER_COUNT != '0' && vars.CI_SELF_HOSTED_RUNNER_COUNT != '' && (needs.test-deploy-swarm.result == 'success' || needs.test-deploy-swarm.result == 'failure') }}
-    uses: ./.github/workflows/test-runner-smoke.yml
-    with:
-      ref: ${{ github.sha }}
-    secrets: inherit
-
-  test-install-make:
-    uses: ./.github/workflows/test-install-make.yml
-  test-install-pkgmgr:
-    uses: ./.github/workflows/test-install-pkgmgr.yml
-
-  test-development:
-    if: ${{ inputs.roles_only != 'true' }}
-    uses: ./.github/workflows/test-environment.yml
-    needs:
-      - images-mirror-missing
-    secrets: inherit
-
-  test-guide:
-    uses: ./.github/workflows/test-guide.yml
-    needs:
-      - images-mirror-missing
-      - build-ci-images
-    secrets: inherit
-
-  report-main-failures:
-    name: 🐛 Report main role failures
-    needs:
-      - test-deploy-single-node-priority
-      - test-deploy-swarm-priority
-      - test-deploy-single-node
-      - test-deploy-swarm
-    if: ${{ always() && github.ref == 'refs/heads/main' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      actions: read
-      contents: read
-    steps:
-      - uses: actions/checkout@v7.0.0
-      - name: 🐛 Open or append issues for failed roles
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: python3 -m cli.meta.ci.report_failures --run-id "${{ github.run_id }}" --repo "${{ github.repository }}"
-
-  done:
-    name: 🏁 Finish pipeline
-    needs:
-      - test-install-make
-      - test-install-pkgmgr
-      - test-deploy-single-node-priority
-      - test-deploy-swarm-priority
-      - test-deploy-single-node
-      - test-deploy-swarm
-      - test-deploy-migration
-      - test-runner-smoke
-      - test-development
-      - test-guide
-      - test
-      - lint
-      - security-codeql
-    if: ${{ always() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: 🛑 Fail if any upstream job failed or was cancelled
-        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
-        run: |
-          echo "Pipeline failed: at least one upstream job failed or was cancelled." >&2
-          exit 1
-      - name: ✅ Done
-        run: echo "All checks passed."

.github/workflows/cron-cleanup-stale.yml

@@ -1,4 +1,4 @@
-name: "🧹 Cleanup: Stale Repository Data"
+name: "⏰ Cleanup: Stale Repository Data"
 
 on:
   schedule:

.github/workflows/cron-images-cleanup-ci.yml

@@ -1,4 +1,4 @@
-name: "🧹 Images: Cleanup CI (GHCR)"
+name: "⏰ Images: Cleanup CI (GHCR)"
 
 on:
   schedule:

.github/workflows/cron-images-mirror-all.yml

@@ -1,4 +1,4 @@
-name: "🪞 Mirror: Docker Hub → GHCR"
+name: "⏰ Mirror: Docker Hub → GHCR"
 
 on:
   schedule:
@@ -87,9 +87,7 @@ jobs:
           cache-dependency-path: pyproject.toml
 
       - name: 📦 Install Python deps
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install PyYAML
+        run: PYTHON=python bash scripts/install/python.sh
 
       - name: 📦 Install skopeo
         run: ./scripts/install/apt.sh skopeo

.github/workflows/cron-release-highest.yml

@@ -1,4 +1,4 @@
-name: "🚀 Release: Highest Missing Version"
+name: "⏰ Release: Highest Missing Version"
 
 on:
   schedule:

.github/workflows/cron-rerun-cancelled.yml

@@ -0,0 +1,42 @@
+name: "⏰ Rerun: Cancelled Branch Runs"
+
+concurrency:
+  group: rerun-cancelled
+  cancel-in-progress: true
+
+on:
+  schedule:
+    - cron: "0 * * * *"
+  workflow_dispatch:
+
+permissions:
+  actions: write
+  contents: read
+
+jobs:
+  rerun:
+    name: 🔁 Rerun the failed jobs of cancelled runs
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - name: ⬇️ Checkout repository
+        uses: actions/checkout@v7.0.0
+
+      - name: ⏱️ Decide whether this hour is due
+        id: due
+        if: ${{ github.event_name == 'schedule' }}
+        shell: bash
+        env:
+          INTERVAL_MINUTES: ${{ vars.CI_RERUN_INTERVAL_MINUTES || '60' }}
+        run: bash scripts/github/rerun/due.sh
+
+      - name: 🔁 Restart the failed jobs per branch
+        if: ${{ steps.due.outputs.due != 'false' }}
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPOSITORY: ${{ github.repository }}
+          MAX_ATTEMPTS: ${{ vars.CI_RERUN_MAX_ATTEMPTS || '0' }}
+          MAX_AGE_HOURS: ${{ vars.CI_RERUN_MAX_AGE_HOURS || '24' }}
+        run: bash scripts/github/rerun/cancelled_branch_runs.sh

.github/workflows/cron-security-codeql.yml

@@ -1,3 +1,5 @@
+# nocheck: comments-valid  vendored CodeQL starter workflow; upstream comments kept verbatim
+#
 # For most projects, this workflow file will not need changing; you simply need
 # to commit it to your repository.
 #
@@ -9,7 +11,7 @@
 # the `language` matrix defined below to confirm you have the correct set of
 # supported CodeQL languages.
 #
-name: "🔒 Scan: CodeQL (Advanced)"
+name: "⏰ Scan: CodeQL (Advanced)"
 
 concurrency:
   group: codeql-${{ github.repository }}-${{ github.ref }}

.github/workflows/cron-security-scorecard.yml

@@ -3,7 +3,7 @@
 # policy, and support documentation.
 # nocheck: comments-valid  vendored OSSF scorecard workflow; upstream comments kept verbatim
 
-name: 🛡️ Scorecard supply-chain security
+name: "⏰ Scorecard supply-chain security"
 on:
   # For Branch-Protection check. Only the default branch is supported. See
   # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection

.github/workflows/cron-update.yml

@@ -1,4 +1,4 @@
-name: "🔄 Update: Versions"
+name: "⏰ Update: Versions"
 
 on:
   push:

.github/workflows/entry-delete-branch.yml

@@ -1,14 +1,15 @@
+# Enters the same concurrency group as entry-push-latest.yml for the deleted
+# branch, which cancels that run and all its child workflow_call runs.
+#
+# On `delete` events `github.ref_name` resolves to the default branch, not the
+# deleted ref, so the group keys on `github.event.ref`. Keying on `ref_name`
+# would collide with the push-to-main CI and cancel it.
+
 name: "🚫 Cancel: Runs on Branch Delete"
 
 on:
   delete:
 
-# Enters the same concurrency group as entry-push-latest.yml for the deleted
-# branch, which cancels that run and all its child workflow_call runs.
-# NOTE: on `delete` events, `github.ref_name` resolves to the default branch
-# (the workflow runs from the default branch context), not the deleted ref.
-# Use `github.event.ref` to get the actually deleted branch name — otherwise
-# this group would collide with the push-to-main CI and cancel it.
 concurrency:
   group: global-ci-${{ github.repository }}-${{ github.event.ref }}
   cancel-in-progress: true
@@ -18,7 +19,6 @@ permissions:
   contents: read
 
 jobs:
-  # Fallback: cancel any remaining active runs via the GitHub API.
   cancel-branch-runs:
     if: ${{ github.event.ref_type == 'branch' }}
     runs-on: ubuntu-latest

.github/workflows/entry-manual-steer.yml

@@ -0,0 +1,118 @@
+name: "🕹️ Manual"
+run-name: "🕹️ ${{ inputs.mode_fail_fast && '🛑 ' || '' }}${{ inputs.test_migration && '📧迁 ' || '' }}${{ inputs.sequencing != 'auto' && format('🐛{0} ', inputs.sequencing == 'serial' && '序' || '并') || '' }}${{ inputs.workspace != 'auto' && format('🧑‍💻{0} ', inputs.workspace == 'true' && '是' || '否') || '' }}${{ inputs.instructions != 'auto' && format('📖{0} ', inputs.instructions == 'true' && '是' || '否') || '' }}${{ inputs.filesystem != '' && format('🗄️{0} ', inputs.filesystem) || '' }}${{ inputs.distros != '' && format('🐧{0} ', inputs.distros) || '' }}${{ inputs.lifecycles != '' && format('♻️{0} ', inputs.lifecycles) || '' }}${{ inputs.modes != 'swarm compose host' && format('🚀{0} ', inputs.modes) || '' }}${{ inputs.priority != '' && format('⭐{0} ', inputs.priority) || '' }}${{ inputs.whitelist != '' && format('🎯{0}', inputs.whitelist) || '🔀 diff (origin/main)' }}"
+
+concurrency:
+  group: global-ci-${{ github.repository }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+    inputs:
+      distros:
+        description: "🐧 Distros to sweep. Empty: one at random. Example: 'arch debian'"
+        required: false
+        default: ""
+        type: string
+      priority:
+        description: "⭐ Roles to deploy first. The rest waits until they are green."
+        required: false
+        default: ""
+        type: string
+      whitelist:
+        description: "🎯 Roles to deploy. Empty: diff vs origin/main. '__ALL__': every role."
+        required: false
+        default: ""
+        type: string
+      lifecycles:
+        description: "♻️ Lifecycle envelope for test-deploy discovery (comma/space-separated)."
+        required: false
+        default: ""
+        type: string
+      modes:
+        description: "🚀 Deploy modes to test. Only the listed ones run their deploy job."
+        required: false
+        default: "swarm compose host"
+        type: string
+      filesystem:
+        description: "🗄️ Space-separated filesystems the per-distro pick draws from (ext4, btrfs, zfs). Empty: all three."
+        required: false
+        default: ""
+        type: string
+      sequencing:
+        description: "🐛 Deploy-mode order per line. 'serial': swarm, then compose, then host. 'parallel': all at once. 'auto': serial above 65 jobs per line."
+        required: false
+        default: "auto"
+        type: choice
+        options:
+          - "auto"
+          - "serial"
+          - "parallel"
+      mode_fail_fast:
+        description: "🛑 Stop a serial line at its first failed mode. Off: keep deploying the remaining modes."
+        required: false
+        default: true
+        type: boolean
+      workspace:
+        description: "🧑‍💻 Run the workspace test (dev-runtime image matrix). 'auto' runs it on a global scope only."
+        required: false
+        default: "auto"
+        type: choice
+        options:
+          - "auto"
+          - "true"
+          - "false"
+      instructions:
+        description: "📖 Run the instructions test (role README Production command). 'auto' runs it on a global scope only."
+        required: false
+        default: "auto"
+        type: choice
+        options:
+          - "auto"
+          - "true"
+          - "false"
+      test_migration:
+        description: "📧 Run the opt-in mail-provider migration scenario (Mailu → Stalwart)."
+        required: false
+        default: false
+        type: boolean
+
+permissions:
+  contents: read
+  packages: write
+  security-events: write
+  actions: write
+  issues: write
+
+jobs:
+  pick-distro:
+    name: 🎲 Pick distro(s)
+    runs-on: ubuntu-latest
+    outputs:
+      distros: ${{ steps.pick.outputs.distros }}
+    steps:
+      - uses: actions/checkout@v7.0.0
+
+      - id: pick
+        shell: bash
+        env:
+          INPUT_DISTROS: ${{ inputs.distros }}
+        run: ./scripts/github/resolve/manual_distros.sh
+
+  ci-orchestrator:
+    name: 🎛️ Orchestrate CI (manual)
+    needs: pick-distro
+    uses: ./.github/workflows/call-orchestrator.yml
+    with:
+      distros: ${{ needs.pick-distro.outputs.distros }}
+      whitelist: ${{ inputs.whitelist }}
+      lifecycles: ${{ inputs.lifecycles }}
+      modes: ${{ inputs.modes }}
+      priority: ${{ inputs.priority }}
+      sequencing: ${{ inputs.sequencing }}
+      mode_fail_fast: ${{ inputs.mode_fail_fast }}
+      filesystem: ${{ inputs.filesystem }}
+      workspace: ${{ inputs.workspace }}
+      instructions: ${{ inputs.instructions }}
+      test_migration: ${{ inputs.test_migration && 'true' || 'false' }}
+      force_cancel_in_progress: true
+    secrets: inherit

.github/workflows/entry-manual.yml

@@ -1,61 +0,0 @@
-name: "⚡ CI: Manual"
-run-name: "Manual CI triggered: ${{ inputs.distros }}; ${{ inputs.priority != '' && format('{0} ⭐; ', inputs.priority) || '' }}${{ inputs.whitelist != '' && inputs.whitelist || 'diff-derived (origin/main)' }}"
-
-concurrency:
-  group: global-ci-${{ github.repository }}-${{ github.ref_name }}
-  cancel-in-progress: true
-
-on:
-  workflow_dispatch:
-    inputs:
-      distros:
-        description: "Space-separated distro list (overrides default). Example: 'arch debian'"
-        required: false
-        default: "debian"
-        type: string
-      whitelist:
-        description: "Empty: diff vs origin/main. '__ALL__': force full deploy. Else: explicit space-separated app ids."
-        required: false
-        default: ""
-        type: string
-      test_migration:
-        description: "Run the opt-in mail-provider migration scenario (Mailu → Stalwart)."
-        required: false
-        default: false
-        type: boolean
-      lifecycles:
-        description: "Lifecycle envelope for test-deploy discovery (comma/space-separated)."
-        required: false
-        default: ""
-        type: string
-      modes:
-        description: "Deploy modes to test (space-separated: 'swarm compose host'). Only the listed modes run their deploy job."
-        required: false
-        default: "swarm compose host"
-        type: string
-      priority:
-        description: "Optional space-separated app ids for the priority line: deployed first in every mode; the regular line runs the remaining roles only after every priority job succeeded."
-        required: false
-        default: ""
-        type: string
-
-permissions:
-  contents: read
-  packages: write
-  security-events: write
-  actions: write
-  issues: write
-
-jobs:
-  ci-orchestrator:
-    name: 🎛️ Orchestrate CI (manual)
-    uses: ./.github/workflows/ci-orchestrator.yml
-    with:
-      distros: ${{ inputs.distros }}
-      whitelist: ${{ inputs.whitelist }}
-      lifecycles: ${{ inputs.lifecycles }}
-      modes: ${{ inputs.modes }}
-      priority: ${{ inputs.priority }}
-      roles_only: ${{ (inputs.priority != '' || (inputs.whitelist != '' && inputs.whitelist != '__ALL__')) && 'true' || 'false' }}
-      test_migration: ${{ inputs.test_migration && 'true' || 'false' }}
-    secrets: inherit

.github/workflows/entry-pr-change-orchestrate.yml

@@ -1,4 +1,4 @@
-name: "⚡ CI: Pull Request"
+name: "🔀 Pull Request"
 run-name: >-
   ${{ github.event_name }} / PR #${{ github.event.pull_request.number }} / ${{ github.event.action }} / ${{ github.event.pull_request.head.label }}
 
@@ -102,7 +102,6 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       whitelist: ${{ steps.resolve.outputs.whitelist }}
-      roles_only: ${{ steps.resolve.outputs.roles_only }}
     steps:
       - name: ⬇️ Checkout
         uses: actions/checkout@v7.0.0
@@ -134,11 +133,16 @@ jobs:
       - validate-pr-branch-prefix
       - detect-affected-roles
     if: ${{ github.event_name == 'pull_request' && needs.detect-pr-scope.outputs.run_ci_orchestrator == 'true' }}
-    uses: ./.github/workflows/ci-orchestrator.yml
+    permissions:
+      actions: write
+      contents: read
+      packages: write
+      security-events: write
+      issues: write
+    uses: ./.github/workflows/call-orchestrator.yml
     with:
       distros: "arch debian ubuntu fedora centos"
       whitelist: ${{ needs.detect-affected-roles.outputs.whitelist }}
-      roles_only: ${{ needs.detect-affected-roles.outputs.roles_only }}
       trusted_pr: ${{ contains(github.event.pull_request.labels.*.name, '🛡️ Trusted') }}
 
   prepare-fork-merge-ref:
@@ -164,7 +168,7 @@ jobs:
 
   build-ci-images-fork:
     needs: prepare-fork-merge-ref
-    uses: ./.github/workflows/images-build-ci.yml
+    uses: ./.github/workflows/call-images-build-ci.yml
     with:
       distros: "arch debian ubuntu fedora centos"
       checkout_ref: ${{ needs.prepare-fork-merge-ref.outputs.trusted == 'true' && github.event.pull_request.head.sha || needs.prepare-fork-merge-ref.outputs.checkout_ref }}
@@ -175,7 +179,7 @@ jobs:
 
   images-mirror-missing-fork:
     needs: prepare-fork-merge-ref
-    uses: ./.github/workflows/images-mirror-missing.yml
+    uses: ./.github/workflows/call-images-mirror-missing.yml
     with:
       repo_root: _pr_source
       source_repository: ${{ github.repository }}

.github/workflows/entry-pr-open-dependabot-close.yml

@@ -1,5 +1,3 @@
-name: "🚫 Dependabot: Close while auto-updates disabled"
-
 # Closes Dependabot PRs while update PRs are gated off via the
 # CI_ENABLE_AUTO_UPDATES repository variable. Dependabot itself cannot read
 # repository variables, so the gate is enforced here.
@@ -8,6 +6,8 @@ name: "🚫 Dependabot: Close while auto-updates disabled"
 # access to repository vars/secrets and a write-scoped GITHUB_TOKEN
 # even for Dependabot's PRs. This workflow does NOT check out PR code.
 
+name: "🚫 Dependabot: Close while auto-updates disabled"
+
 on:
   pull_request_target:
     types: [opened, reopened]
@@ -25,4 +25,4 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           PR_URL: ${{ github.event.pull_request.html_url }}
-        run: gh pr close "$PR_URL" --comment "Auto-closed by \`dependabot-close.yml\`. The repository variable \`CI_ENABLE_AUTO_UPDATES\` is not set to \`true\`, so update PRs are disabled. Set it to \`true\` under **Settings → Secrets and variables → Actions → Variables** to allow Dependabot PRs."
+        run: gh pr close "$PR_URL" --comment "Auto-closed by \`entry-pr-open-dependabot-close.yml\`. The repository variable \`CI_ENABLE_AUTO_UPDATES\` is not set to \`true\`, so update PRs are disabled. Set it to \`true\` under **Settings → Secrets and variables → Actions → Variables** to allow Dependabot PRs."

.github/workflows/entry-push-latest.yml

@@ -1,4 +1,4 @@
-name: "⚡ CI: Push"
+name: "📤 Push"
 
 concurrency:
   group: global-ci-${{ github.repository }}-${{ github.ref_name }}
@@ -129,7 +129,7 @@ jobs:
       - run-policy
       - pick-distro
       - resolve-priority
-    uses: ./.github/workflows/ci-orchestrator.yml
+    uses: ./.github/workflows/call-orchestrator.yml
     with:
       distros: ${{ needs.pick-distro.outputs.distros }}
       priority: ${{ needs.resolve-priority.outputs.priority }}
@@ -172,7 +172,7 @@ jobs:
       - pick-distro
       - release-guard
     if: needs.run-policy.outputs.should_run == 'true' && needs.release-guard.outputs.allowed == 'true'
-    uses: ./.github/workflows/release-version.yml
+    uses: ./.github/workflows/call-release-version.yml
     with:
       tag: ${{ needs.pick-distro.outputs.version_tag }}
     secrets: inherit

.github/workflows/test-environment.yml

@@ -1,87 +0,0 @@
-name: "💻 Test: Development Environment"
-
-on:
-  workflow_dispatch:
-  workflow_call:
-
-permissions:
-  actions: write
-  contents: read
-  packages: read
-
-jobs:
-  test-environment:
-    name: 🧑‍💻 Environment (${{ matrix.dev_runtime_image }})
-    runs-on: ubuntu-latest
-    timeout-minutes: 345
-
-    strategy:
-      fail-fast: false
-      matrix:
-        dev_runtime_image:
-          - manjarolinux/base
-          - debian:bookworm
-          - ubuntu:latest
-          - fedora:latest
-          - quay.io/centos/centos:latest
-
-    env:
-      DEV_RUNTIME_IMAGE: ${{ matrix.dev_runtime_image }}
-      NIX_CONFIG: |
-        access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
-      INFINITO_CACHE_REGISTRY_MAX_SIZE: 16g
-      INFINITO_CACHE_REGISTRY_HOST_PATH: /mnt/cache/registry/mirror
-      INFINITO_CACHE_REGISTRY_CA_HOST_PATH: /mnt/cache/registry/ca
-      INFINITO_CACHE_PACKAGE_BLOBSTORE_MAX: 8g
-      INFINITO_CACHE_PACKAGE_HEAP: 1g
-      INFINITO_CACHE_PACKAGE_DIRECT_MEM: 1g
-      INFINITO_CACHE_PACKAGE_HOST_PATH: /mnt/cache/package/data
-
-    steps:
-      - name: 🧹 Free disk space
-        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
-        with:
-          tool-cache: true
-          android: true
-          dotnet: true
-          haskell: true
-          large-packages: true
-          docker-images: true
-          swap-storage: false
-
-      - name: ⬇️ Checkout repository
-        uses: actions/checkout@v7.0.0
-
-      - name: ▶️ Run dev environment routine
-        shell: bash
-        run: |
-          set -euo pipefail
-          HOST_WORKDIR="$(pwd -P)"
-
-          docker run --rm \
-            -e GITHUB_ACTIONS=true \
-            -e GITHUB_REPOSITORY="${{ github.repository }}" \
-            -e GITHUB_REPOSITORY_OWNER="${{ github.repository_owner }}" \
-            -e GITHUB_SHA="${{ github.sha }}" \
-            -e DEV_RUNTIME_IMAGE \
-            -e INFINITO_DISTRO=debian \
-            -e INFINITO_IMAGE=infinito-debian \
-            -e INFINITO_PULL_POLICY=never \
-            -e INFINITO_BUILD=1 \
-            -e NIX_CONFIG \
-            -e INFINITO_CACHE_REGISTRY_MAX_SIZE \
-            -e INFINITO_CACHE_REGISTRY_HOST_PATH \
-            -e INFINITO_CACHE_REGISTRY_CA_HOST_PATH \
-            -e INFINITO_CACHE_PACKAGE_BLOBSTORE_MAX \
-            -e INFINITO_CACHE_PACKAGE_HEAP \
-            -e INFINITO_CACHE_PACKAGE_DIRECT_MEM \
-            -e INFINITO_CACHE_PACKAGE_HOST_PATH \
-            --network host \
-            -v "${HOST_WORKDIR}:${HOST_WORKDIR}" \
-            -v "/mnt/docker:/var/lib/docker" \
-            -v "/var/run/docker.sock:/var/run/docker.sock" \
-            -w "${HOST_WORKDIR}" \
-            "${DEV_RUNTIME_IMAGE}" \
-            bash -lc '
-              bash ./scripts/tests/environment/00_orchestrator.sh
-            '

.markdownlint-cli2.jsonc

@@ -34,8 +34,8 @@
     ".venv/**",
     "**/__pycache__/**",
     ".github/PULL_REQUEST_TEMPLATE/**",
-    ".agents/skills/**",
-    ".claude/skills/**",
+    ".agents/**",
+    ".claude/**",
     "skills/**"
   ]
 }

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 default_install_hook_types:
   - pre-commit
-  - pre-merge-commit
+  - commit-msg
   - pre-push
   - post-checkout
 
@@ -16,13 +16,13 @@ repos:
         stages:
           - pre-commit
       - id: make-test-merge-signed
-        name: Run make test-merge-signed (pre-merge-commit gate)
+        name: Run make test-merge-signed (merge gate)
         entry: make test-merge-signed
         language: system
         pass_filenames: false
         always_run: true
         stages:
-          - pre-merge-commit
+          - commit-msg
       - id: make-lint
         name: Run make lint (pre-push gate)
         entry: make lint

CHANGELOG.md

@@ -1,5 +1,118 @@
 # Changelog
 
+## [12.0.0] - 2026-08-08
+
+* **Docker Swarm as a second deployment mode.** Roles can now be deployed across
+  multiple hosts instead of being limited to a single Compose stack. New
+  *svc-swarm-node* initializes the cluster, while *svc-swarm-manager* marks the Raft
+  managers in the inventory. Shared state is provided through
+  *svc-storage-nfs-server* and *svc-storage-nfs-client*. *svc-registry-docker*
+  distributes locally built images, and *svc-registry-cache* provides a Docker Hub
+  pull-through cache so additional nodes do not multiply external pulls and
+  rate-limit usage. Deployment, dependency resolution, networking, health checks and
+  role templates now understand both Compose and Swarm semantics. See the
+  [svc-swarm-node](roles/svc-swarm-node/README.md) and
+  [svc-storage-nfs-server](roles/svc-storage-nfs-server/README.md) READMEs.
+
+* **Central engines and object storage.** A new *engine* lookup extends the existing
+  shared-database pattern to non-RDBMS services. Consumers can use centrally
+  provisioned Redis, Memcached, RabbitMQ, Elasticsearch, Typesense, Qdrant and Unbound
+  instances with isolated credentials, or retain an embedded sidecar.
+  *sys-svc-objstore* and the new *objstore* lookup provide the same abstraction for
+  SeaweedFS and MinIO, with lazy provider inclusion and object-storage integration
+  across fourteen application roles.
+
+* **Backup and disaster recovery.** *svc-bkp-container-2-local* is superseded by
+  *svc-bkp-volume-2-local*, making backup policy explicit per volume. The role prefers
+  filesystem snapshots when available and can prepare btrfs hosts for snapshot-based
+  backups itself. New *svc-bkp-nfs-2-local* protects the shared NFS export,
+  *svc-bkp-secrets-2-local* captures secret material, and
+  *svc-bkp-local-2-device* synchronizes backups to mounted external media. A unified
+  *recover* CLI drives restores, while Compose recovery checks and a full Swarm
+  disaster-recovery drill continuously verify the path. Rsync no longer retains
+  superseded files as *~* copies.
+
+* **Deployment and packaging foundations.** A host deployment mode joins Compose and
+  Swarm in the deployment tooling. The former *sys-package*, *sys-aur* and
+  *sys-aur-install* roles were replaced by a distribution-independent package
+  registry. Volume metadata now uses one canonical schema, single-file mounts become
+  Swarm configs, and service placement, replica counts, dependencies, addresses and
+  health probes resolve through deployment-aware lookups.
+
+* **Isolated development worktrees.** Parallel branches can use independent stacks
+  while sharing one cache layer, preventing containers and network state from
+  colliding. New *swarm-app-exec*, *swarm-diagnostic*, zombie-cluster and cross-mode
+  roundtrip targets improve live inspection and Compose/Swarm parity testing.
+
+* **CI and diagnostics.** Deployment runs are planned separately per mode and divided
+  into starred priority and regular lines, with branch-new roles scheduled first.
+  Matrix entries serialize across branches, large lines serialize their deployment
+  modes, and failed roles can be retriggered on the original distribution set;
+  *--strict* limits this to hard failures. Cancelled runs can have their failed jobs
+  rerun, and role failures on *main* open or update dedicated issues. Deployment stages
+  are sourced from *categories.yml*.
+
+  Rescue collection now uses one indexed diagnostic pipeline covering journal and
+  kernel windows, resolver and network state, PostgreSQL activity, container-kill
+  attribution, storage devices and NFS-Ganesha thread state. The Swarm recovery drill
+  and README deployment guide run across every supported distribution. CI also varies
+  Docker's data-root filesystem so ext4, btrfs and ZFS-specific paths receive real
+  coverage.
+
+* **Testing and documentation tooling.** New *test-e2e-cli* exercises the command-line
+  interface end to end. Lint and test jobs publish per-target and per-test summaries,
+  failed target logs are replayed after the summary, and the complexity matrix shows
+  whether a role already exists on *origin/main*. Generated role READMEs now provide
+  consistent status tables, quick-start instructions and dependency diagrams.
+
+* **New lint contracts.** Port variables must resolve through lookups, scheduler
+  services must declare *replicas: 1*, and volume-owning stack roles must declare
+  their backup policy. *role_path* is forbidden in shell and command bodies.
+  Deployment-mode defaults, container-inspection resolvers, asynchronous-task
+  rationales and health-check declarations are validated centrally. Hadolint
+  directives and documented *nocheck* exceptions are recognized instead of producing
+  false positives.
+
+* **Image delivery and frontend dependencies.** Image references now resolve from
+  *meta/services.yml* as their single source of truth, replacing unpinned *latest*
+  tags. *web-app-bigbluebutton* pins its component images separately. *web-svc-cdn*
+  builds and serves mirrored frontend packages through an internal one-shot service,
+  with the package-cache CA trusted by build containers.
+
+* **Application improvements.** Penpot gained hardened OIDC, LDAP and native-login
+  variants, SeaweedFS-backed asset tests and Swarm-ready health checks. Pi-hole
+  received its v6-compatible login and password flow, Keycloak RBAC integration and
+  expanded persona coverage. Further deployment, replication and startup fixes landed
+  across Akaunting, BigBlueButton, ERPNext, Gitea, Keycloak, Magento, Mailu, Mastodon,
+  Matomo, MediaWiki, Nextcloud, OpenProject, Pixelfed, Semaphore, Taiga, WordPress,
+  XWiki and the internal CDN.
+
+* **Lifecycle promotions:** *web-app-checkmk*, *web-app-jellyfin*, *web-app-n8n*,
+  *svc-registry-docker* and *svc-registry-cache* advance to *beta*.
+
+* **Image and dependency version jumps (net since 11.6.0):**
+  * *web-app-gitlab*: omnibus *gitlab-ee* 19.1.1 to CNG component images v19.1.2
+  * *web-app-mattermost*: 11.8.3 to 11.9.0
+  * *web-app-opentalk*: v2.8.1 to v2.9.0
+  * *web-app-prometheus*: v3.13.0 to v3.13.1
+  * *web-app-joomla*: 6.1.1-php8.3-apache to 6.1.2-php8.3-apache
+  * *web-app-baserow*: 2.3.0 to 2.3.1
+  * *web-app-magento*: PHP 8.5-fpm to 8.4-fpm
+  * *backup-docker-to-local*: 3.1.1 to 3.1.3
+  * *baudolo*: pinned to 3.4.1
+
+**Removed**
+
+* *sys-package*, *sys-aur* and *sys-aur-install*, superseded by the
+  distribution-independent package registry.
+* *svc-bkp-container-2-local*, superseded by the volume-oriented backup roles.
+
+**Contributors**
+
+* [Kevin Veen-Birkenbach](https://veen.world): Swarm and NFS deployment mode,
+  central engines and object storage, backup and recovery rework, CI pipeline,
+  linting and tooling
+
 ## [11.6.0] - 2026-07-08
 
 * New *web-app-n8n* role: n8n workflow automation (Community Edition). SSO is wired through a trusted-header edge hook that is copied and mounted only when SSO is enabled (native SSO is enterprise-only in CE), LDAP sign-in is supported, and per-scenario Playwright coverage exercises the CE edge-SSO model. Ships at lifecycle *alpha*; subnet and port collisions with the new *web-app-semaphore* role were resolved on the way in. See the [web-app-n8n README](roles/web-app-n8n/README.md).

Dockerfile

@@ -1,5 +1,3 @@
-# syntax=docker/dockerfile:1
-
 # Base image (pkgmgr) selector. SPOT lives in default.env's
 # INFINITO_PARENT_IMAGE and is forwarded as a build arg by compose.yml /
 # scripts/image/build.sh.
@@ -8,9 +6,6 @@
 ARG INFINITO_PARENT_IMAGE
 FROM ${INFINITO_PARENT_IMAGE} AS full
 
-# hadolint DL4006: ensure pipefail is set for RUN instructions that use pipes
-SHELL ["/bin/bash", "-o", "pipefail", "-lc"]
-
 ARG NIX_CONFIG
 
 ARG INFINITO_SRC_DIR
@@ -21,6 +16,9 @@ ENV PYTHON="${INFINITO_VENV_DIR}/bin/python"
 ENV PIP="${INFINITO_VENV_DIR}/bin/python -m pip"
 ENV PATH="${INFINITO_VENV_DIR}/bin:${PATH}"
 
+# hadolint DL4006: ensure pipefail is set for RUN instructions that use pipes
+SHELL ["/bin/bash", "-o", "pipefail", "-lc"]
+
 RUN set -euo pipefail; \
   cat /etc/os-release || true; \
   if [ -f /etc/nix/nix.conf ]; then \
@@ -32,10 +30,11 @@ COPY . ${INFINITO_SRC_DIR}
 
 # hadolint ignore=DL3008,DL3033,DL3041,SC1090
 RUN set -euo pipefail; \
-  source <(grep -hE '^INFINITO_(PYTHON|DOCKER_CLI|PACKAGE)_INSTALL_SCRIPT=' "${INFINITO_SRC_DIR}/default.env"); \
+  source <(grep -hE '^INFINITO_(PYTHON|DOCKER_CLI|PACKAGE|FILESYSTEM)_INSTALL_SCRIPT=' "${INFINITO_SRC_DIR}/default.env"); \
   /bin/bash "${INFINITO_SRC_DIR}/${INFINITO_PYTHON_INSTALL_SCRIPT:?}"; \
   /bin/bash "${INFINITO_SRC_DIR}/${INFINITO_DOCKER_CLI_INSTALL_SCRIPT:?}"; \
-  /bin/bash "${INFINITO_SRC_DIR}/${INFINITO_PACKAGE_INSTALL_SCRIPT:?}"
+  /bin/bash "${INFINITO_SRC_DIR}/${INFINITO_PACKAGE_INSTALL_SCRIPT:?}"; \
+  /bin/bash "${INFINITO_SRC_DIR}/${INFINITO_FILESYSTEM_INSTALL_SCRIPT:?}"
 
 RUN set -euo pipefail; \
   systemctl mask systemd-firstboot.service first-boot-complete.target || true; \
@@ -44,6 +43,7 @@ RUN set -euo pipefail; \
 ENV container=docker
 STOPSIGNAL SIGRTMIN+3
 
+# hadolint ignore=SC3040
 RUN set -euo pipefail; \
   export NIX_CONFIG="${NIX_CONFIG:-}"; \
   "${INFINITO_SRC_DIR}/scripts/docker/entry.sh" --compile -- true
@@ -53,7 +53,7 @@ WORKDIR /
 COPY scripts/docker/healthcheck.sh /usr/local/bin/healthcheck.sh
 RUN chmod +x /usr/local/bin/healthcheck.sh
 HEALTHCHECK --interval=5s --timeout=20s --start-period=30s --retries=20 \
-  CMD /usr/local/bin/healthcheck.sh
+  CMD ["/usr/local/bin/healthcheck.sh"]
 
 ENTRYPOINT ["/bin/bash", "-c", "exec \"${INFINITO_SRC_DIR}/scripts/docker/entry.sh\" \"$@\"", "--"]
 

Makefile

@@ -65,7 +65,7 @@ build-cleanup:
 .PHONY: build-dependency
 # Pull the build dependency image.
 build-dependency:
-	@docker pull ghcr.io/kevinveenbirkenbach/pkgmgr-$${INFINITO_DISTRO}:stable
+	@docker pull "$${INFINITO_PARENT_IMAGE:?Run 'make dotenv' to generate the .env single source of truth}"
 
 .PHONY: build-missing
 # Build the local image if it is missing.
@@ -351,7 +351,7 @@ install-lint-force:
 .PHONY: install-python
 # Install Python tooling.
 install-python: install-venv
-	@bash scripts/install/python.sh
+	@bash scripts/install/python.sh deploy
 
 .PHONY: install-python-dev
 # Install Python tooling including lint and dev dependencies.
@@ -684,24 +684,34 @@ swarm-shell:
 	@test -n '$(name)' || { echo 'usage: make swarm-shell name=<cluster-id> [node=<container>]'; exit 2; }
 	@SWARM_NAME='$(name)' node='$(node)' bash scripts/tests/deploy/act/shell_node.sh
 
+SWARM_DISTROS = $(or $(distros),$${INFINITO_DISTRO:?})
+
 .PHONY: swarm-zombie
 # Run a swarm matrix-app test and leave the cluster alive afterwards for post-mortem inspection.
 # Param app: matrix application id (e.g. web-app-baserow).
+# Param distros: optional single distro the cluster runs on (default: INFINITO_DISTRO from .env).
 # Param variant: optional matrix variant index to deploy (default 0); a multi-variant app runs one cluster per swarm-zombie, so pick the round to validate.
 # Param disable: optional comma-separated provider keys removed from the test inventory (e.g. matomo,dashboard,prometheus,email,css).
 # Param name: optional cluster-id prefix for the container + network names (parallel/named clusters); release with the same name=.
+# Param step_timeout: optional minute budget for the matrix-deploy step (default 690).
 # Note: Use `make swarm-exec` / `make swarm-shell` to inspect, `make swarm-down` to release.
 swarm-zombie: install-act
-	@test -n '$(app)' || { echo 'usage: make swarm-zombie app=<application_id> [variant=<idx>] [name=<cluster-id>] [disable=<keys>]'; exit 2; }
+	@test -n '$(app)' || { echo 'usage: make swarm-zombie app=<application_id> [distros=<distro>] [variant=<idx>] [name=<cluster-id>] [disable=<keys>]'; exit 2; }
 	@SWARM_NAME='$(or $(name),$(app))' INFINITO_KEEP_SWARM_NODES=false bash scripts/tests/deploy/swarm/utils/clean/teardown.sh
 	@bash scripts/tests/deploy/act/down_act_outer.sh
 	@ACT_RM=false \
 	 ACT_BIND=true \
-	 ACT_ENV='INFINITO_KEEP_SWARM_NODES=true;INFINITO_APP_DISCOVERY_RUNNER=host;INFINITO_DEPLOY_MODE=swarm;disable=$(disable);SWARM_NAME=$(or $(name),$(app))' \
-	 ACT_WORKFLOW=.github/workflows/test-deploy-swarm.yml \
+	 ACT_ENV="INFINITO_KEEP_SWARM_NODES=true; \
+	 INFINITO_APP_DISCOVERY_RUNNER=host; \
+	 INFINITO_DEPLOY_MODE=swarm; \
+	 disable=$(disable); \
+	 SWARM_NAME=$(or $(name),$(app)); \
+	 INFINITO_SWARM_STEP_TIMEOUT_MINUTES=$(or $(step_timeout),690); \
+	 INFINITO_DISTROS=$(SWARM_DISTROS)" \
+	 ACT_WORKFLOW=.github/workflows/call-test-deploy-swarm.yml \
 	 ACT_JOB=swarm \
 	 ACT_MATRIX='apps:$(app);variant:$(or $(variant),0)' \
-	 ACT_INPUTS='whitelist=$(app)' \
+	 ACT_INPUTS="whitelist=$(app) distros=$(SWARM_DISTROS)" \
 	 bash scripts/tests/deploy/act/workflow.sh
 
 .PHONY: system-purge
@@ -711,7 +721,6 @@ system-purge:
 
 .PHONY: test
 # Run the full test pipeline.
-# Note: parallel execution with fail-fast.
 test: install
 	@bash scripts/make/parallel.sh \
 		test-external \
@@ -746,7 +755,7 @@ test-main-merged:
 	@bash scripts/git/assert/main_merged.sh
 
 .PHONY: test-merge-signed
-# Verify every commit an in-progress merge brings in (HEAD..MERGE_HEAD) is signed (pre-merge-commit gate).
+# Verify every commit an in-progress merge brings in (HEAD..MERGE_HEAD) is signed (commit-msg gate).
 test-merge-signed:
 	@bash scripts/git/assert/merge_signed.sh
 
@@ -758,6 +767,13 @@ test-migration:
 	@INFINITO_TEST_MIGRATION=true \
 	bash roles/web-app-stalwart/files/test.sh
 
+.PHONY: test-performance
+# Run the runtime-performance suite (not part of the `test` fan-out).
+test-performance: install
+	@INFINITO_TEST_TYPE="performance" \
+	INFINITO_COMPILE=0 \
+	bash scripts/tests/code/wrapper.sh
+
 .PHONY: test-signed
 # Verify HEAD is signed.
 # Note: `git log %G?` returns N for unsigned; gates the pre-push hook against unsigned tips.
@@ -776,6 +792,32 @@ test-unit: install
 	INFINITO_COMPILE=0 \
 	bash scripts/tests/code/wrapper.sh
 
+.PHONY: worktree-down
+# Stop a branch worktree's stack, release the checkout and free its slot.
+# Usage: make worktree-down branch=<name> [base=<dir>] [force=true]
+# Param branch: branch the worktree was created for (required).
+# Param base: parent directory the worktree lives in (default ~/.local/share/worktrees/<domain>/<account>/<repo>).
+# Param force: true discards uncommitted changes in the worktree.
+worktree-down:
+	@test -n '$(branch)' || { echo 'usage: make worktree-down branch=<name> [base=<dir>] [force=true]'; exit 2; }
+	@bash scripts/system/worktree/down.sh '$(branch)' '$(base)' '$(or $(force),false)'
+
+.PHONY: worktree-prune
+# Drop registrations of worktrees whose directory is gone, freeing the branches they claim.
+# Usage: make worktree-prune
+worktree-prune:
+	@bash scripts/system/worktree/prune.sh
+
+.PHONY: worktree-up
+# Check a branch out into an isolated worktree with its own subnet, ports and container names.
+# Usage: make worktree-up branch=<name> [base=<dir>]
+# Note: the worktree shares the primary checkout's cache stack instead of starting its own.
+# Param branch: branch to check out (required).
+# Param base: parent directory for the worktree (default ~/.local/share/worktrees/<domain>/<account>/<repo>).
+worktree-up:
+	@test -n '$(branch)' || { echo 'usage: make worktree-up branch=<name> [base=<dir>]'; exit 2; }
+	@bash scripts/system/worktree/up.sh '$(branch)' '$(base)'
+
 .PHONY: wsl2-dns-setup
 # Set up DNS on WSL2.
 wsl2-dns-setup:

aliases

@@ -16,8 +16,8 @@ alias i8ceco='m clean-container-owned'
 alias i8cepa='m clean-pycache-dirs'
 alias i8cesa='m clean-stale-nfs'
 alias i8cesu='m clean-sudo'
-alias i8cifail='__i8cifail(){ [ -z "${1:-}" ] && { echo "usage: i8cifail <run-id|url> [total|swarm|compose]" >&2; return 2; }; infinito administration deploy ci trigger --failed "${2:-total}" --run "$1"; }; __i8cifail' # retrigger failed roles of run $1 (id or url), scope $2 (default total)
-alias i8cifo='__i8cifo(){ [ -z "${1:-}" ] && { echo "usage: i8cifo <run-id|url> [total|swarm|compose]" >&2; return 2; }; infinito administration deploy ci trigger --failed "${2:-total}" --strict --run "$1"; }; __i8cifo' # retrigger only hard-failed roles of run $1, excluding cancelled/aborted; scope $2 (default total)
+alias i8cifail='__i8cifail(){ [ -z "${1:-}" ] && { echo "usage: i8cifail <run-id|url> [total|swarm|compose]" >&2; return 2; }; infinito administration deploy ci trigger --failed "${2:-total}" --run "$1"; }; __i8cifail' # retrigger failed and never-deployed priority roles of run $1 (id or url), scope $2 (default total), reproducing that run's configuration
+alias i8cifo='__i8cifo(){ [ -z "${1:-}" ] && { echo "usage: i8cifo <run-id|url> [total|swarm|compose]" >&2; return 2; }; infinito administration deploy ci trigger --failed "${2:-total}" --strict --run "$1"; }; __i8cifo' # like i8cifail but only hard-failed roles, excluding cancelled/aborted; scope $2 (default total)
 alias i8cire='__i8cire(){ make quality && gips && i8cifail "$1" swarm; }; __i8cire'
 alias i8co='m compose-deploy'
 alias i8co2='m console'
@@ -97,6 +97,8 @@ alias i8pi='pcd infinito' # cd to the infinito repo
 alias i8puci='__i8puci(){ gips && i8cifail "$1" swarm; }; __i8puci' # sign-push + retrigger failed swarm CI jobs of run $1, skipping the quality gate
 alias i8pufail='__i8pufail(){ gips && i8cifail "$1"; }; __i8pufail' # sign-push, then retrigger all failed roles (compose+swarm) of run $1
 alias i8pufo='__i8pufo(){ gips && i8cifo "$1"; }; __i8pufo' # sign-push, then retrigger only hard-failed roles of run $1, excluding cancelled/aborted
+alias i8puma='__i8puma(){ local gates=() ok=1; case "${3:-}" in "") ;; true|false) gates=(-f workspace="$3" -f instructions="$3");; *) ok=0;; esac; { [ -z "${1:-}" ] || [ "$ok" = 0 ]; } && { echo "usage: i8puma <priority app ids> [distros] [true|false]" >&2; return 2; }; i8source && gips && gh workflow run entry-manual.yml --repo "$(git remote get-url "$(git config --get remote.pushDefault)")" --ref "$(git rev-parse --abbrev-ref HEAD)" -f priority="$(python3 -m utils.roles.display --encode "$1")" -f distros="${2:-${INFINITO_DISTROS}}" "${gates[@]}"; }; __i8puma' # sign-push, then dispatch a manual CI run on the push remote (remote.pushDefault): priority line $1, distros $2 (default INFINITO_DISTROS), whitelist diff-derived; $3 true forces the workspace and instructions tests, false suppresses them, omitted leaves them on the workflow auto scope
+alias i8puwl='__i8puwl(){ [ -z "${1:-}" ] && { echo "usage: i8puwl <whitelist app ids> [distros]" >&2; return 2; }; i8source && gh workflow run entry-manual.yml --repo "$(git remote get-url "$(git config --get remote.pushDefault)")" --ref "$(git rev-parse --abbrev-ref HEAD)" -f whitelist="$(python3 -m utils.roles.display --encode "$1")" -f distros="${2:-${INFINITO_DISTROS}}" -f workspace=false -f instructions=false; }; __i8puwl' # dispatch a manual CI run on the push remote (remote.pushDefault) limited to whitelist $1, distros $2 (default INFINITO_DISTROS), with the workspace and instructions tests off; pushes nothing, so the run uses whatever the remote already holds for the current branch
 alias i8qu='m quality'
 alias i8quhi='m quality-high'
 alias i8re='m requirements-archive'
@@ -133,8 +135,12 @@ alias i8temesi='m test-merge-signed'
 alias i8temi='m test-migration'
 alias i8teni='m test-integration'
 alias i8tenu='m test-unit'
+alias i8tepe='m test-performance'
 alias i8tesi='m test-signed'
 alias i8texe='m test-external'
 alias i8wo='m wsl2-dns-setup'
+alias i8wodo='m worktree-down'
+alias i8wopr='m worktree-prune'
 alias i8wose='m wsl2-systemd-check'
 alias i8wotu='m wsl2-trust-windows'
+alias i8woup='m worktree-up'

cli/administration/deploy/ci/runs.py

@@ -10,9 +10,11 @@ Deploy jobs are matched by the compose/swarm/host glyph (from the symbol
 glossary, the single source of truth) their test-deploy-{compose,swarm,host}.yml
 matrix titles them with -- NOT the words "Compose"/"Swarm"/"Host", which no
 real job carries (matching those made ``--failed swarm`` silently find
-nothing). The orchestrator prepends a
-"caller / " prefix and GitHub appends a " <variant>" shard suffix (e.g.
-" 0,1"); both are tolerated.
+nothing). What follows the glyph is the role's display name
+(``utils.roles.display``), decoded back to the role id; a bare role id from a
+run that predates the display names decodes just as well. The orchestrator
+prepends a "caller / " prefix and the shard suffix (e.g. " 0,1") rides along
+in the display name; both are tolerated.
 """
 
 from __future__ import annotations
@@ -22,6 +24,9 @@ import re
 import subprocess
 import sys
 
+from utils.distros import distro_names
+from utils.github import run_name
+from utils.roles.display import display_names
 from utils.symbol_glossary import to_emoji
 
 PASS = "✅"  # noqa: S105  emoji glyph, not a credential
@@ -32,6 +37,17 @@ MISSING = "➖"
 
 MODES = ("docker", "swarm", "host")
 
+CONFIG_INPUTS = (
+    "distros",
+    "modes",
+    "lifecycles",
+    "filesystem",
+    "sequencing",
+    "mode_fail_fast",
+    "workspace",
+    "instructions",
+)
+
 MODE_GLYPHS = {
     "docker": to_emoji("compose"),
     "swarm": to_emoji("swarm"),
@@ -39,10 +55,7 @@ MODE_GLYPHS = {
 }
 _GLYPH_MODE = {glyph: mode for mode, glyph in MODE_GLYPHS.items()}
 
-_JOB_RE = re.compile(
-    rf"({'|'.join(map(re.escape, MODE_GLYPHS.values()))})"
-    r"\s+([a-z0-9]+(?:-[a-z0-9]+)+)(?:\s+[\d,]+)?\s*$"
-)
+_JOB_RE = re.compile(rf"({'|'.join(map(re.escape, MODE_GLYPHS.values()))})\s*(.+?)\s*$")
 
 
 def _effective(job: dict) -> str:
@@ -66,17 +79,20 @@ def cell(state: str) -> str:
 
 def _iter_deploy_jobs(jobs: list[dict]):
     """Yield ``(app, mode, job)`` for every compose/swarm deploy job."""
+    codec = display_names()
     for job in jobs:
         match = _JOB_RE.search(str(job.get("name", "")))
         if not match:
             continue
-        yield match.group(2), _GLYPH_MODE[match.group(1)], job
+        app = codec.decode(match.group(2))
+        if app is not None:
+            yield app, _GLYPH_MODE[match.group(1)], job
 
 
 def app_of_job(name: str) -> str | None:
     """The role id a deploy job ``name`` encodes, or None if it is not one."""
     match = _JOB_RE.search(name)
-    return match.group(2) if match else None
+    return display_names().decode(match.group(2)) if match else None
 
 
 _SEVERITY = {"success": 0, "running": 1, "failure": 3}
@@ -224,6 +240,162 @@ def fetch_jobs(run_id: str, repo: str | None = None) -> list[dict]:
     )
 
 
+def fetch_run(run_id: str, repo: str | None = None) -> dict:
+    """Jobs plus the run title, in one ``gh`` call.
+
+    The REST API answers ``inputs: null`` for a finished workflow_dispatch, so
+    the inputs are recovered from the run itself: short ones from the title
+    (:func:`config_from_title`), the long ones from a called job's log
+    (:func:`inputs_from_jobs`), which the title truncates.
+    """
+    return json.loads(
+        _gh(["run", "view", run_id, "--json", "jobs,displayTitle"], repo=repo)
+    )
+
+
+_INPUT_RE = re.compile(r"^\S+\s+(?P<name>[A-Za-z_][A-Za-z0-9_]*): ?(?P<value>.*)$")
+
+
+def inputs_from_jobs(jobs: list[dict], repo: str) -> dict[str, str]:
+    """The dispatch inputs a called job's log records verbatim.
+
+    GitHub cuts ``display_title`` at 512 UTF-8 bytes (measured on two runs,
+    both exactly 512) and closes it with a literal ``...``. The run-name
+    template renders ``priority`` second-to-last and ``whitelist`` last, and
+    on the ``--failed`` path the whitelist is empty, so the cut always lands
+    inside the priority list: measured 21 of 69 roles survived, the 21st a
+    half-token. A CJK display name costs 3 bytes per character, so the budget
+    is ~22 display names against ~28 bare role ids. Every job of the *called*
+    workflow opens its log with an ``##[group] Inputs`` block holding the full
+    values, so that is the SPOT for anything the title cannot hold.
+
+    Args:
+        jobs: the run's jobs. Only depth-1 jobs are read: a deeper one echoes
+            its own workflow's inputs, which carry no ``priority``.
+        repo: ``owner/repo`` the run lives in.
+
+    Returns:
+        input name -> value, empty when the run has no called job.
+    """
+    called = next(
+        (
+            job
+            for job in jobs
+            if str(job.get("name", "")).count(" / ") == 1 and job.get("databaseId")
+        ),
+        None,
+    )
+    if called is None:
+        return {}
+    log = _gh(["api", f"repos/{repo}/actions/jobs/{called['databaseId']}/logs"])
+    inputs: dict[str, str] = {}
+    inside = False
+    for line in log.splitlines():
+        if line.endswith("##[group] Inputs"):
+            inside = True
+            continue
+        if inside:
+            if line.endswith("##[endgroup]"):
+                break
+            match = _INPUT_RE.match(line)
+            if match:
+                inputs[match["name"]] = match["value"].strip()
+    return inputs
+
+
+def dispatched_priority(source: dict, repo: str) -> str:
+    """The source run's ``priority`` input, whole.
+
+    The title's priority segment is the cross-check, not the source: it proves
+    a priority line existed even when truncation mangled it, so an empty log
+    read is a broken reader -- expired logs, a moved Inputs block, an inlined
+    orchestrator -- rather than a run dispatched without one.
+
+    Args:
+        source: the run as :func:`fetch_run` returns it.
+        repo: ``owner/repo`` the run lives in.
+    """
+    priority = inputs_from_jobs(source["jobs"], repo).get("priority", "")
+    if not priority and run_name.value_from_title(source["displayTitle"], "priority"):
+        raise SystemExit(
+            "Source run has a priority line but its job log records no inputs; "
+            "the Inputs block moved or the log expired. Retriggering now would "
+            "silently drop every priority role."
+        )
+    return priority
+
+
+def untriggered_priority(
+    priority: str, statuses: dict[str, dict[str, str]]
+) -> list[str]:
+    """Roles the source run's priority line named but never deployed at all.
+
+    A priority role can fall behind the discovery budget cut in every mode, so
+    the run holds no job for it and :func:`failed_roles` cannot see it — it is
+    neither green nor red, it simply never ran. Carrying it into the retrigger
+    is the only way it ever gets deployed.
+
+    Args:
+        priority: the raw ``priority`` input, as :func:`inputs_from_jobs` reads
+            it back. A truncated value would smuggle a half-token into the
+            retrigger's role set, so an undecodable name raises instead.
+        statuses: role -> mode -> state, from :func:`parse_role_statuses`.
+    """
+    codec = display_names()
+    named = []
+    for name in priority.split():
+        role = codec.decode(name)
+        if role is None:
+            raise SystemExit(
+                f"Cannot resolve priority entry {name!r} to a role. The source "
+                f"run's priority input is corrupt or truncated; retriggering "
+                f"with it would deploy a role set that is silently incomplete."
+            )
+        named.append(role)
+    return sorted(role for role in named if role not in statuses)
+
+
+def distros_from_jobs(jobs: list[dict]) -> str:
+    """Distros the run actually swept, read back from its discover job names.
+
+    A manual run dispatched with no distro list resolves one at random, so its
+    title records nothing and only the discover jobs — which name the distros
+    they discovered for — still carry the answer. Matched by content, not by
+    job-name format: the parenthesised group whose every token is a declared
+    distro (meta/distros.yml SPOT) is the list.
+    """
+    known = set(distro_names())
+    for job in jobs:
+        for group in re.findall(r"\(([^()]*)\)", str(job.get("name", ""))):
+            tokens = group.split()
+            if tokens and set(tokens) <= known:
+                return " ".join(tokens)
+    return ""
+
+
+def config_from_title(title: str) -> dict[str, str]:
+    """Configuration inputs a manual run was dispatched with, keyed by input
+    name. An input left on its workflow default renders no title segment and
+    is absent here, so the retrigger leaves it on that same default.
+
+    Only the configuration is recovered, never the selection: ``whitelist``
+    and ``priority`` say which roles ran, and a retrigger computes those
+    itself. Runs from any other entry point carry an unrelated title and
+    yield nothing at all.
+    """
+    recorded = run_name.values_from_title(title)
+    return {name: recorded[name] for name in CONFIG_INPUTS if name in recorded}
+
+
+def config_from_run(title: str, jobs: list[dict]) -> dict[str, str]:
+    """The source run's configuration, with the distros the title does not
+    record recovered from its jobs (:func:`distros_from_jobs`)."""
+    config = config_from_title(title)
+    if not config.get("distros"):
+        config = {**config, "distros": distros_from_jobs(jobs)}
+    return {name: value for name, value in config.items() if value}
+
+
 def find_last_deploy_run(
     branch: str, repo: str | None = None, limit: int = 15
 ) -> dict | None:
@@ -243,7 +415,7 @@ def find_last_deploy_run(
             "-L",
             str(limit),
             "--json",
-            "databaseId,url,workflowName,createdAt,status",
+            "databaseId,url,workflowName,createdAt,status,displayTitle",
         ],
         repo=repo,
     )
@@ -261,11 +433,26 @@ def dispatch_workflow(
     whitelist: str = "",
     *,
     priority: str = "",
+    config: dict[str, str] | None = None,
     repo: str | None = None,
 ) -> None:
+    """Dispatch *workflow*, carrying *config* (see :func:`config_from_title`)
+    over verbatim. An input left out keeps the workflow's own default.
+
+    The role selections go out as display names (``utils.roles.display``), so
+    the run title reads them back the way the deploy jobs are titled. The
+    workflow decodes them again before any consumer sees a role id.
+    """
+    codec = display_names()
     args = ["workflow", "run", workflow, "--ref", ref]
     if whitelist:
-        args += ["-f", f"whitelist={whitelist}"]
+        args += ["-f", f"whitelist={codec.encode_list(whitelist)}"]
     if priority:
-        args += ["-f", f"priority={priority}"]
+        args += ["-f", f"priority={codec.encode_list(priority)}"]
+    args += [
+        arg
+        for name, value in (config or {}).items()
+        if value
+        for arg in ("-f", f"{name}={value}")
+    ]
     _gh(args, repo=repo)

cli/administration/deploy/ci/trigger/__main__.py

@@ -1,6 +1,6 @@
-"""Trigger the manual CI run (entry-manual.yml) for the current branch.
+"""Trigger the manual CI run (entry-manual-steer.yml) for the current branch.
 
-entry-manual.yml reads the "__ALL__" whitelist sentinel as "force a full
+entry-manual-steer.yml reads the "__ALL__" whitelist sentinel as "force a full
 deploy across all roles".
 """
 
@@ -11,10 +11,18 @@ import sys
 
 from cli.administration.deploy.ci import runs
 
-_WORKFLOW = "entry-manual.yml"
+_WORKFLOW = "entry-manual-steer.yml"
 _ALL = "__ALL__"
 
 
+def _fetch(run: str, repo: str) -> dict:
+    """The jobs and title of *run*, given as a URL or a bare id (which
+    resolves against *repo*, the current branch's own)."""
+    if run.isdigit():
+        return runs.fetch_run(run, repo=repo)
+    return runs.fetch_run(runs.run_id_from_url(run), repo=runs.slug_from_url(run))
+
+
 def main(argv: list[str] | None = None) -> int:
     p = argparse.ArgumentParser(
         prog="infinito administration deploy ci trigger",
@@ -23,7 +31,8 @@ def main(argv: list[str] | None = None) -> int:
             "Default: trigger every role. With --failed: the roles that "
             "failed in the last run form the priority line and the full "
             "run follows once they are green. With --apps: an explicit "
-            "role list as whitelist."
+            "role list as whitelist. Whenever a source run is read, its "
+            "configuration is carried over so the retrigger reproduces it."
         ),
     )
     group = p.add_mutually_exclusive_group()
@@ -35,8 +44,9 @@ def main(argv: list[str] | None = None) -> int:
         choices=("total", "swarm", "compose", "docker", "host"),
         metavar="{total,swarm,compose,host}",
         help=(
-            "Re-trigger roles that were not green in the last run as the "
-            "priority line; the remaining roles run after they succeed. "
+            "Re-trigger roles that were not green in the source run as the "
+            "priority line, together with the priority roles that run never "
+            "deployed at all; the remaining roles follow once they succeed. "
             "Optional scope: 'total' (default; failed in any mode), "
             "'swarm', 'compose', or 'host'."
         ),
@@ -52,9 +62,12 @@ def main(argv: list[str] | None = None) -> int:
         default=None,
         metavar="URL|ID",
         help=(
-            "Run URL or bare run id whose results determine the --failed apps "
-            "(a bare id resolves against the current branch's repo). Default: "
-            "the latest deploy run on the current branch."
+            "Run URL or bare run id to reproduce: its configuration (distros, "
+            "modes, lifecycles, filesystem, sequencing, mode_fail_fast, "
+            "workspace, instructions) is carried over, and with --failed its "
+            "results also pick the apps. A bare id resolves against the current "
+            "branch's repo. With --failed and no --run: the latest deploy run "
+            "on the current branch."
         ),
     )
     p.add_argument(
@@ -72,6 +85,19 @@ def main(argv: list[str] | None = None) -> int:
     branch = runs.current_branch()
     repo = runs.resolve_repo()
 
+    source = None
+    if args.run:
+        source = _fetch(args.run, repo)
+    elif args.failed is not None:
+        run = runs.find_last_deploy_run(branch, repo=repo)
+        if run is None:
+            print(
+                f"No CI run with deploy jobs found on {repo}@{branch}.",
+                file=sys.stderr,
+            )
+            return 1
+        source = {"jobs": run["_jobs"], "displayTitle": run.get("displayTitle", "")}
+
     whitelist = ""
     priority = ""
     if args.apps is not None:
@@ -81,40 +107,37 @@ def main(argv: list[str] | None = None) -> int:
         whitelist = apps
     elif args.failed is not None:
         scope = "docker" if args.failed == "compose" else args.failed
-        if args.run:
-            if args.run.isdigit():
-                jobs = runs.fetch_jobs(args.run, repo=repo)
-            else:
-                jobs = runs.fetch_jobs(
-                    runs.run_id_from_url(args.run), repo=runs.slug_from_url(args.run)
-                )
-        else:
-            run = runs.find_last_deploy_run(branch, repo=repo)
-            if run is None:
-                print(
-                    f"No CI run with deploy jobs found on {repo}@{branch}.",
-                    file=sys.stderr,
-                )
-                return 1
-            jobs = run["_jobs"]
-        failed = runs.failed_roles(
-            runs.parse_role_statuses(jobs), scope, strict=args.strict
+        statuses = runs.parse_role_statuses(source["jobs"])
+        failed = runs.failed_roles(statuses, scope, strict=args.strict)
+        untriggered = runs.untriggered_priority(
+            runs.dispatched_priority(source, repo), statuses
         )
-        if not failed:
+        if not failed and not untriggered:
             print(f"Nothing failed ({args.failed}) in that run; not triggering.")
             return 0
-        priority = " ".join(failed)
+        if untriggered:
+            print(f"Priority roles that never deployed: {' '.join(untriggered)}")
+        priority = " ".join(sorted(set(failed) | set(untriggered)))
     else:
         whitelist = _ALL
 
+    config: dict[str, str] = {}
+    if source is not None:
+        config = runs.config_from_run(source["displayTitle"], source["jobs"])
+
     if priority:
         label = f"priority {priority}, then the remaining roles"
     elif whitelist == _ALL:
         label = "all roles"
     else:
         label = whitelist
+    carried = ", ".join(f"{name}={value}" for name, value in config.items())
     print(f"Triggering {_WORKFLOW} on {repo}@{branch} for: {label}")
-    runs.dispatch_workflow(_WORKFLOW, branch, whitelist, priority=priority, repo=repo)
+    if carried:
+        print(f"Carrying over from the source run: {carried}")
+    runs.dispatch_workflow(
+        _WORKFLOW, branch, whitelist, priority=priority, config=config, repo=repo
+    )
     print("Dispatched. Watch with: infinito administration deploy ci status")
     return 0
 

cli/administration/deploy/dedicated/runner.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import datetime
+import json
 import os
 import subprocess
 import sys
@@ -35,27 +36,18 @@ def run_ansible_playbook(
     start_time = datetime.datetime.now(tz=datetime.UTC)
     print(f"\n▶️ Script started at: {start_time.isoformat()}\n")
 
-    # ---------------------------------------------------------
-    # 1) Cleanup Phase (wrapper-level)
-    # ---------------------------------------------------------
     if modes.get("MODE_CLEANUP", False):
         print("\n🧹 Cleaning up...\n", flush=True)
         run_make(repo_root, "clean")
     else:
         print("\n🧹 Cleanup skipped (MODE_CLEANUP not set or False)\n")
 
-    # ---------------------------------------------------------
-    # 2) Build Phase
-    # ---------------------------------------------------------
     if not skip_build:
         print("\n🛠️  Running project build (make setup)...\n")
         run_make(repo_root, "setup")
     else:
         print("\n🛠️  Build skipped (--skip-build)\n")
 
-    # ---------------------------------------------------------
-    # 3) `disable` env var consistency guard
-    # ---------------------------------------------------------
     try:
         assert_services_disabled_inventory_consistency_from_env(
             inventory_dir=Path(inventory).resolve().parent,
@@ -65,9 +57,6 @@ def run_ansible_playbook(
         print(f"\n[ERROR] {exc}\n", file=sys.stderr)
         sys.exit(1)
 
-    # ---------------------------------------------------------
-    # 4) Inventory Validation Phase
-    # ---------------------------------------------------------
     if modes.get("MODE_ASSERT") is False:
         print("\n🔍 Inventory assertion explicitly disabled (MODE_ASSERT=false)\n")
     else:
@@ -85,22 +74,17 @@ def run_ansible_playbook(
             )
             sys.exit(1)
 
-    # ---------------------------------------------------------
-    # 5) Build ansible-playbook command
-    # ---------------------------------------------------------
     cmd: list[str] = ["ansible-playbook", "-i", inventory, playbook_path]
 
     if limit:
         cmd.extend(["-l", limit])
 
-    # Wrapper-provided extra-vars first; user can override via passthrough -e later.
     if allowed_applications:
         joined = ",".join(allowed_applications)
         cmd.extend(["-e", f"APPLICATIONS_WHITELIST={joined}"])
 
-    for key, value in modes.items():
-        val = str(value).lower() if isinstance(value, bool) else str(value)
-        cmd.extend(["-e", f"{key}={val}"])
+    if modes:
+        cmd.extend(["-e", json.dumps(modes)])
 
     if password_file:
         cmd.extend(["--vault-password-file", password_file])
@@ -114,7 +98,6 @@ def run_ansible_playbook(
     if verbose:
         cmd.append("-" + "v" * verbose)
 
-    # Native ansible-playbook flags passthrough (must come last for override behavior)
     if ansible_args:
         cmd.extend(ansible_args)
 

cli/administration/deploy/development/README.md

@@ -26,7 +26,8 @@ flowchart TD
 ```
 
 - `initialize` / `reinstall` rebuild the inventory through `init.py`; this
-  is the only path that prunes the include set.
+  is the only path that prunes the include set. `deploy` re-derives the
+  same pruned set for its deploy ids (see below).
 - `update` reuses the existing inventory; `disable` reaches ansible as an
   extra-var that renders services disabled, but does not touch the include.
 
@@ -49,6 +50,43 @@ flowchart TD
     K --> L
 ```
 
+## deploy/cli.py: re-derive the same include
+
+`init.py` writes the inventory; `deploy/cli.py` decides which application
+ids to hand to `cli.administration.deploy.dedicated`. Both derive their
+per-round set from the same `plan_dev_inventory_matrix` closure and MUST
+apply the same `prune_orphans_after_disable` cut with the same
+round-merged services map. `validate_application_ids` rejects any id that
+is absent from the inventory, so a deploy-side set wider than the
+init-side set aborts the run before the first role deploys.
+
+```mermaid
+flowchart TD
+    PLAN["plan_dev_inventory_matrix(primary_apps)"] --> IR["include_R (per round)"]
+
+    IR --> INITP{"disable set?"}
+    INITP -->|yes| IP["prune_orphans_after_disable"]
+    INITP -->|no| IK["include_R"]
+    IP --> IW["build_dev_inventory → inventory dir"]
+    IK --> IW
+
+    IR --> DEPP{"disable set?"}
+    DEPP -->|yes| DP["prune_orphans_after_disable<br/>(same round_overrides)"]
+    DEPP -->|no| DK["include_R"]
+    DP --> DD["round_deploy_ids"]
+    DK --> DD
+
+    DD --> VAL{"validate_application_ids<br/>vs inventory"}
+    IW --> VAL
+    VAL -->|equal sets| RUN["ansible-playbook"]
+    VAL -->|deploy ⊃ inventory| ABORT["exit 1: not present in inventory"]:::abort
+    classDef abort fill:#fee,stroke:#c00;
+```
+
+Both sides pass `primary_apps` (not the round closure) as the BFS seed and
+the variant-merged services map for that round, so the reachability walk
+sees the same topology the inventory baked.
+
 ## Closure resolution (per round)
 
 `plan_dev_inventory_matrix` → `_resolve_round_include` runs

cli/administration/deploy/development/compose.py

@@ -190,7 +190,7 @@ class Compose:
         args: list[str] = ["up", "-d"]
         if no_build:
             args.append("--no-build")
-        if self.profile.registry_cache_active():
+        if self.profile.owns_cache_stack():
             self._generate_package_frontend_certs(env)
             args += ["registry-cache", "package-cache", "package-cache-frontend"]
         args += ["coredns", "infinito"]
@@ -199,8 +199,9 @@ class Compose:
 
         self.wait_for_healthy()
 
-        if self.profile.registry_cache_active():
+        if self.profile.owns_cache_stack():
             self._bootstrap_package_cache(env)
+        if self.profile.registry_cache_active():
             self._install_package_frontend_ca_in_runner()
 
         if run_entry_init:

cli/administration/deploy/development/coredns.py

@@ -4,14 +4,12 @@ import itertools
 import os
 import shutil
 import subprocess
+import tempfile
 from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from pathlib import Path
 
 from utils.env.parser import parse_static_env
 
-if TYPE_CHECKING:
-    from pathlib import Path
-
 
 @dataclass(frozen=True)
 class CoreDNSCorefileRenderer:
@@ -116,13 +114,16 @@ class CoreDNSCorefileRenderer:
 
         env = self._load_env_file(env_file)
 
-        tmp_file = out_file.with_suffix(out_file.suffix + ".tmp")
+        fd, tmp_name = tempfile.mkstemp(
+            dir=out_file.parent, prefix=out_file.name + ".", suffix=".tmp"
+        )
+        tmp_file = Path(tmp_name)
 
         self._log("Rendering Corefile via envsubst (atomic write)")
         try:
             with (
                 tmpl_file.open("r", encoding="utf-8") as fin,
-                tmp_file.open("w", encoding="utf-8") as fout,
+                os.fdopen(fd, "w", encoding="utf-8") as fout,
             ):
                 subprocess.check_call(
                     [envsubst],
@@ -131,19 +132,20 @@ class CoreDNSCorefileRenderer:
                     env=env,
                     cwd=self.repo_root,
                 )
+
+            size = tmp_file.stat().st_size
+            if size == 0:
+                raise RuntimeError(f"Rendered Corefile is empty: {tmp_file}")
+
+            tmp_file.chmod(0o644)
+            tmp_file.replace(out_file)
         except OSError as exc:
             raise RuntimeError(
                 f"Failed to write temporary Corefile: {tmp_file}"
             ) from exc
+        finally:
+            tmp_file.unlink(missing_ok=True)
 
-        if not tmp_file.exists():
-            raise RuntimeError("envsubst did not produce an output file")
-
-        size = tmp_file.stat().st_size
-        if size == 0:
-            raise RuntimeError(f"Rendered Corefile is empty: {tmp_file}")
-
-        tmp_file.replace(out_file)
         self._log(f"Rendered successfully ({size} bytes)")
 
         if show_preview:

cli/administration/deploy/development/deploy/cli.py

@@ -9,7 +9,13 @@ from cli.administration.deploy.development.common import (
     make_compose,
     resolve_container,
 )
-from cli.administration.deploy.development.inventory import plan_dev_inventory_matrix
+from cli.administration.deploy.development.inventory import (
+    _build_services_overrides_for_round as build_services_overrides_for_round,
+)
+from cli.administration.deploy.development.inventory import (
+    plan_dev_inventory_matrix,
+    prune_orphans_after_disable,
+)
 from cli.administration.deploy.development.variant_select import (
     add_variant_args,
     apply_variant_filter,
@@ -132,9 +138,28 @@ def handler(args: argparse.Namespace) -> int:
         include_roles,
         purge_roles,
     ) in enumerate(plan):
-        round_deploy_ids = [
-            role for role in include_roles if role not in disabled_app_ids
-        ]
+        if disabled_app_ids:
+            round_overrides = build_services_overrides_for_round(
+                roles_dir=str(compose.repo_root / "roles"),
+                round_index=round_index,
+                primary_app_variants={
+                    a: round_variants[a] for a in primary_app_ids if a in round_variants
+                },
+            )
+            round_deploy_ids, pruned = prune_orphans_after_disable(
+                include=include_roles,
+                primary_apps=primary_app_ids,
+                disabled_app_ids=disabled_app_ids,
+                services_overrides=round_overrides,
+            )
+            round_deploy_ids = list(round_deploy_ids)
+            if pruned:
+                print(
+                    f">>> `disable` orphan-pruned {len(pruned)} transitive dep(s) "
+                    f"from round {round_index}: {','.join(pruned)}"
+                )
+        else:
+            round_deploy_ids = list(include_roles)
 
         if plan_index > 0:
             purge_targets = [

cli/administration/deploy/development/down.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import os
-import shutil
+import subprocess
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -24,8 +24,17 @@ def _wipe_docker_root(docker_root: Path) -> None:
         print(f">>> Docker root does not exist, nothing to clean: {docker_root}")
         return
     print(f">>> CI cleanup: wiping Docker root: {docker_root}")
-    shutil.rmtree(docker_root, ignore_errors=True)
-    docker_root.mkdir(parents=True, exist_ok=True)
+    subprocess.run(["sudo", "rm", "-rf", str(docker_root)], check=True)
+    subprocess.run(["sudo", "mkdir", "-p", str(docker_root)], check=True)
+    subprocess.run(
+        ["sudo", "chown", f"{os.getuid()}:{os.getgid()}", str(docker_root)],
+        check=True,
+    )
+    leftovers = sorted(entry.name for entry in docker_root.iterdir())
+    if leftovers:
+        raise RuntimeError(
+            f"Docker root survived the wipe: {docker_root} still holds {leftovers}"
+        )
 
 
 def _should_wipe_docker_root() -> bool:

cli/administration/deploy/development/env.py

@@ -2,16 +2,29 @@ from __future__ import annotations
 
 import os
 
-VALID_DISTROS: tuple[str, ...] = ("arch", "debian", "ubuntu", "fedora", "centos")
+from utils.distros import distro_names
+
+VALID_DISTROS: tuple[str, ...] = distro_names()
 
 
 def compose_file_args() -> list[str]:
-    """Compose `-f` flags shared by up and down flows."""
+    """Compose `-f` flags shared by up and down flows.
+
+    Each override is gated on the resource it needs being present, not on the
+    instance slot: an override whose `:?` guard is unsatisfied would abort the
+    stack outright, so a checkout that was never handed a shared git dir or
+    cache network simply does not layer that file in.
+    """
     from .profile import Profile
 
+    profile = Profile()
     out = ["-f", "compose.yml"]
-    if Profile().registry_cache_active():
+    if profile.shared_git_dir():
+        out += ["-f", "compose/worktree.override.yml"]
+    if profile.registry_cache_active():
         out += ["-f", "compose/cache.override.yml"]
+        if profile.shared_cache_network():
+            out += ["-f", "compose/cache.shared.override.yml"]
     if (os.environ.get("INFINITO_PUBLISH_PORTS") or "").strip().lower() == "false":
         out += ["-f", "compose/noports.override.yml"]
     return out
@@ -23,7 +36,7 @@ def resolve_distro() -> str:
     if not distro:
         raise SystemExit(
             "INFINITO_DISTRO is not set. Run 'make dotenv' (or source scripts/meta/env/load.sh) "
-            "or export INFINITO_DISTRO=<arch|debian|ubuntu|fedora|centos> "
+            f"or export INFINITO_DISTRO=<{'|'.join(VALID_DISTROS)}> "
             "before invoking cli.administration.deploy.development."
         )
     if distro not in VALID_DISTROS:

cli/administration/deploy/development/init.py

@@ -54,12 +54,6 @@ def add_parser(sub: argparse._SubParsersAction) -> None:
         help="Comma-separated list of application ids to include (no deps resolution).",
     )
 
-    p.add_argument(
-        "--threshold-gib",
-        type=int,
-        default=100,
-        help="Free-space threshold (GiB) below which STORAGE_CONSTRAINED is enabled (default: 100).",
-    )
     p.add_argument(
         "--force-storage-constrained",
         choices=["true", "false"],
@@ -112,12 +106,11 @@ def handler(args: argparse.Namespace) -> int:
             raise SystemExit("--vars must be a JSON object")
         extra_vars = parsed
 
-    if args.force_storage_constrained is not None:
-        storage_constrained = args.force_storage_constrained == "true"
-    else:
-        storage_constrained = detect_storage_constrained(
-            compose, threshold_gib=int(args.threshold_gib)
-        )
+    forced_storage_constrained = (
+        None
+        if args.force_storage_constrained is None
+        else args.force_storage_constrained == "true"
+    )
 
     plan = plan_dev_inventory_matrix(
         roles_dir=str(compose.repo_root / "roles"),
@@ -133,6 +126,7 @@ def handler(args: argparse.Namespace) -> int:
     services_disabled = os.environ.get("disable", "")
     roles_dir = str(compose.repo_root / "roles")
     built_includes: dict[str, tuple[str, ...]] = {}
+    round_storage_constrained: dict[str, bool] = {}
     for _round_index, inv_dir, round_variants, include_R, _purge_set in plan:
         if disabled_app_ids:
             round_overrides = build_services_overrides_for_round(
@@ -162,6 +156,12 @@ def handler(args: argparse.Namespace) -> int:
             )
             continue
         built_includes[inv_dir] = round_include
+        storage_constrained = (
+            forced_storage_constrained
+            if forced_storage_constrained is not None
+            else detect_storage_constrained(compose, primary_apps, round_variants)
+        )
+        round_storage_constrained[inv_dir] = storage_constrained
         spec = DevInventorySpec(
             inventory_dir=inv_dir,
             include=round_include,
@@ -181,20 +181,20 @@ def handler(args: argparse.Namespace) -> int:
         print(
             f">>> Inventory initialized at {inv_dir} "
             f"(include={','.join(shown)} "
-            f"storage_constrained={storage_constrained}){suffix}"
+            f"storage_constrained={round_storage_constrained.get(inv_dir)}){suffix}"
         )
     else:
         print(
             f">>> Matrix inventory initialized in {len(plan)} folders "
-            f"(primary_apps={','.join(primary_apps)} "
-            f"storage_constrained={storage_constrained}):"
+            f"(primary_apps={','.join(primary_apps)}):"
         )
         for round_index, inv_dir, round_variants, include_R, _purge_set in plan:
             non_zero = {a: i for a, i in round_variants.items() if i}
             shown = built_includes.get(inv_dir, include_R)
             print(
                 f"    [round {round_index}] {inv_dir} "
-                f"include={','.join(shown)}"
+                f"include={','.join(shown)} "
+                f"storage_constrained={round_storage_constrained.get(inv_dir)}"
                 + (f"  variants={non_zero}" if non_zero else "")
             )
     return 0

cli/administration/deploy/development/profile.py

@@ -19,3 +19,25 @@ class Profile:
     def registry_cache_active(self) -> bool:
         """True iff the cache stack should be loaded (local dev only)."""
         return not self.is_ci()
+
+    def instance(self) -> int:
+        """Slot index of this checkout; 0 is the primary one.
+
+        Bookkeeping only: the compose layering below keys off the concrete
+        resources a worktree was handed, never off this number, so a stray
+        slot cannot make compose demand a value nobody set.
+        """
+        raw = (os.environ.get("INFINITO_INSTANCE") or "0").strip()
+        return int(raw) if raw.isdigit() else 0
+
+    def shared_git_dir(self) -> str:
+        """Path of the checkout's shared .git directory, empty when it owns one."""
+        return (os.environ.get("INFINITO_GIT_COMMON_DIR") or "").strip()
+
+    def shared_cache_network(self) -> str:
+        """Docker network of a cache stack owned elsewhere, empty when unset."""
+        return (os.environ.get("INFINITO_CACHE_NETWORK") or "").strip()
+
+    def owns_cache_stack(self) -> bool:
+        """True iff this instance runs the cache stack rather than sharing it."""
+        return self.registry_cache_active() and not self.shared_cache_network()

cli/administration/deploy/development/storage.py

@@ -2,32 +2,30 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
+from utils.storage.constrained import is_constrained, required_storage_bytes
+
 if TYPE_CHECKING:
     from .compose import Compose
 
 
-def detect_storage_constrained(compose: Compose, *, threshold_gib: int = 100) -> bool:
+def detect_storage_constrained(
+    compose: Compose, app_ids: list[str], variants: dict[str, int] | None = None
+) -> bool:
     """
-    Return True if the filesystem that contains DockerRootDir has less than
-    `threshold_gib` GiB free space.
+    Return True if the declared storage need of app_ids and their transitive
+    dependencies, at the given variant overlay, exceeds the free space on the
+    filesystem holding DockerRootDir.
 
     We intentionally measure the DockerRootDir filesystem because this is where
     images/volumes/build cache usually grow (especially in CI / Docker-in-Docker).
     """
-    threshold_bytes = threshold_gib * 1024 * 1024 * 1024
-
     cmd = [
         "bash",
         "-lc",
         r"""
 set -euo pipefail
-root="$(docker info -f '{{.DockerRootDir}}' 2>/dev/null || true)"
-if [ -z "${root}" ]; then
-  root="/var/lib/docker"
-fi
-
-free="$(df -PB1 "${root}" | awk 'NR==2{print $4}')"
-printf "%s\n" "${free}"
+root="$(docker info -f '{{.DockerRootDir}}')"
+df -PB1 "${root}" | awk 'NR==2{print $4}'
 """,
     ]
 
@@ -35,10 +33,12 @@ printf "%s\n" "${free}"
     if r.returncode != 0:
         return False
 
-    txt = (r.stdout or "").strip()
     try:
-        free_bytes = int(txt)
+        free_bytes = int((r.stdout or "").strip())
     except ValueError:
         return False
 
-    return free_bytes < threshold_bytes
+    return is_constrained(
+        free_bytes=free_bytes,
+        required_bytes=required_storage_bytes(app_ids, variants),
+    )

cli/administration/inventory/provision/cli.py

@@ -35,20 +35,6 @@ def _resolve_inventory_file(
     return (inventory_dir / "devices.yml").resolve()
 
 
-def _resolve_roles_dir(project_root: Path, roles_dir_arg: str | None) -> Path:
-    return (
-        Path(roles_dir_arg) if roles_dir_arg else (project_root / "roles")
-    ).resolve()
-
-
-def _resolve_categories_file(roles_dir: Path, categories_file_arg: str | None) -> Path:
-    return (
-        Path(categories_file_arg)
-        if categories_file_arg
-        else (roles_dir / "categories.yml")
-    ).resolve()
-
-
 def _resolve_mirrors_file(project_root: Path, mirror_arg: str | None) -> Path | None:
     """
     --mirror can be used in two ways:
@@ -60,7 +46,6 @@ def _resolve_mirrors_file(project_root: Path, mirror_arg: str | None) -> Path |
     if mirror_arg is None:
         return None
 
-    # argparse uses const="mirrors.yml" when flag is present without value
     candidate = Path(mirror_arg)
     if not candidate.is_absolute():
         candidate = (project_root / candidate).resolve()
@@ -121,14 +106,6 @@ def main(argv: list[str] | None = None) -> int:
         default=None,
         help="Vault password file path. Default: <inventory-dir>/.password",
     )
-    parser.add_argument(
-        "--roles-dir", default=None, help="Path to roles/ (default: <repo-root>/roles)."
-    )
-    parser.add_argument(
-        "--categories-file",
-        default=None,
-        help="Path to roles/categories.yml (default: <roles-dir>/categories.yml).",
-    )
     parser.add_argument(
         "--workers",
         type=int,
@@ -144,7 +121,6 @@ def main(argv: list[str] | None = None) -> int:
         ),
     )
 
-    # New: mirroring support
     parser.add_argument(
         "--mirror",
         nargs="?",
@@ -182,8 +158,7 @@ def main(argv: list[str] | None = None) -> int:
     project_root = detect_project_root(Path(__file__).resolve())
     env = build_env_with_project_root(project_root)
 
-    roles_dir = _resolve_roles_dir(project_root, args.roles_dir)
-    categories_file = _resolve_categories_file(roles_dir, args.categories_file)
+    roles_dir = (project_root / "roles").resolve()
     mirrors_file = _resolve_mirrors_file(project_root, args.mirror)
 
     inventory_dir = Path(args.inventory_dir).resolve()
@@ -192,7 +167,6 @@ def main(argv: list[str] | None = None) -> int:
     inventory_file = _resolve_inventory_file(inventory_dir, args.inventory_file)
     host_vars_file = (inventory_dir / "host_vars" / f"{args.host}.yml").resolve()
 
-    # Vault password file
     if args.vault_password_file:
         vault_password_file = Path(args.vault_password_file).resolve()
     else:
@@ -225,10 +199,7 @@ def main(argv: list[str] | None = None) -> int:
     )
     dyn_inv = generate_dynamic_inventory(
         host=args.host,
-        roles_dir=roles_dir,
-        categories_file=categories_file,
         tmp_inventory=tmp_inventory,
-        project_root=project_root,
         env=env,
     )
 
@@ -242,7 +213,6 @@ def main(argv: list[str] | None = None) -> int:
     dyn_children = (dyn_inv.get("all", {}) or {}).get("children", {}) or {}
     application_ids = sorted(dyn_children.keys())
 
-    # Merge inventory file
     if inventory_file.exists():
         print(f"[INFO] Merging into existing inventory: {inventory_file}")
         base_inv = load_yaml(inventory_file)
@@ -253,7 +223,6 @@ def main(argv: list[str] | None = None) -> int:
     merged_inv = _merge_inventories(base_inv, dyn_inv, host=args.host)
     dump_yaml(inventory_file, merged_inv)
 
-    # Host vars
     print(f"[INFO] Ensuring host_vars for host '{args.host}' at {host_vars_file}")
     ensure_host_vars_file(
         host_vars_file=host_vars_file,
@@ -283,7 +252,6 @@ def main(argv: list[str] | None = None) -> int:
                     f"--app-variants[{app_id!r}] must be an integer, got {raw_index!r}: {exc}"
                 )
 
-    # Credentials
     if application_ids:
         print(
             f"[INFO] Generating credentials for {len(application_ids)} applications..."
@@ -317,13 +285,10 @@ def main(argv: list[str] | None = None) -> int:
         )
         apply_vars_overrides(host_vars_file=host_vars_file, json_str=args.vars)
 
-    # Mirror overrides should win (requested behavior: "überschreiben")
     if mirrors_file is not None:
         print(f"[INFO] Applying mirror overrides from: {mirrors_file}")
         apply_mirror_overrides(host_vars_file=host_vars_file, mirrors_file=mirrors_file)
 
-    # Disable services listed in the `disable` env var (space- or comma-separated).
-    # Also removes the provider roles from the inventory.
     apply_services_disabled_from_env(
         host_vars_file=host_vars_file,
         inventory_file=inventory_file,

cli/administration/inventory/provision/inventory_generator.py

@@ -12,10 +12,7 @@ if TYPE_CHECKING:
 
 def generate_dynamic_inventory(
     host: str,
-    roles_dir: Path,
-    categories_file: Path,
     tmp_inventory: Path,
-    project_root: Path,
     env: dict[str, str] | None,
 ) -> dict[str, Any]:
     """

cli/contributing/mirror/providers.py

@@ -4,6 +4,8 @@ import subprocess
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING
 
+from utils.docker.mirror import mirror_image_base
+
 if TYPE_CHECKING:
     import argparse
 
@@ -43,7 +45,13 @@ class GHCRProvider(RegistryProvider):
         return cls(args.ghcr_namespace, args.ghcr_repository, args.ghcr_prefix)
 
     def image_base(self, image: ImageRef) -> str:
-        return f"ghcr.io/{self.namespace}/{self.repository}/{self.prefix}/{image.registry}/{image.name}"
+        return mirror_image_base(
+            image.registry,
+            image.name,
+            namespace=self.namespace,
+            repository=self.repository,
+            prefix=self.prefix,
+        )
 
     def tag_exists(self, image: ImageRef) -> bool:
         """
@@ -100,16 +108,13 @@ class GHCRProvider(RegistryProvider):
         src = f"docker://{image.source}"
 
         try:
-            # Fast path
             self._run_copy(src=src, dest=dest)
 
         except subprocess.CalledProcessError as e:
-            # Always print skopeo output for debugging
             output = (e.stdout or "") + (e.stderr or "")
             if output.strip():
                 print(output, flush=True)
 
-            # Fallback: force recompress (avoids cross-repo blob reuse)
             if self._looks_like_blob_reuse_problem(e):
                 self._run_copy(
                     src=src,

cli/meta/categories/README.md

@@ -1,3 +1,3 @@
 # Categories 🗂️
 
-Inspect role categories declared under `roles/categories.yml`, including the set of paths that the dispatcher treats as invokable.
+Inspect role categories declared under `meta/categories.yml`, including the set of paths that the dispatcher treats as invokable.

cli/meta/categories/invokable/command.py

@@ -1,10 +1,13 @@
 #!/usr/bin/env python3
 """
-CLI for extracting invokable or non-invokable role paths from a nested roles YAML file.
+CLI for extracting invokable or non-invokable role paths from the category SPOT.
 
 Layout assumption: this module sits under ``cli/`` and reads
-``<repo_root>/roles/categories.yml`` plus filter plugins under
+``<repo_root>/meta/categories.yml`` plus filter plugins under
 ``<repo_root>/plugins/filter/``.
+
+The two ``plugins.filter.invokable_paths`` imports are bound at module
+level so ``unittest.mock.patch`` can replace them in tests.
 """
 
 from __future__ import annotations
@@ -14,9 +17,6 @@ import sys
 
 import yaml
 
-# IMPORTANT:
-# These imports must exist at module level so unittest.mock.patch()
-# can replace them in tests.
 from plugins.filter.invokable_paths import (
     get_invokable_paths,
     get_non_invokable_paths,
@@ -27,14 +27,7 @@ from . import PROJECT_ROOT
 
 def main() -> None:
     parser = argparse.ArgumentParser(
-        description="Extract invokable or non-invokable role paths from a nested roles YAML file."
-    )
-
-    parser.add_argument(
-        "roles_file",
-        nargs="?",
-        default=None,
-        help="Path to roles/categories.yml (optional).",
+        description="Extract invokable or non-invokable role paths from the category SPOT."
     )
 
     parser.add_argument(
@@ -60,15 +53,14 @@ def main() -> None:
 
     args = parser.parse_args()
 
-    # Ensure repo root is importable (CLI use outside repo root)
     repo_root = PROJECT_ROOT
     sys.path.insert(0, str(repo_root))
 
     try:
         if args.non_invokable:
-            paths = get_non_invokable_paths(args.roles_file, args.suffix)
+            paths = get_non_invokable_paths(suffix=args.suffix)
         else:
-            paths = get_invokable_paths(args.roles_file, args.suffix)
+            paths = get_invokable_paths(suffix=args.suffix)
 
     except FileNotFoundError as e:
         print(f"Error: {e}", file=sys.stderr)

cli/meta/ci/dev_runtime.py

@@ -0,0 +1,52 @@
+"""Answer the development-runtime matrix questions from the distro SPOT.
+
+Exists so workflow steps can ask for the matrix, and for the distro behind a
+matrix entry, without embedding a python one-liner per question.
+
+    python3 -m cli.meta.ci.dev_runtime images
+    python3 -m cli.meta.ci.dev_runtime distro --image fedora:latest
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from utils.distros import (
+    FILE_META_DISTROS,
+    dev_runtime_images,
+    distro_of_dev_runtime_image,
+)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    commands = parser.add_subparsers(dest="command", required=True)
+    commands.add_parser("images", help="Print the matrix images as a JSON array.")
+    distro = commands.add_parser(
+        "distro", help="Print the distro a matrix image belongs to."
+    )
+    distro.add_argument(
+        "--image", required=True, help="Matrix image, e.g. fedora:latest."
+    )
+    args = parser.parse_args(argv)
+
+    if args.command == "images":
+        images = list(dev_runtime_images())
+        if not images:
+            print(
+                f"{FILE_META_DISTROS} declares no dev runtime image; "
+                "an empty matrix would skip every workspace job silently",
+                file=sys.stderr,
+            )
+            return 1
+        print(json.dumps(images))
+        return 0
+
+    print(distro_of_dev_runtime_image(args.image))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

cli/meta/ci/guide_select.py

@@ -1,20 +1,18 @@
-"""Pick the role, distro, and deploy mode for the Guide test.
+"""Pick the role and deploy mode for the Guide test.
 
 Role selection: an explicit ``--role`` wins; otherwise a random pick from
 ``--priority``, then ``--whitelist``, then every invokable role inside the
 tested lifecycle envelope (``INFINITO_LIFECYCLES`` in ``default.env``) whose
-guide deploy mode is not skipped via ``meta/tests.yml``. The distro is
-a random pick from ``INFINITO_DISTROS``, resolved to that distro's pkgmgr base
-image (it ships systemd plus the build toolchain, so a host role can boot
-systemd and install straight inside it). The mode is ``host`` for a role that
-ships no container stack (installed straight onto the machine), else
-``compose``. Output is ``key=value`` lines for ``$GITHUB_ENV``.
+guide deploy mode is not skipped via ``meta/tests.yml``. The mode is ``host``
+for a role that ships no container stack (installed straight onto the machine),
+else ``compose``. The distro is not picked here: the role is replayed on every
+distro by ``scripts/tests/deploy/distros.sh``. Output is ``key=value`` lines
+for ``$GITHUB_ENV``.
 """
 
 from __future__ import annotations
 
 import argparse
-import os
 import random
 import sys
 from typing import TYPE_CHECKING
@@ -32,10 +30,6 @@ from utils.roles.validation.invokable import (
 if TYPE_CHECKING:
     from pathlib import Path
 
-_RUNTIME_IMAGE = "ghcr.io/kevinveenbirkenbach/pkgmgr-{distro}:stable"
-
-_DISTROS = ("arch", "debian", "ubuntu", "fedora", "centos")
-
 
 def _tokens(raw: str) -> list[str]:
     return [t for t in raw.split() if t]
@@ -75,18 +69,10 @@ def main(argv: list[str] | None = None) -> int:
             return 1
         role = random.choice(pool)  # noqa: S311 - test-role pick, not cryptographic
 
-    distros = [d for d in _tokens(os.environ["INFINITO_DISTROS"]) if d in _DISTROS]
-    if not distros:
-        print("guide_select: no known distro in INFINITO_DISTROS", file=sys.stderr)
-        return 1
-    distro = random.choice(distros)  # noqa: S311 - test-distro pick, not cryptographic
-
     mode = _guide_mode(PROJECT_ROOT / "roles" / role)
 
     print(f"GUIDE_ROLE={role}")
     print(f"GUIDE_MODE={mode}")
-    print(f"INFINITO_DISTRO={distro}")
-    print(f"GUIDE_RUNTIME_IMAGE={_RUNTIME_IMAGE.format(distro=distro)}")
     return 0
 
 

cli/meta/ci/image_ref.py

@@ -0,0 +1,59 @@
+"""Render a distro image reference from the ``meta/distros.yml`` SPOT.
+
+Exists so shell and workflow steps can compose the same reference the env
+layer composes, instead of spelling ``ghcr.io/...`` out a second time.
+
+    python3 -m cli.meta.ci.image_ref --kind environment \\
+        --distro debian --owner acme --repository infinito-nexus --tag ci-abc
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from utils.distros import (
+    IMAGE_ENVIRONMENT,
+    IMAGE_PKGMGR,
+    environment_image,
+    pkgmgr_image,
+)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--kind",
+        required=True,
+        choices=(IMAGE_PKGMGR, IMAGE_ENVIRONMENT),
+        help="Which image reference to render.",
+    )
+    parser.add_argument("--distro", required=True, help="Distro id, e.g. debian.")
+    parser.add_argument("--owner", required=True, help="Registry owner.")
+    parser.add_argument("--tag", required=True, help="Image tag.")
+    parser.add_argument(
+        "--repository",
+        default="",
+        help="Repository name; required for --kind environment.",
+    )
+    args = parser.parse_args(argv)
+
+    if args.kind == IMAGE_PKGMGR:
+        print(pkgmgr_image(args.distro, owner=args.owner, tag=args.tag))
+        return 0
+
+    if not args.repository:
+        parser.error("--repository is required for --kind environment")
+    print(
+        environment_image(
+            args.distro,
+            owner=args.owner,
+            repository=args.repository,
+            tag=args.tag,
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

cli/meta/ci/plan.py

@@ -32,6 +32,7 @@ from cli.meta.roles.applications.complexity.model import (
 )
 from cli.meta.roles.applications.complexity.render import _dwidth
 from utils.cache.files import PROJECT_ROOT
+from utils.roles.display import display_names
 from utils.symbol_glossary import to_emoji
 
 _STAR = to_emoji("priority")
@@ -181,6 +182,10 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--cli", action="store_true")
     args = parser.parse_args(argv)
 
+    codec = display_names()
+    args.whitelist = codec.decode_list(args.whitelist)
+    args.priority = codec.decode_list(args.priority)
+
     if args.lifecycles.strip():
         os.environ["INFINITO_LIFECYCLES"] = args.lifecycles
 

cli/meta/ci/query.py

@@ -34,6 +34,7 @@ import sys
 
 from cli.meta.ci import slots
 from utils.cache.files import PROJECT_ROOT, read_text
+from utils.roles.display import display_names
 
 MODES = ("compose", "swarm", "host")
 
@@ -41,7 +42,7 @@ MODES = ("compose", "swarm", "host")
 def expands_variants(mode: str) -> bool:
     """Whether *mode* queries, ranks and triggers one CI job per variant.
     Swarm does (``role#variant`` tokens through the whole chain); compose
-    and host bundle every variant into whole-role jobs (variant_bundles
+    and host bundle every variant into whole-role jobs (variant.bundles
     SPOT), so their query rows stay whole-role too."""
     return mode == "swarm"
 
@@ -206,8 +207,9 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--format", choices=("json",), dest="fmt")
     args = parser.parse_args(argv)
 
-    whitelist = os.environ["INFINITO_WHITELIST"]
-    blacklist = os.environ["INFINITO_BLACKLIST"]
+    codec = display_names()
+    whitelist = codec.decode_list(os.environ["INFINITO_WHITELIST"])
+    blacklist = codec.decode_list(os.environ["INFINITO_BLACKLIST"])
 
     if args.matrix:
         return subprocess.run(

cli/meta/ci/sequencing.py

@@ -0,0 +1,107 @@
+"""Decide whether a CI line deploys its modes in sequence or in parallel.
+
+Usage:
+  python -m cli.meta.ci.sequencing [--modes "swarm compose host"]
+      [--whitelist "..."] [--blacklist "..."] [--lifecycles "..."]
+      [--choice auto|serial|parallel]
+
+A *line* is the priority (⭐) or the regular (🔁) half of an orchestrator
+run. Its job count is the sum of the deploy-matrix entries every active
+mode contributes, counted on the row basis the discover steps use: swarm
+selections are ``role#variant`` tokens mapping 1:1 onto jobs, compose and
+host selections are whole roles whose variants pack into bundles
+(utils.github.variant.bundles, the same packing the matrix runs).
+
+GitHub cancels a job that sat queued for 24 hours. A line that starts every
+mode at once floods the runner pool, so the tail of its matrix waits past
+that cut and dies unrun. ``auto`` therefore serialises the modes -- swarm,
+then compose, then host, heaviest first -- once the line exceeds
+``INFINITO_CI_SEQUENTIAL_THRESHOLD`` jobs, which restarts the queue clock
+per mode. Below the threshold the modes run in parallel for wall-clock.
+
+The count omits the runner-storage filter scripts/meta/resolve/apps.sh
+applies inside GitHub Actions, so it can only overestimate, erring towards
+the sequential layout that cannot be cancelled.
+
+Writes ``jobs=<n>`` and ``sequencing=<serial|parallel>``, ready to append to
+``$GITHUB_OUTPUT``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+
+from cli.meta.ci.query import MODES, discover, expands_variants
+from utils.cache.applications import get_variants
+from utils.github.variant.bundles import compose_bundle_counts
+from utils.roles.display import display_names
+
+CHOICES = ("auto", "serial", "parallel")
+
+
+def mode_jobs(mode: str, *, whitelist: str, blacklist: str, lifecycles: str) -> int:
+    """Deploy-matrix jobs *mode* contributes to the line."""
+    selection = discover(
+        mode, whitelist=whitelist, blacklist=blacklist, lifecycles=lifecycles
+    )
+    if expands_variants(mode):
+        return len(selection)
+    return sum(compose_bundle_counts(selection, get_variants()).values())
+
+
+def line_jobs(modes: str, *, whitelist: str, blacklist: str, lifecycles: str) -> int:
+    return sum(
+        mode_jobs(mode, whitelist=whitelist, blacklist=blacklist, lifecycles=lifecycles)
+        for mode in MODES
+        if mode in modes.split()
+    )
+
+
+def threshold() -> int:
+    return int(os.environ["INFINITO_CI_SEQUENTIAL_THRESHOLD"])
+
+
+def decide(jobs: int | None, choice: str) -> str:
+    """``choice`` verbatim unless it is ``auto``, which compares *jobs*
+    against the threshold."""
+    if choice != "auto":
+        return choice
+    return "serial" if jobs > threshold() else "parallel"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Decide serial or parallel deploy modes for one CI line."
+    )
+    parser.add_argument("--modes", default=" ".join(MODES))
+    parser.add_argument("--whitelist", default="")
+    parser.add_argument("--blacklist", default="")
+    parser.add_argument("--lifecycles", default="")
+    parser.add_argument("--choice", default="auto", choices=CHOICES)
+    args = parser.parse_args(argv)
+
+    codec = display_names()
+    args.whitelist = codec.decode_list(args.whitelist)
+    args.blacklist = codec.decode_list(args.blacklist)
+
+    if args.lifecycles.strip():
+        os.environ["INFINITO_LIFECYCLES"] = args.lifecycles
+
+    jobs = None
+    if args.choice == "auto":
+        jobs = line_jobs(
+            args.modes,
+            whitelist=args.whitelist,
+            blacklist=args.blacklist,
+            lifecycles=args.lifecycles,
+        )
+
+    print(f"jobs={'' if jobs is None else jobs}")
+    print(f"sequencing={decide(jobs, args.choice)}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

cli/meta/ci/slots.py

@@ -50,12 +50,16 @@ _DEPLOY_CALLERS = frozenset(
     {
         "test-deploy-single-node",
         "test-deploy-single-node-priority",
+        "test-deploy-compose",
+        "test-deploy-compose-priority",
+        "test-deploy-host",
+        "test-deploy-host-priority",
         "test-deploy-swarm",
         "test-deploy-swarm-priority",
     }
 )
 
-_ORCHESTRATOR = ".github/workflows/ci-orchestrator.yml"
+_ORCHESTRATOR = ".github/workflows/call-orchestrator.yml"
 _DYNAMIC_MATRIX_ESTIMATE = 5
 _DEFAULT_JOB_LIMIT = 256
 _MIN_MODE_SLOTS = 3

cli/meta/ci/subset_roles.py

@@ -2,14 +2,14 @@
 
 Activated by the ``🧩 Subset`` label: the entry-pull-request workflow only
 runs this module when the label is present. It gives fork contributors the
-selective-CI capability that ``entry-manual.yml`` (``workflow_dispatch``)
+selective-CI capability that ``entry-manual-steer.yml`` (``workflow_dispatch``)
 grants maintainers, which forks cannot trigger.
 
 It reads the PR body from ``$PR_BODY``, extracts the first fenced code block
 whose YAML defines a ``roles:`` list, validates that every listed id is an
 existing ``roles/<id>`` directory, and writes ``whitelist`` (space-separated)
-and ``roles_only=true`` to ``$GITHUB_OUTPUT`` -- the same outputs the
-diff-derived resolver produces.
+to ``$GITHUB_OUTPUT`` -- the same output the diff-derived resolver
+produces.
 
 Unlike the fail-safe diff resolver (which widens to ``__ALL__`` on any
 problem), the subset path is strict: invalid YAML, a missing/empty ``roles:``
@@ -95,7 +95,6 @@ def main() -> None:
     if github_output:
         with Path(github_output).open("a", encoding="utf-8") as handle:
             handle.write(f"whitelist={whitelist}\n")
-            handle.write("roles_only=true\n")
 
 
 if __name__ == "__main__":

cli/meta/roles/applications/complexity/model.py

@@ -8,7 +8,7 @@ import subprocess
 from typing import TYPE_CHECKING, Any, NamedTuple
 
 from utils.cache.applications import get_variants
-from utils.github.variant_bundles import compose_bundle_counts
+from utils.github.variant.bundles import compose_bundle_counts
 from utils.roles.lifecycle import tested_lifecycles
 from utils.roles.meta_lookup import (
     MetaServicesShapeError,

cli/meta/roles/order/run/__main__.py

@@ -8,11 +8,11 @@ Group-agnostic role call order.
 - 📦 Phase 2 (main): every remaining invokable role that is NOT a preload
   service, topologically sorted by run_after among themselves. Roles that
   become ready together fall out in stage order (constructor -> workstation
-  -> server -> destructor) via ``roles/categories.yml``, then category
+  -> server -> destructor) via ``meta/categories.yml``, then category
   run_after, then name.
 
 Derived purely from ``meta/*.yml`` (run_after + the service registry) and
-``roles/categories.yml`` (stage), NOT from ``tasks/stages/*.yml``, so it
+``meta/categories.yml`` (stage), NOT from ``tasks/stages/*.yml``, so it
 stays independent of how the stage playbooks happen to be split.
 
 Examples:

cli/meta/roles/services/called/__init__.py

@@ -5,7 +5,7 @@ A role's `meta/services.yml` MAY contain (per-service-entity):
 
     <entity>:
       required_by:
-        categories: [web, web.app, ...]   # category handles (cf. roles/categories.yml)
+        categories: [web, web.app, ...]   # category handles (cf. meta/categories.yml)
         roles: [web-app-yourls, ...]      # full role names incl. category prefix
 
 A `required_by` block MAY instead nest `compose:` / `swarm:` sub-blocks to
@@ -25,6 +25,15 @@ non-skip status) in the run log slice that this verifier scans.
 A missing event indicates a coverage regression in the deploy chain
 (typically: a once-flag swallowed an include before the role could run).
 
+Two log-parsing trip-wires shape the regexes below. The non-skip status
+match is anchored to line start (after optional indent) so a substring in
+arbitrary task output cannot fake a status. And Ansible's `log_path` file
+(`ANSIBLE_LOG_PATH`, what the swarm/compose deploy feeds this verifier)
+prefixes every line with `<ts> p=<pid> u=<user> n=<name> <LVL>| `; left
+unstripped that prefix defeats both the line anchor and the `\\nTASK [`
+block split, so every required role would false-positive as "did not
+execute".
+
 CLI: `python -m cli.meta.roles.services.called --help`
 """
 
@@ -40,18 +49,10 @@ from utils.cache.yaml import load_yaml_any
 from utils.roles.mapping import ROLE_FILE_META_SERVICES
 
 _ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
-# Status indicators that prove a task block was NOT skipped. We anchor to
-# line start (after optional indent) so substring matches in arbitrary
-# task output cannot trigger a false positive.
 _NON_SKIP_RE = re.compile(
     r"^\s*(?:ok|changed|fatal|included):\s+",
     re.MULTILINE,
 )
-# Ansible's log_path file (ANSIBLE_LOG_PATH, what the swarm/compose deploy feeds
-# this verifier) prefixes every line with "<ts> p=<pid> u=<user> n=<name> <LVL>| ".
-# Unstripped, that prefix defeats both the line-anchored _NON_SKIP_RE and the
-# "\nTASK [" block split below (the TASK header no longer follows the newline),
-# so every required role false-positives as "did not execute".
 _LOG_PREFIX_RE = re.compile(
     r"^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d+ p=\d+ u=\S+ n=\S+ \w+\| ",
     re.MULTILINE,
@@ -79,8 +80,11 @@ _SWARM_MARKER_ROLES = frozenset({"svc-swarm-node", "svc-swarm-manager"})
 
 
 def _deploy_mode(deployed_role_ids: list[str]) -> str:
-    # No DEPLOYMENT_MODE in scope post-deploy: infer it from the role set, since a
-    # swarm deploy always pulls in the swarm-node/manager infra and compose never does.
+    """Deploy mode inferred from the role set.
+
+    ``DEPLOYMENT_MODE`` is out of scope post-deploy, but a swarm deploy
+    always pulls in the swarm-node/manager infra and compose never does.
+    """
     return "swarm" if _SWARM_MARKER_ROLES.intersection(deployed_role_ids) else "compose"
 
 
@@ -120,8 +124,6 @@ def required_role_ids(
             rb = entry.get("required_by")
             if not isinstance(rb, dict):
                 continue
-            # required_by may scope to a deploy mode via 'compose'/'swarm' sub-keys;
-            # a flat {categories, roles} applies in every mode.
             if "compose" in rb or "swarm" in rb:
                 rb = rb.get(deploy_mode) or {}
                 if not isinstance(rb, dict):
@@ -233,6 +235,9 @@ def verify(
     Reads from `log_byte_offset` to end and asserts that every required
     role's body actually executed in that slice (i.e. at least one TASK
     block for the role ended with a non-skip status).
+
+    An unreadable log proves coverage neither way and is reported as
+    "missing" for every required role, so the operator notices.
     """
     required = required_role_ids(
         roles_dir=roles_dir, deployed_role_ids=deployed_role_ids
@@ -247,8 +252,6 @@ def verify(
     else:
         log_content = _host_log_slice(log_path=log_path, byte_offset=log_byte_offset)
     if log_content is None:
-        # Missing log == we cannot prove coverage either way; treat as
-        # missing for every required role so the operator notices.
         return False, sorted(required)
 
     missing = [

compose.yml

@@ -82,6 +82,7 @@ services:
       INFINITO_IP6: ${INFINITO_IP6}
       INFINITO_PLAYWRIGHT_KEEP: ${INFINITO_PLAYWRIGHT_KEEP}
       INFINITO_SRC_DIR: ${INFINITO_SRC_DIR}
+      INFINITO_SWARM_MGR_IP: ${INFINITO_SWARM_MGR_IP}
       PYTHONHTTPSVERIFY: "1"
       GIT_SSL_NO_VERIFY: "0"
 

compose/cache.override.yml

@@ -140,8 +140,8 @@ services:
       - ${INFINITO_CACHE_PACKAGE_PIP_CONF:?Set by 'cli deploy development up' / 'make compose-up' (driver-injected, not produced by 'make
         dotenv')}:/etc/pip.conf:ro
       - ${INFINITO_CACHE_PACKAGE_NPMRC:?Set by 'cli deploy development up' / 'make compose-up' (driver-injected, not produced by 'make dotenv')}:/root/.npmrc:ro
-      - ${INFINITO_CACHE_PACKAGE_APT_LIST:?Set by 'cli deploy development up' / 'make compose-up' (driver-injected, not produced by 'make
-        dotenv')}:/etc/apt/sources.list.d/package-cache.list:ro
+      - ./compose/package-cache/apt/${INFINITO_DISTRO:?Run 'make dotenv' to generate the .env single source of
+        truth}.list:/etc/apt/sources.list.d/package-cache.list:ro
       - ${INFINITO_CACHE_PACKAGE_FRONTEND_CA_FILE:?Set by 'cli deploy development up' / 'make compose-up' (driver-injected, not produced by 'make
         dotenv')}:/opt/package-frontend-ca.crt:ro
       - ./scripts/docker/cache/package-frontend/ca.sh:/usr/local/bin/package-frontend-ca.sh:ro

compose/cache.shared.override.yml

@@ -0,0 +1,18 @@
+---
+# Layered on cache.override.yml for secondary instances, which reach the
+# primary instance's cache stack instead of running one of their own.
+# See docs/contributing/environment/cache.md.
+
+services:
+  infinito:
+    # Exception: required:false still makes compose CREATE the cache services,
+    # which would claim the primary instance's IPs outside this subnet.
+    depends_on: !reset null
+    networks:
+      cache:
+
+networks:
+  cache:
+    external: true
+    name: ${INFINITO_CACHE_NETWORK:?Set by 'make worktree-up' from the running
+      cache stack; empty on the primary instance}

compose/package-cache-frontend/upstreams.conf

@@ -12,15 +12,36 @@ proxy_connect_timeout 30s;
 # Docker embedded DNS, pinned to bypass image-specific resolv.conf order.
 resolver 127.0.0.11 ipv6=off valid=30s;
 
+# Nexus answers 502 when a proxy's remote is unreachable (measured) and has no
+# group repository type for apt, so the mirror failover sits here rather than in
+# the repo layout. 5xx only: a 404 means the file genuinely is not in the suite,
+# and apt probes for missing files often enough that retrying those would double
+# the request rate.
+#
+# These stay prefix locations. A regex location forfeits nginx's implicit
+# `/debian` -> `/debian/` redirect, and the URL-reachability test probes exactly
+# that slash-less form through the runner's DNS hijack, so it turns into a 404.
 server {
 	listen 80;
 	server_name deb.debian.org;
 	location /debian/ {
 		rewrite ^/debian/(.*)$ /repository/apt-debian/$1 break;
+		proxy_intercept_errors on;
+		error_page 502 503 504 = @debian_mirror;
+		proxy_pass http://package-cache:8081;
+	}
+	location @debian_mirror {
+		rewrite ^/repository/apt-debian/(.*)$ /repository/apt-debian-mirror/$1 break;
 		proxy_pass http://package-cache:8081;
 	}
 	location /debian-security/ {
 		rewrite ^/debian-security/(.*)$ /repository/apt-debian-security/$1 break;
+		proxy_intercept_errors on;
+		error_page 502 503 504 = @debian_security_mirror;
+		proxy_pass http://package-cache:8081;
+	}
+	location @debian_security_mirror {
+		rewrite ^/repository/apt-debian-security/(.*)$ /repository/apt-debian-security-mirror/$1 break;
 		proxy_pass http://package-cache:8081;
 	}
 }
@@ -30,6 +51,12 @@ server {
 	server_name archive.ubuntu.com;
 	location /ubuntu/ {
 		rewrite ^/ubuntu/(.*)$ /repository/apt-ubuntu/$1 break;
+		proxy_intercept_errors on;
+		error_page 502 503 504 = @ubuntu_mirror;
+		proxy_pass http://package-cache:8081;
+	}
+	location @ubuntu_mirror {
+		rewrite ^/repository/apt-ubuntu/(.*)$ /repository/apt-ubuntu-mirror/$1 break;
 		proxy_pass http://package-cache:8081;
 	}
 }
@@ -38,6 +65,12 @@ server {
 	server_name security.ubuntu.com;
 	location /ubuntu/ {
 		rewrite ^/ubuntu/(.*)$ /repository/apt-ubuntu-security/$1 break;
+		proxy_intercept_errors on;
+		error_page 502 503 504 = @ubuntu_security_mirror;
+		proxy_pass http://package-cache:8081;
+	}
+	location @ubuntu_security_mirror {
+		rewrite ^/repository/apt-ubuntu-security/(.*)$ /repository/apt-ubuntu-security-mirror/$1 break;
 		proxy_pass http://package-cache:8081;
 	}
 }

compose/package-cache/README.md

@@ -8,4 +8,4 @@ For services, activation, coverage, and operations of the local cache stack, see
 
 - `pip.conf`: pip configuration. Mounted at `/etc/pip.conf`. Routes pip through `pypi-proxy`.
 - `npmrc`: npm configuration. Mounted at `/root/.npmrc`. Routes npm through `npm-proxy`.
-- `apt.list`: apt sources. Mounted at `/etc/apt/sources.list.d/package-cache.list`. Routes apt through `apt-debian` (Debian runners) and `apt-ubuntu` (Ubuntu runners).
+- `apt/<distro>.list`: apt sources, one file per `INFINITO_DISTRO`. The matching file is mounted at `/etc/apt/sources.list.d/package-cache.list`. `apt/debian.list` routes apt through `apt-debian`, `apt/ubuntu.list` through `apt-ubuntu`; the pacman/dnf distros get an empty file, because `apt-get update` fails as a whole on a single unreachable source and must not carry a foreign distro's entry.

compose/package-cache/apt.list

@@ -1,13 +0,0 @@
-# Routes apt-get traffic from the infinito runner through the
-# package-cache (Sonatype Nexus 3 OSS) pull-through proxy. Two repo
-# entries to cover both Debian (`apt-debian`) and Ubuntu (`apt-ubuntu`)
-# upstreams; the runner reads only the entry that matches its distro
-# at runtime via apt's standard suite-resolution. The bootstrap helper
-# pins debian -> bookworm and ubuntu -> jammy; older runners still
-# resolve fine because Nexus serves the entire upstream tree.
-#
-# Uses the real mirror hostnames (mapped to the cache frontend via
-# extra_hosts in compose/cache.override.yml), not the `package-cache`
-# service name which coredns — the container's resolver — cannot resolve.
-deb [trusted=yes] http://deb.debian.org/debian bookworm main
-deb [trusted=yes] http://archive.ubuntu.com/ubuntu jammy main

compose/package-cache/apt/arch.list

@@ -0,0 +1,3 @@
+# Empty on purpose: Arch uses pacman, which the package-cache does not
+# proxy. The bind mount is unconditional in the compose overrides, and
+# docker would create a host directory here if the source were missing.

compose/package-cache/apt/centos.list

@@ -0,0 +1,4 @@
+# Empty on purpose: CentOS uses dnf, which reaches the package-cache
+# through `yum-rocky`, not through apt. The bind mount is unconditional
+# in the compose overrides, and docker would create a host directory
+# here if the source were missing.

compose/package-cache/apt/debian.list

@@ -0,0 +1,11 @@
+# apt sources for the Debian runner: routes apt-get through the
+# package-cache (Sonatype Nexus 3 OSS) pull-through proxy.
+#
+# Real mirror hostname, not the `package-cache` service name, which
+# coredns — the container's resolver — cannot resolve; the compose
+# overrides map the hostname to the cache frontend via extra_hosts.
+#
+# One file per INFINITO_DISTRO: apt fails the whole `apt-get update`
+# when a single source is unreachable, so a Debian runner must never
+# carry the Ubuntu entry.
+deb [trusted=yes] http://deb.debian.org/debian bookworm main

compose/package-cache/apt/fedora.list

@@ -0,0 +1,4 @@
+# Empty on purpose: Fedora uses dnf, which reaches the package-cache
+# through `yum-fedora`, not through apt. The bind mount is unconditional
+# in the compose overrides, and docker would create a host directory
+# here if the source were missing.

compose/package-cache/apt/ubuntu.list

@@ -0,0 +1,11 @@
+# apt sources for the Ubuntu runner: routes apt-get through the
+# package-cache (Sonatype Nexus 3 OSS) pull-through proxy.
+#
+# Real mirror hostname, not the `package-cache` service name, which
+# coredns — the container's resolver — cannot resolve; the compose
+# overrides map the hostname to the cache frontend via extra_hosts.
+#
+# One file per INFINITO_DISTRO: apt fails the whole `apt-get update`
+# when a single source is unreachable, so an Ubuntu runner must never
+# carry the Debian entry.
+deb [trusted=yes] http://archive.ubuntu.com/ubuntu jammy main

compose/swarm/Dockerfile

@@ -1,27 +1,53 @@
 # Node image for the swarm DR test cluster, layered on the distro's infinito
 # image so python + ansible + the CLI (venv) are already baked -- the per-node
 # .deb bootstrap is gone. Adds only the node extras: dnsmasq (lab DNS), openssh
-# server (backup pull chain), cryptsetup + e2fsprogs (LUKS drill device).
+# server (backup pull chain), cryptsetup + e2fsprogs (LUKS drill device), kmod
+# (modprobe, for the data-root filesystem).
 # Distro-agnostic install (apt/dnf/pacman) since INFINITO_IMAGE is per-distro.
-# systemd-resolved is masked so it never races dnsmasq for port 53.
+# systemd-resolved is masked so it never races dnsmasq for port 53. The pacman
+# branch refreshes the keyring and does a full -Syu: the pinned parent image can
+# be weeks old, so a partial -Sy would hit unknown packager keys and would link
+# fresh binaries against the image's older libraries.
+#
+# udevd is masked because it claims the device-mapper nodes and leaves symlinks
+# to /dev/dm-N targets a tmpfs /dev never gets, while without it libdevmapper
+# creates them.
+#
+# /etc/shadow is 0000 on the RHEL-family parents, where the host's unix-chkpwd
+# AppArmor profile then denies the setuid helper CAP_DAC_READ_SEARCH and every
+# PAM account lookup fails; a hard 0600 instead of u+rw breaks debian's setgid
+# helper.
+#
+# The zfs userland arrives with the parent image and enables six units by
+# preset. All of them run before scripts/tests/deploy/utils/filesystem/
+# docker_dataroot.sh creates the pool, so on a node they can only fail and
+# leave systemd degraded; they are masked here rather than in the parent, which
+# a real host also boots.
 ARG INFINITO_IMAGE
 FROM ${INFINITO_IMAGE}
 
 RUN set -eux; \
     if command -v apt-get >/dev/null 2>&1; then \
         apt-get update -qq; \
-        apt-get install -y -qq --no-install-recommends dnsmasq openssh-server cryptsetup e2fsprogs; \
+        apt-get install -y -qq --no-install-recommends dnsmasq openssh-server cryptsetup e2fsprogs elfutils kmod python3-apt; \
         rm -rf /var/lib/apt/lists/*; \
     elif command -v dnf >/dev/null 2>&1; then \
-        dnf install -y dnsmasq openssh-server cryptsetup e2fsprogs; \
+        dnf install -y dnsmasq openssh-server cryptsetup e2fsprogs elfutils kmod; \
         dnf clean all; \
     elif command -v pacman >/dev/null 2>&1; then \
-        pacman -Sy --noconfirm dnsmasq openssh cryptsetup e2fsprogs; \
+        pacman -Sy --noconfirm --needed archlinux-keyring; \
+        pacman -Syu --noconfirm --needed dnsmasq openssh cryptsetup e2fsprogs elfutils kmod; \
+        rm -rf /var/cache/pacman/pkg/*; \
     else \
         echo "no supported package manager (apt/dnf/pacman)" >&2; exit 1; \
     fi; \
     systemctl mask systemd-resolved; \
+    systemctl mask systemd-udevd systemd-udevd-kernel.socket systemd-udevd-control.socket systemd-udevd-varlink.socket systemd-udev-trigger; \
+    systemctl mask modprobe@.service; \
+    systemctl mask zfs-import-cache zfs-import-scan zfs-mount zfs-share zfs-volume-wait zfs-zed; \
     systemctl enable ssh 2>/dev/null || systemctl enable sshd; \
+    chmod u+rw /etc/shadow; \
+    if [ -e /etc/gshadow ]; then chmod u+rw /etc/gshadow; fi; \
     mkdir -p /etc/systemd/journald.conf.d; \
     printf '[Journal]\nSystemMaxUse=256M\nRuntimeMaxUse=64M\n' > /etc/systemd/journald.conf.d/99-ci.conf; \
     rm -f /var/lib/dbus/machine-id; \

compose/swarm/cache.override.yml

@@ -6,8 +6,11 @@ x-cache: &cache
     - "deb.debian.org:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
     - "archive.ubuntu.com:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
     - "security.ubuntu.com:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
+    - "infinito-registry-cache:${INFINITO_CACHE_REGISTRY_IP:?Run 'make dotenv' to generate the .env single source of truth}"
   volumes:
-    - ../package-cache/apt.list:/etc/apt/sources.list.d/package-cache.list:ro
+    - ../package-cache/apt/${INFINITO_DISTRO:?Run 'make dotenv' to generate the .env single source of truth}.list:/etc/apt/sources.list.d/package-cache.list:ro
+    - ${INFINITO_CACHE_REGISTRY_CA_HOST_PATH:?Run 'make dotenv' to generate the .env single source of truth}:/opt/registry-cache-ca:ro
+    - ../../scripts/docker/cache/registry-ca.sh:/usr/local/bin/registry-cache-ca.sh:ro
 
 services:
   nfs-server:

compose/swarm/compose.yml

@@ -6,9 +6,12 @@ x-node: &node
       INFINITO_IMAGE: "${INFINITO_IMAGE:?run make dotenv / build the distro image first}"
   privileged: true
   cgroup: host
+  pids_limit: ${INFINITO_SWARM_NODE_PIDS_LIMIT:?run make dotenv first}
   security_opt:
     - seccomp=unconfined
     - apparmor=unconfined
+  environment:
+    - DM_DISABLE_UDEV=1
   tmpfs:
     - /run
     - /run/lock

compose/swarm/playbook.yml

@@ -2,12 +2,11 @@
 # Lab-only node provisioning for the simulated cluster; never a production
 # role. Runs over ansible_connection: docker right after `compose up` (python +
 # ansible + the CLI are baked into the node image via the distro's infinito
-# image, so no .deb bootstrap is needed): waits for systemd, captures node IPs
-# (also into $GITHUB_ENV for later CI steps), wires lab DNS (sys-hostname skips
-# /etc/hosts on DOCKER_IN_CONTAINER hosts; the dnsmasq wildcard points the app
-# domain at the manager's ingress), then unpacks the current repo onto every
-# node. Host-side concerns (image build, compose up) live in
-# scripts/tests/deploy/swarm/routine/01_bootstrap.sh.
+# image, so no .deb bootstrap is needed): waits for systemd, wires lab DNS
+# (sys-hostname skips /etc/hosts on DOCKER_IN_CONTAINER hosts; the dnsmasq
+# wildcard points the app domain at the manager's ingress), then unpacks the
+# current repo onto every node. Host-side concerns (image build, compose up)
+# live in scripts/tests/deploy/swarm/routine/01_bootstrap.sh.
 - name: provision the simulated cluster nodes
   hosts: all
   gather_facts: false
@@ -20,7 +19,6 @@
     lab_wrk2: "{{ lookup('env', 'WRK2') }}"
     lab_nfs: "{{ lookup('env', 'NFS_SERVER') }}"
     lab_backup: "{{ lookup('env', 'BACKUP_NODE') }}"
-    lab_net: "{{ lookup('env', 'SWARM_LAB_NETWORK') }}"
     lab_domain: "{{ lookup('env', 'INFINITO_DOMAIN') }}"
     lab_export_base: "{{ lookup('env', 'NFS_EXPORT_BASE') }}"
     lab_node_src: "{{ lookup('env', 'INFINITO_NODE_SRC_DIR') }}"
@@ -29,8 +27,18 @@
     lab_is_mgr: "{{ inventory_hostname == lab_mgr }}"
     lab_is_worker: "{{ inventory_hostname in [lab_wrk1, lab_wrk2] }}"
     lab_dns_node: "{{ lab_is_mgr or lab_is_worker }}"
-    lab_domain_target: "{{ lab_mgr_ip }}"
+    lab_domain_target: "{{ lookup('env', 'INFINITO_SWARM_MGR_IP') }}"
+    lab_filesystem: "{{ lookup('env', 'INFINITO_DOCKER_FILESYSTEM') }}"
+    lab_filesystem_required: "{{ lookup('env', 'INFINITO_DOCKER_FILESYSTEM_REQUIRED') | default('false', true) }}"
+    lab_dataroot_size: "{{ '60G' if lab_is_mgr else '30G' }}"
+    lab_no_proxy: "localhost,127.0.0.1,::1,infinito-registry-cache,{{ lab_mgr }},{{ lab_wrk1 }},{{ lab_wrk2 }},{{ lab_nfs }},{{ lab_backup }},gcr.io,storage.googleapis.com,googleusercontent.com"
   tasks:
+    - name: require the lab manager IP (lookup('env') yields '' for a missing key, and an empty dnsmasq address= is valid syntax)
+      assert:
+        that: lab_domain_target | length > 0
+        fail_msg: INFINITO_SWARM_MGR_IP is empty; it comes from scripts/tests/deploy/swarm/utils/topology/base.sh
+      run_once: true
+
     - block:
         - name: wait for systemd ready (status from stdout; rc is nonzero when degraded)
           command: systemctl is-system-running
@@ -74,51 +82,36 @@
         state: started
       timeout: 30
 
+    - name: put the node docker data root on the stated filesystem (before anything is pulled)
+      script: >-
+        {{ lab_repo_root }}/scripts/tests/deploy/utils/filesystem/docker_dataroot.sh
+        {{ lab_filesystem }} {{ lab_filesystem_required }} {{ lab_dataroot_size }} {{ inventory_hostname }}
+      timeout: 600
+      when: lab_filesystem | length > 0
+      changed_when: true
+      register: lab_dataroot
+
+    - name: surface the node's effective data root filesystem
+      debug:
+        msg: "{{ lab_dataroot.stdout_lines | select('search', '^docker-dataroot-filesystem: status=') | list }}"
+      when: not (lab_dataroot is skipped)
+
+    - name: probe the registry-cache CA (mounted by cache.override.yml, absent without a local cache stack)
+      stat:
+        path: /opt/registry-cache-ca/ca.crt
+      timeout: 30
+      register: lab_registry_ca
+
+    - name: wire the node docker daemon to the host registry cache
+      include_tasks: registry_cache.yml
+      when: lab_registry_ca.stat.exists
+
     - name: wipe stale NFS export state (as root in-node; the act runner cannot)
       command: find {{ lab_export_base }} -mindepth 1 -delete
       timeout: 120
       when: inventory_hostname == lab_nfs
       changed_when: true
 
-    - name: capture the node IPs
-      command:
-        argv:
-          - docker
-          - inspect
-          - "{{ item }}"
-          - --format
-          - "{{ '{{(index .NetworkSettings.Networks \"' ~ lab_net ~ '\").IPAddress}}' }}"
-      loop:
-        - "{{ lab_mgr }}"
-        - "{{ lab_nfs }}"
-        - "{{ lab_backup }}"
-      timeout: 30
-      delegate_to: localhost
-      run_once: true
-      register: lab_ips
-      changed_when: false
-
-    - name: expose the IPs to every host
-      set_fact:
-        lab_mgr_ip: "{{ lab_ips.results[0].stdout }}"
-        lab_nfs_ip: "{{ lab_ips.results[1].stdout }}"
-        lab_backup_ip: "{{ lab_ips.results[2].stdout }}"
-      run_once: true
-
-    - name: publish the IPs to $GITHUB_ENV (read by later CI steps; not visible in this one)
-      lineinfile:
-        path: "{{ lookup('env', 'GITHUB_ENV') }}"
-        line: "{{ item }}"
-        create: true
-        mode: "0644"
-      loop:
-        - "MGR_IP={{ lab_mgr_ip }}"
-        - "NFS_IP={{ lab_nfs_ip }}"
-        - "BACKUP_IP={{ lab_backup_ip }}"
-      timeout: 30
-      delegate_to: localhost
-      run_once: true
-
     - name: render the app-domain hosts entries
       command: python3 -m utils.tests.swarm.write.hosts_entries
       args:
@@ -140,26 +133,19 @@
       timeout: 30
       when: lab_is_mgr or inventory_hostname in [lab_nfs, lab_backup]
 
-    - name: write the dnsmasq lab config
-      copy:
-        dest: /etc/dnsmasq.d/infinito.conf
-        content: |
-          bind-dynamic
-          no-resolv
-          server=/{{ lab_mgr }}/{{ lab_wrk1 }}/{{ lab_wrk2 }}/{{ lab_nfs }}/{{ NETWORK_DOCKER_EMBEDDED_DNS_RESOLVER }}
-          {% for resolver in NETWORK_PUBLIC_DNS_RESOLVERS %}
-          server={{ resolver }}
-          {% endfor %}
-          address=/{{ lab_domain }}/{{ lab_domain_target }}
+    - name: own the whole dnsmasq config (fedora and centos ship local-service, which implies bind-interfaces and dnsmasq refuses next to our bind-dynamic)
+      template: # nocheck: stack-host-template - lab node system config, every DNS node writes its own /etc/dnsmasq.conf and IS_STACK_HOST does not exist in this play
+        src: dnsmasq.conf.j2
+        dest: /etc/dnsmasq.conf
       timeout: 30
       when: lab_dns_node
 
-    - name: enable and restart dnsmasq
+    - name: enable and restart dnsmasq (arch is Type=dbus, so systemd waits its full 90s DefaultTimeoutStartSec before reporting)
       systemd:
         name: dnsmasq
         enabled: true
         state: restarted
-      timeout: 60
+      timeout: 120
       when: lab_dns_node
 
     - name: point resolv.conf at dnsmasq

compose/swarm/registry_cache.yml

@@ -0,0 +1,23 @@
+---
+- name: install the registry-cache CA into the node trust store
+  command: /usr/local/bin/registry-cache-ca.sh
+  timeout: 60
+  changed_when: true
+
+- name: ensure the docker drop-in directory exists
+  file:
+    path: /etc/systemd/system/docker.service.d
+    state: directory
+    mode: "0755"
+  timeout: 30
+
+- name: route the node docker daemon through the registry cache
+  copy:
+    dest: /etc/systemd/system/docker.service.d/registry-cache-proxy.conf
+    mode: "0644"
+    content: |
+      [Service]
+      Environment="HTTP_PROXY=http://infinito-registry-cache:3128"
+      Environment="HTTPS_PROXY=http://infinito-registry-cache:3128"
+      Environment="NO_PROXY={{ lab_no_proxy }}"
+  timeout: 30

compose/swarm/templates/dnsmasq.conf.j2

@@ -0,0 +1,6 @@
+no-resolv
+server=/{{ lab_mgr }}/{{ lab_wrk1 }}/{{ lab_wrk2 }}/{{ lab_nfs }}/{{ NETWORK_DOCKER_EMBEDDED_DNS_RESOLVER }}
+{% for resolver in NETWORK_PUBLIC_DNS_RESOLVERS %}
+server={{ resolver }}
+{% endfor %}
+address=/{{ lab_domain }}/{{ lab_domain_target }}

compose/worktree.override.yml

@@ -0,0 +1,12 @@
+---
+# Layered on compose.yml for secondary instances. A git worktree's .git is a
+# file pointing at an absolute path inside the primary checkout, so git only
+# works in the container when that directory is mounted where the pointer
+# expects it.
+
+services:
+  infinito:
+    volumes:
+      - ${INFINITO_GIT_COMMON_DIR:?Set by 'make worktree-up' from git rev-parse
+        --git-common-dir}:${INFINITO_GIT_COMMON_DIR:?Set by 'make worktree-up'
+        from git rev-parse --git-common-dir}

default.env

@@ -34,8 +34,8 @@ INFINITO_BUILD=1
 # `1` keeps BuildKit on (default); `0` forces the compose wrapper to use the legacy builder.
 INFINITO_BUILDKIT_ENABLED=1
 
-# Bind-mounted apt sources.list fragment for the package-cache frontend.
-INFINITO_CACHE_PACKAGE_APT_LIST=./compose/package-cache/apt.list
+# External docker network carrying the shared cache stack; empty on the primary instance, which owns the stack itself.
+INFINITO_CACHE_NETWORK=
 
 # Host directory holding the package-cache frontend CA materials.
 INFINITO_CACHE_PACKAGE_FRONTEND_CA_DIR=/var/cache/infinito/core/cache/package/frontend/ca
@@ -82,9 +82,15 @@ INFINITO_CACHE_REGISTRY_IP=172.30.0.6
 # Bind-mounted registry-cache proxy configuration consumed by the cache container.
 INFINITO_CACHE_REGISTRY_PROXY_CONF=./compose/registry-cache/proxy.conf
 
+# Budget distros.sh spends before skipping the rest; stays under the 350-min step cap, which stays under the 355-min job cap.
+INFINITO_CI_DISTRO_BUDGET_SECONDS=20400
+
 # Job budget per workflow run cli.meta.ci.slots divides between the deploy matrices (GitHub matrix cap: 256).
 INFINITO_CI_JOB_LIMIT=256
 
+# Deploy jobs a CI line may queue at once before cli.meta.ci.sequencing serialises its modes (GitHub cancels a job queued for 24h).
+INFINITO_CI_SEQUENTIAL_THRESHOLD=65
+
 # Compose service container name; generator overrides as infinito_nexus_${INFINITO_DISTRO}.
 INFINITO_CONTAINER=infinito_nexus_debian
 
@@ -106,9 +112,6 @@ INFINITO_DISCOVERY_SORT="desc integrated,asc covered_by,desc embeds,desc consume
 # Distro the build/test matrix is currently iterating over (generator may override).
 INFINITO_DISTRO=debian
 
-# Space-separated list of distros the build/test matrix iterates over.
-INFINITO_DISTROS="arch debian ubuntu fedora centos"
-
 # Domain the host-side DNS-fallback / dnsmasq config wires up (consumed by scripts/system/network/dns/).
 INFINITO_DNS_DOMAIN=infinito.test
 
@@ -121,6 +124,15 @@ INFINITO_DNS_IP=172.30.0.53
 # Repo-relative path of the docker CLI install script.
 INFINITO_DOCKER_CLI_INSTALL_SCRIPT=roles/sys-svc-container/files/install-cli.sh
 
+# Filesystem the docker data root runs on; empty until the per-distro draw in scripts/tests/deploy/utils/filesystem/resolve.sh fills it.
+INFINITO_DOCKER_FILESYSTEM=
+
+# Space-separated subset of ext4/btrfs/zfs the per-distro draw may use; empty draws from all three.
+INFINITO_DOCKER_FILESYSTEM_ALLOWED=
+
+# `true` when INFINITO_DOCKER_FILESYSTEM_ALLOWED narrowed the draw; a host that cannot deliver the pick then fails instead of declining.
+INFINITO_DOCKER_FILESYSTEM_REQUIRED=
+
 # Mount point inside the infinito container where INFINITO_DOCKER_VOLUME lands.
 INFINITO_DOCKER_MOUNT=/var/lib/docker
 
@@ -133,6 +145,9 @@ INFINITO_DOMAIN=infinito.test
 # INFINITO_DOMAIN escaped for inclusion in regex patterns (literal `.`); keep in sync with INFINITO_DOMAIN.
 INFINITO_DOMAIN_RE=infinito\.test
 
+# Repo-relative path of the filesystem userland install script (ext4, btrfs, zfs).
+INFINITO_FILESYSTEM_INSTALL_SCRIPT=scripts/install/filesystem.sh
+
 # Repository name of the operator's fork; composed into INFINITO_FORK_REPOSITORY_URL by utils/env/handlers/infinito/fork_repository_url.py.
 INFINITO_FORK_REPOSITORY_NAME=infinito-nexus-core
 
@@ -142,12 +157,18 @@ INFINITO_GATEWAY=172.30.0.1
 # Prefix path under which GHCR mirror lookups search for cached images.
 INFINITO_GHCR_MIRROR_PREFIX=mirror
 
+# Shared .git directory a worktree checkout points at; empty on the primary instance, which carries its own.
+INFINITO_GIT_COMMON_DIR=
+
 # Run-context flag like INFINITO_DEPLOY_MODE: the single-node workflow sets 'true' so host discovery drops compose-covered roles.
 INFINITO_HOST_EXCLUDE_COMPOSE=
 
 # Image tag used to address the published infinito image (CI publishes 'latest' on push).
 INFINITO_IMAGE_TAG=latest
 
+# Slot index isolating a parallel checkout: 0 owns the cache stack, 1..255 shift subnet, bind IP and container names.
+INFINITO_INSTANCE=0
+
 # Optional JSON object merged into the generated dev-inventory vars (e.g. the migration scenario pins MAIL_PROVIDER for its legacy leg).
 INFINITO_INVENTORY_EXTRA_VARS=
 
@@ -280,6 +301,12 @@ INFINITO_SWARM_NFS_IP=192.168.244.13
 # NFS-server sidecar base name in the act-swarm-test cluster.
 INFINITO_SWARM_NFS_NAME=nfs-server
 
+# Process ceiling per DinD node.
+INFINITO_SWARM_NODE_PIDS_LIMIT=8192
+
+# Ceiling in minutes for one distro's swarm matrix deploy in scripts/tests/deploy/swarm/routine/00_one.sh; swarm's step level.
+INFINITO_SWARM_STEP_TIMEOUT_MINUTES=350
+
 # Docker label stamped on every act-swarm-test container (DinD + NFS sidecar) so cleanup selects by label, not a guessed name.
 INFINITO_SWARM_TEST_LABEL=infinito.swarm-test
 

docs/administration/ci-runners.md

@@ -10,7 +10,7 @@ Every push runs the same gate you can run locally:
 make test          # lint + unit + integration + external
 ```
 
-On GitHub Actions this is followed by the **deploy-test matrix** — real app deploys across Debian/Ubuntu/Arch/Fedora in throwaway containers, with Playwright/CLI end-to-end checks (`.github/workflows/test-deploy-*.yml`). This is the heavy part and the reason self-hosted runners exist.
+On GitHub Actions this is followed by the **deploy-test matrix** — real app deploys across Debian/Ubuntu/Arch/Fedora in throwaway containers, with Playwright/CLI end-to-end checks (`.github/workflows/call-test-deploy-*.yml`). This is the heavy part and the reason self-hosted runners exist.
 
 ## Self-hosted runners (opt-in overflow)
 

docs/agents/action/iteration/roundtrip.md

@@ -35,5 +35,5 @@ The roundtrip is a gate, not a debugger: when a step goes red it stops and leave
 ## Recovery & gotchas
 
 - On a recent Docker engine the swarm step can abort at job setup; run `make act-runner-image` once and prefix the run with `ACT_PLATFORM_IMAGE=local/act-runner-fixed:latest` (see [Workflow Loop](workflow.md)).
-- The swarm-test nfs-server serves shared storage from userspace `nfs-ganesha` (the `ganesha` flavor of `svc-storage-nfs-server`, selected when `RUNTIME in ['dev', 'act']`), which runs inside the privileged DinD node and never touches the host kernel `nfsd`, so an interrupted export cannot wedge the host. Production keeps the `kernel` flavor on real hosts, where `nfsd` is not shared across clusters. If a swarm step ever hangs at `Reload NFS exports` from a stuck kernel-NFS mount, you MUST recover with `make swarm-clean` on the host (needs sudo): it removes what the docker CLI can, then runs `umount -f -l /var/lib/infinito` + `exportfs -ua` + `systemctl restart docker`. The no-sudo sandbox only prints those commands, so the host recovery is mandatory and the sweep MUST stop until it is done; never fall back to compose-only.
+- The swarm-test nfs-server serves shared storage from userspace `nfs-ganesha` (the `ganesha` flavor of `svc-storage-nfs-server`, the default in every runtime), which runs inside the privileged DinD node and never touches the host kernel `nfsd`, so an interrupted export cannot wedge the host. The `kernel` flavor is a per-inventory override for real hosts, where `nfsd` is not shared across clusters. If a swarm step ever hangs at `Reload NFS exports` from a stuck kernel-NFS mount, you MUST recover with `make swarm-clean` on the host (needs sudo): it removes what the docker CLI can, then runs `umount -f -l /var/lib/infinito` + `exportfs -ua` + `systemctl restart docker`. The no-sudo sandbox only prints those commands, so the host recovery is mandatory and the sweep MUST stop until it is done; never fall back to compose-only.
 - Distinct `SWARM_NAME` per app means several apps can keep their clusters in parallel under `keep=true`; release each one with `make swarm-down name=<app>`.

docs/agents/action/iteration/swarm.md

@@ -32,6 +32,6 @@ An `swarm-zombie` run rebuilds a full DinD cluster over tens of minutes, so you
 
 ### NFS server flavor
 
-The swarm-test nfs-server uses the userspace `nfs-ganesha` (`ganesha`) flavor for local runtimes (`RUNTIME in ['dev', 'act']`) and the `kernel` flavor for real GitHub Actions CI and production, so CI exercises the production flavor. Kernel `nfsd` in a privileged DinD node is wedge-prone: the export can be present yet the controller mount returns ENOENT, and `exportfs` can stick in D-state.
+The nfs-server uses the userspace `nfs-ganesha` (`ganesha`) flavor in every runtime; `services.nfs-server.flavor` in `roles/svc-storage-nfs-server/meta/services.yml` is the SPOT, and `kernel` is a per-inventory override. Ganesha is packaged for the `Debian` and `RedHat` os families only. Kernel `nfsd` in a privileged DinD node is wedge-prone: the export can be present yet the controller mount returns ENOENT, and `exportfs` can stick in D-state.
 If a swarm step hangs at `Reload NFS exports` from a stuck kernel-NFS mount, you MUST recover with `make swarm-clean` on the host; see the NFS gotcha in [Roundtrip Loop](roundtrip.md).
 You MUST NOT fall back to a compose-only sweep while swarm is blocked.

docs/agents/action/iteration/workflow.md

@@ -17,7 +17,7 @@ For the swarm deploy loop, see [Swarm Loop](swarm.md).
 - If the workflow uses a distro matrix, you MUST iterate on one distro at a time instead of rerunning the whole matrix during the default debug loop.
 - Debian SHOULD be the preferred distro for that focused workflow iteration unless the failure is clearly distro-specific or the user asked for a different distro.
 - When you constrain an Act matrix run through `ACT_MATRIX`, you MUST use Act's `key:value` syntax instead of `key=value`. Otherwise Act may ignore the filter and rerun the whole matrix.
-- For `.github/workflows/test-environment.yml`, the preferred focused Debian example is `make act-workflow ACT_WORKFLOW=.github/workflows/test-environment.yml ACT_JOB=test-environment ACT_MATRIX='dev_runtime_image:debian:bookworm'`.
+- For `.github/workflows/call-test-workspace.yml`, the preferred focused Debian example is `make act-workflow ACT_WORKFLOW=.github/workflows/call-test-workspace.yml ACT_JOB=test-workspace ACT_MATRIX="dev_runtime_image:$(python3 -c 'from utils.distros import dev_runtime_image; print(dev_runtime_image("debian"))')"`.
 - You SHOULD avoid jumping straight to repeated remote CI reruns when `make act-workflow` can validate the workflow locally and the user agreed to use it.
 
 ## Recovery & gotchas

docs/agents/action/push-trigger-pull.md

@@ -66,13 +66,13 @@ Compute scope from `git diff --name-only origin/main...<branch>`.
 - `distros`: a change counts as **distro-specific** only if the diff is confined to either
   - files under `packaging/<distro>/` for one or more specific distros, or
   - role tasks gated by a distro conditional (e.g. `when: ansible_os_family == '...'`, `when: ansible_distribution == '...'`), touching only the branches for one or more specific distros.
-  Pass those distros as a space-separated list via `-f distros="<distros>"`. In every other case, omit `-f distros`; `entry-manual.yml` then applies its own default (currently `debian` only, it does NOT run all distros).
+  Pass those distros as a space-separated list via `-f distros="<distros>"`. In every other case, omit `-f distros`; `entry-manual-steer.yml` then applies its own default (currently `debian` only, it does NOT run all distros).
 
 Before triggering, capture the currently latest run id for this workflow+branch as `<prev-run-id>` (empty string if none exists). This is required so the post-trigger resolve can distinguish the new run from any stale completed run on the same branch:
 
 ```
 gh run list \
-  --workflow=entry-manual.yml \
+  --workflow=entry-manual-steer.yml \
   --branch=<branch> \
   --event=workflow_dispatch \
   --limit=1 \
@@ -83,7 +83,7 @@ gh run list \
 Then trigger:
 
 ```
-gh workflow run entry-manual.yml --ref <branch> \
+gh workflow run entry-manual-steer.yml --ref <branch> \
   [-f distros="<distros>"] \
   [-f whitelist="<role>"]
 ```

docs/contributing/actions/debugging/ci.md

@@ -33,7 +33,7 @@ The per-job logs carry the real failing task and its error, and the artefacts ca
 
 You SHOULD use targeted manual CI jobs instead of rerunning the full pipeline if you only need one focused check.
 
-Prefer the manual workflow in [entry-manual.yml](../../../../.github/workflows/entry-manual.yml):
+Prefer the manual workflow in [entry-manual-steer.yml](../../../../.github/workflows/entry-manual-steer.yml):
 
 - Select your branch.
 - Use `debian` unless you have a clear reason to use a different distro.

docs/contributing/actions/release.md

@@ -51,8 +51,8 @@ A `v*` tag push fans out into two CI workflows:
 
 | Workflow | Purpose |
 |---|---|
-| [release-version.yml](../../../.github/workflows/release-version.yml) | Backfills CI images for the version tag if missing, retags them to release tags, and publishes the version-specific image set. |
-| [release-highest.yml](../../../.github/workflows/release-highest.yml) | Decides whether the new version is the highest published version and, if so, advances the `latest` floating tag. |
+| [call-release-version.yml](../../../.github/workflows/call-release-version.yml) | Backfills CI images for the version tag if missing, retags them to release tags, and publishes the version-specific image set. |
+| [cron-release-highest.yml](../../../.github/workflows/cron-release-highest.yml) | Decides whether the new version is the highest published version and, if so, advances the `latest` floating tag. |
 
 Both run from the trusted workflow ref and check out the version tag for the actual build payload, so the release artefacts match the tagged commit byte-for-byte.
 

docs/contributing/actions/testing/external.md

@@ -66,7 +66,7 @@ make test-external INFINITO_TEST_PATTERN=test_image_versions.py
 
 ## CI Workflow 🤖
 
-External tests run inside `.github/workflows/test.yml` via `make test`.
+External tests run inside `.github/workflows/call-test.yml` via `make test`.
 It MAY be triggered manually and is intentionally separate from the default
 `make test` flow. It is also wired into the orchestrated code-testing gate in
 CI so the live check can run there without changing local default validation.

docs/contributing/actions/testing/suppression.md

@@ -90,7 +90,10 @@ The placement rule is per check. The catalog "Position" column uses these labels
 | `compose-structure`         | same or above; head (first 30 lines)  | [test_compose_template_structure.py](../../../../tests/lint/ansible/jinja/test_compose_template_structure.py)                                                                                                                             | Exempts a `roles/*/templates/*compose.yml.j2` line, or the whole file (head marker), from the canonical compose skeleton: base-include header, `{{ service_name }}:` service keys each preceded by their `{% set service_name %}`, all per-service `{% set %}` knobs on two-space-indented dash-free lines above the key, `{% include %}` at four spaces inside the service body, and column-0 `compose_volumes` / `compose_networks` at the tail. The layout renders correctly because the `stack_host_template` action plugin sets `lstrip_blocks=True` for `compose.yml.j2`. Reserved for legitimate deviations that genuinely cannot follow the skeleton. |
 | `seaweedfs-required`        | head (first 30 lines)                 | [test_database_requires_seaweedfs.py](../../../../tests/integration/roles/meta/services/test_database_requires_seaweedfs.py)                                                                                                             | Exempts a database-backed `web-app-*` role from the requirement to declare a seaweedfs object-store service. Place in the head of `roles/<role>/meta/services.yml`. Use only when the application genuinely needs no object storage. |
 | `templated-conditional`     | same or above                         | [test_no_templated_conditionals.py](../../../../tests/lint/ansible/test_no_templated_conditionals.py)                                                                                                                                   | Exempts a `when` / `failed_when` / `changed_when` / `until` value that legitimately carries `{{ }}` / `{% %}`. Reserved for genuine edge cases; the default path is a bare expression. Per-item render/apply data guards belong on a `condition:` data field (honored by the sys-svc-compose render helper and `LISTMONK_SETTINGS`), not `when:`. |
-| `container-address-quote`   | same or above                         | [test_container_address_never_quoted.py](../../../../tests/lint/ansible/shell/test_container_address_never_quoted.py)                                                                                                                    | Allows `*_CONTAINER_ADDRESS \| quote` in a role task file. Reserved for sites that never see the swarm `$(...)` fragment (`DEPLOYMENT_MODE == 'compose'`-gated tasks, `compose_mode_force` roles) or consumers that re-evaluate the argument themselves. |
+| `container-address-quote`   | same or above                         | [test_address_never_quoted.py](../../../../tests/lint/ansible/shell/container/test_address_never_quoted.py)                                                                                                                              | Allows `*_CONTAINER_ADDRESS \| quote` in a role task file. Reserved for sites that never see the swarm `$(...)` fragment (`DEPLOYMENT_MODE == 'compose'`-gated tasks, `compose_mode_force` roles) or consumers that re-evaluate the argument themselves. |
+| `container-exec-resolver`   | same or above; `argv:` target item    | [test_exec_uses_resolver.py](../../../../tests/lint/ansible/shell/container/test_exec_uses_resolver.py)                                                                                                                                  | Allows a `container exec` / `container inspect` target that does not route through the `container_address` lookup. Reserved for a container ID already resolved at `delegate_to` runtime (`*_LOCAL_CID`), where the lookup would return a manager-only subshell, and for deliberate bare-name partner probes gated on compose co-location. |
+| `container-inspect-type`    | same or above; `argv:` inspect item   | [test_inspect_pins_type.py](../../../../tests/lint/ansible/shell/container/test_inspect_pins_type.py)                                                                                                                                    | Allows an untyped `container inspect` under `roles/`. Default path: pin `--type container`, because a role's overlay network carries the same entity name as its main container, so an untyped inspect can resolve the network and report `map has no entry for key "State"`. Use the typed subcommand (`container volume inspect` etc.) when the target genuinely is not a container. |
+| `min-storage-headroom`      | same line OR comment block above key  | [test_min_storage_headroom.py](../../../../tests/external/update/docker/test_min_storage_headroom.py)                                                                                                                                    | Allows a `roles/*/meta/services.yml` `min_storage` to exceed the measured compressed size of its own `image:`/`version:` by more than 1 GB. MUST be paired with a `Reason:` on the same comment line naming what occupies the space. Reserved for services whose data volume genuinely dominates the footprint; the default path is to lower the value, because an over-declared budget makes a host look too small and prunes a cache the deploy did not need to lose. |
 
 ## Tool-Native `noqa` Codes 🧹
 

docs/contributing/artefact/files/compose.yml.md

@@ -77,7 +77,6 @@ The env-var contracts each service expects strictly via `${VAR:?…}` (consumed
 | `INFINITO_CACHE_PACKAGE_PORT` | Host-side port mapped to Nexus REST/UI. Bound to `${INFINITO_BIND_IP}` only. |
 | `INFINITO_CACHE_PACKAGE_PIP_CONF` | Bind source for `/etc/pip.conf`. Driver-injected by `make compose-up` under the `cache` profile. |
 | `INFINITO_CACHE_PACKAGE_NPMRC` | Bind source for `/root/.npmrc`. Driver-injected. |
-| `INFINITO_CACHE_PACKAGE_APT_LIST` | Bind source for `/etc/apt/sources.list.d/package-cache.list`. Driver-injected. |
 | `INFINITO_CACHE_PACKAGE_FRONTEND_CA_DIR` | Host directory for the frontend CA. |
 | `INFINITO_CACHE_PACKAGE_FRONTEND_CERTS_DIR` | Host directory for per-hostname leaf certs. |
 | `INFINITO_CACHE_PACKAGE_FRONTEND_IP` | Static IPv4 address for the frontend on the compose default network. |