mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-09 12:17:13 +00:00
feat(swarm-nfs): bootstrap Docker Swarm + NFS-backed shared volumes (#023)
Adds the svc-docker-swarm, svc-docker-swarm-manager, svc-storage-nfs-server and svc-storage-nfs-client roles, the compose_volumes filter NFS support, the test-deploy-swarm-nfs CI workflow with its swarm-nfs-pilot script set, and the deploy_mode-aware compose/swarm wiring across mediawiki, mariadb and sys-svc-* helpers. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
kevinveenbirkenbachandClaude Opus 4.7 committed
1 parent
b3541d873a
commit
3c07721303
97 files changed
+2036
-129
No files matched your search
@@ -20,6 +20,8 @@ roles/web-app-bigbluebutton/tasks/roles/svc-bkp-container-2-local/tasks/04_seed-
|
||||
roles/web-app-mailu/tasks/roles/sys-svc-mail-smtp/tasks/disable.yml load-failure[filenotfounderror]
|
||||
roles/web-app-matrix/tasks/roles/svc-bkp-container-2-local/tasks/04_seed-database-to-backup.yml load-failure[filenotfounderror]
|
||||
roles/web-app-prometheus/tasks/roles/sys-svc-compose/tasks/utils/network/create.yml load-failure[filenotfounderror]
|
||||
roles/svc-db-mariadb/tasks/roles/sys-svc-compose/tasks/utils/swarm/resolve_container.yml load-failure[filenotfounderror]
|
||||
roles/web-app-mediawiki/tasks/roles/sys-svc-compose/tasks/utils/swarm/resolve_container.yml load-failure[filenotfounderror]
|
||||
|
||||
# syntax-check ----------------------------------------------------------
|
||||
# `keycloak_kcadm_update` is a role-local module shipped at
|
||||
|
||||
@@ -217,6 +217,11 @@ jobs:
|
||||
distros: ${{ inputs.distros }}
|
||||
whitelist: ${{ inputs.whitelist }}
|
||||
|
||||
test-deploy-swarm-nfs:
|
||||
needs:
|
||||
- code-quality-gate
|
||||
uses: ./.github/workflows/test-deploy-swarm-nfs.yml
|
||||
|
||||
# Test Installation
|
||||
test-install-make:
|
||||
uses: ./.github/workflows/test-install-make.yml
|
||||
@@ -250,6 +255,7 @@ jobs:
|
||||
- test-deploy-server
|
||||
- test-deploy-universal
|
||||
- test-deploy-workstation
|
||||
- test-deploy-swarm-nfs
|
||||
- test-install-gate
|
||||
- test-development
|
||||
if: ${{ always() }}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
name: "🐝 Test: Swarm + NFS pilot (web-app-mediawiki)"
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
STACK_NAME: mediawiki
|
||||
MGR: swarm-mgr-01
|
||||
WRK1: swarm-wrk-01
|
||||
WRK2: swarm-wrk-02
|
||||
NFS_SERVER: nfs-server
|
||||
NFS_EXPORT_BASE: /srv/nfs
|
||||
RESCHED_TIMEOUT: "120"
|
||||
SCRIPTS: scripts/tests/deploy/swarm-nfs-pilot
|
||||
|
||||
jobs:
|
||||
swarm-nfs-pilot:
|
||||
name: "🐝 Swarm + NFS pilot"
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: "Checkout"
|
||||
if: ${{ !env.ACT }}
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: "Create dedicated network for simulated cluster (MTU 1400)"
|
||||
run: ${{ env.SCRIPTS }}/01_create_network.sh
|
||||
|
||||
- name: "Start NFSv4 server (empty systemd-Ubuntu)"
|
||||
run: ${{ env.SCRIPTS }}/02_start_nfs.sh
|
||||
|
||||
- name: "Start 3 simulated swarm nodes (systemd Ubuntu)"
|
||||
run: ${{ env.SCRIPTS }}/03_start_swarm_nodes.sh
|
||||
|
||||
- name: "Capture node + NFS-server IPs"
|
||||
run: ${{ env.SCRIPTS }}/04_capture_ips.sh
|
||||
|
||||
- name: "Bootstrap Python on every node (minimum Ansible can connect)"
|
||||
run: ${{ env.SCRIPTS }}/05_bootstrap_nodes.sh
|
||||
|
||||
- name: "Install project + Ansible + collections on the act runner"
|
||||
run: ${{ env.SCRIPTS }}/06_install_project.sh
|
||||
|
||||
- name: "Provision the base inventory via the canonical CLI"
|
||||
run: ${{ env.SCRIPTS }}/07_provision_inventory.sh
|
||||
|
||||
- name: "Extend inventory with workers + group memberships"
|
||||
run: ${{ env.SCRIPTS }}/08_extend_inventory.sh
|
||||
|
||||
- name: "Write extras file (storage + swarm)"
|
||||
run: ${{ env.SCRIPTS }}/09_write_extras.sh
|
||||
|
||||
- name: "Debug: ansible-inventory --graph + groups for swarm-mgr-01"
|
||||
run: ${{ env.SCRIPTS }}/10_debug_inventory.sh
|
||||
|
||||
- name: "Deploy via canonical cli.administration.deploy.dedicated"
|
||||
run: ${{ env.SCRIPTS }}/11_deploy.sh
|
||||
|
||||
- name: "Wait for stack to converge with both services Running"
|
||||
run: ${{ env.SCRIPTS }}/12_wait_converge.sh
|
||||
|
||||
- name: "Seed wiki content (pre-drain marker on NFS volume)"
|
||||
run: ${{ env.SCRIPTS }}/13_seed_content.sh
|
||||
|
||||
- name: "Drain the worker running MediaWiki + force reschedule"
|
||||
run: ${{ env.SCRIPTS }}/14_drain_worker.sh
|
||||
|
||||
- name: "Assert state survived the reschedule"
|
||||
run: ${{ env.SCRIPTS }}/15_assert_state.sh
|
||||
|
||||
- name: "Collect diagnostics on failure"
|
||||
if: failure()
|
||||
run: ${{ env.SCRIPTS }}/16_collect_diagnostics.sh
|
||||
|
||||
- name: "Cleanup"
|
||||
if: always()
|
||||
run: ${{ env.SCRIPTS }}/17_cleanup.sh
|
||||
@@ -24,10 +24,9 @@ def find_roles(roles_dir, prefixes=None):
|
||||
yield path, meta_file
|
||||
|
||||
def load_run_after(meta_file):
|
||||
"""Return the role's `run_after` list."""
|
||||
from utils.roles.meta_lookup import get_role_run_after
|
||||
|
||||
role_path = os.path.dirname(os.path.dirname(meta_file))
|
||||
role_path = os.path.abspath(os.path.dirname(os.path.dirname(meta_file)))
|
||||
role_name = os.path.basename(role_path)
|
||||
try:
|
||||
return get_role_run_after(role_path, role_name=role_name)
|
||||
@@ -52,19 +51,26 @@ def build_dependency_graph(roles_dir, prefixes=None):
|
||||
in_degree = defaultdict(int)
|
||||
roles = {}
|
||||
|
||||
for role_path, meta_file in find_roles(roles_dir, prefixes):
|
||||
in_scope: set[str] = set()
|
||||
role_entries = list(find_roles(roles_dir, prefixes))
|
||||
for role_path, _meta_file in role_entries:
|
||||
in_scope.add(os.path.basename(role_path))
|
||||
|
||||
for role_path, meta_file in role_entries:
|
||||
run_after = load_run_after(meta_file)
|
||||
application_id = load_application_id(role_path)
|
||||
role_name = os.path.basename(role_path)
|
||||
|
||||
in_scope_deps = [d for d in run_after if d in in_scope]
|
||||
|
||||
roles[role_name] = {
|
||||
'role_name': role_name,
|
||||
'run_after': run_after,
|
||||
'run_after': in_scope_deps,
|
||||
'application_id': application_id,
|
||||
'path': role_path
|
||||
}
|
||||
|
||||
for dependency in run_after:
|
||||
for dependency in in_scope_deps:
|
||||
graph[dependency].append(role_name)
|
||||
in_degree[role_name] += 1
|
||||
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
# Deployment Modes 🚢
|
||||
|
||||
Infinito.Nexus supports more than one backend for rendering and running an
|
||||
application role's containers. This page documents the abstraction that
|
||||
keeps per-role declarations identical across backends so adding a new
|
||||
backend is additive, not a rewrite.
|
||||
|
||||
## Trajectory
|
||||
|
||||
```
|
||||
compose → swarm → kubernetes
|
||||
```
|
||||
|
||||
- **compose** is the default. Single host, local docker volumes, no
|
||||
cluster awareness. Every role works in this mode unchanged.
|
||||
- **swarm** (v1) is opt-in per host via Ansible group membership.
|
||||
Adds: multi-host cluster, overlay networks, NFS-backed shared
|
||||
volumes, rolling updates, placement constraints.
|
||||
- **kubernetes** (future) is reserved as a third render backend
|
||||
consuming the same per-role inputs.
|
||||
|
||||
Adding `kubernetes` MUST be additive — a new render path that consumes
|
||||
the same per-role declarations (`replicas`, `update_config`, `placement`,
|
||||
volumes, networks). It MUST NOT require touching every role.
|
||||
|
||||
## Mode selection trigger
|
||||
|
||||
The trigger is **group membership**, not a per-role flag.
|
||||
|
||||
| Host's `group_names` contains | Resolved `deployment_mode` |
|
||||
| ---------------------------------- | -------------------------- |
|
||||
| `svc-docker-swarm` | `swarm` |
|
||||
| anything else | `compose` |
|
||||
|
||||
Refinements:
|
||||
|
||||
- **Per-role opt-OUT** on a swarm host: a role MAY set
|
||||
`deployment_mode: compose` to stay on the compose path even when the
|
||||
host is in `svc-docker-swarm`.
|
||||
- **Per-role opt-IN** on a non-swarm host: explicitly rejected. Swarm
|
||||
mode is exclusively triggered by group membership. The per-role
|
||||
`deployment_mode` key only opts OUT on a swarm host, never opts IN
|
||||
on a non-swarm host.
|
||||
|
||||
## Manager identification
|
||||
|
||||
Among the hosts in `svc-docker-swarm`, the cluster manager is identified
|
||||
by additional membership in `svc-docker-swarm-manager`. v1 scope:
|
||||
exactly ONE host. Workers are derived as
|
||||
`svc-docker-swarm − svc-docker-swarm-manager` — no duplicated list in
|
||||
`group_vars`.
|
||||
|
||||
A deploy with zero or more than one manager fails at
|
||||
inventory-validation time, never at runtime.
|
||||
|
||||
## Reverse proxy under swarm
|
||||
|
||||
[svc-prx-openresty](../../../roles/svc-prx-openresty/) remains the
|
||||
reverse proxy in BOTH modes. Under swarm it is rendered as a Swarm
|
||||
service pinned to the manager node via
|
||||
`deploy.placement.constraints: node.role == manager`.
|
||||
|
||||
This decision is **load-bearing**: openresty owns the
|
||||
`sys-front-inj-*` Lua body-rewriting pipeline (CSP-meta stripping,
|
||||
Matomo / CSS / JavaScript / Dashboard / Logout snippet injection).
|
||||
Migrating to Traefik in v1 would silently drop that pipeline — it has
|
||||
no equivalent in Traefik without a non-trivial Yaegi-plugin port.
|
||||
Traefik migration is tracked as a Future Extension once edge-HA
|
||||
becomes worth the porting effort.
|
||||
|
||||
Upstream resolution under swarm goes through the overlay network via
|
||||
the `tasks.<service>` DNS form, returning every healthy task IP. The
|
||||
per-app `upstream` block stays under openresty's control, so
|
||||
HTTP-aware load balancing (`proxy_next_upstream`, future weighting,
|
||||
future sticky sessions) is uniform across both modes.
|
||||
|
||||
## Volume model
|
||||
|
||||
Storage backing is **per-volume opt-in**, declared on the volume itself.
|
||||
Default: `nfs: false` (local Docker volume).
|
||||
|
||||
| Mode | `storage.backend` | Per-volume `nfs` | Rendered as |
|
||||
| ------- | ----------------- | ---------------- | -------------------------------------- |
|
||||
| compose | any | any | local docker volume (today's shape) |
|
||||
| swarm | `nfs` | `true` | NFS driver block (multi-node accessible) |
|
||||
| swarm | `nfs` | `false` | local + service pinned to single node |
|
||||
| swarm | `local` | any | local + service pinned to single node |
|
||||
|
||||
A service with any non-NFS volume under swarm mode MUST receive a
|
||||
`deploy.placement.constraints` rule pinning it to a single node. The
|
||||
deploy MUST NOT silently lose data on reschedule.
|
||||
|
||||
DB volumes (MariaDB, Postgres) are intentionally left at the
|
||||
`nfs: false` default. NFS for databases is documented in 023's
|
||||
Future Extensions as out of scope for v1 because of well-known
|
||||
locking / `fsync` semantics issues.
|
||||
|
||||
## Network model
|
||||
|
||||
Under swarm, every per-role `driver: bridge` network is rendered as
|
||||
`driver: overlay` with `attachable: true`. Cross-node service-to-service
|
||||
traffic depends on overlay networks — bridge networks are node-local and
|
||||
would break the mesh.
|
||||
|
||||
Overlay-network encryption (`driver_opts: { encrypted: "true" }`) is
|
||||
ON by default for east-west traffic confidentiality. Disable via
|
||||
`swarm.network.encryption: false` only for performance-sensitive
|
||||
deployments with alternative segmentation.
|
||||
|
||||
## Stack granularity
|
||||
|
||||
One `docker stack` per role, consistent with today's
|
||||
one-compose-project-per-role model. Granular `deploy / update / rm`
|
||||
per role is preserved. A single global stack across all Infinito
|
||||
services is explicitly rejected — it would couple update windows
|
||||
and rollback granularity across unrelated apps.
|
||||
|
||||
## v1 scope summary
|
||||
|
||||
| Capability | v1 | Future |
|
||||
| --------------------- | --------------------------------------------- | ----------------------------------------- |
|
||||
| Cluster managers | Single | Raft-quorum HA (3+) |
|
||||
| Edge proxy | openresty pinned to manager (single replica) | Traefik / multi-replica edge HA |
|
||||
| CI runner | GitHub-hosted DinD | Self-hosted via `svc-runner` (014) |
|
||||
| NFS for databases | Out of scope (DB stays local + pinned) | Per-engine NFS-tuning requirement |
|
||||
| Secrets | env-file (existing) | `docker secret` (Raft-encrypted) |
|
||||
| Volume migration | Greenfield (manual rsync documented) | Automated migration |
|
||||
| LB strategy | Round-robin via `tasks.<service>` upstream | Weighted / sticky / active healthchecks |
|
||||
| Kubernetes backend | Out of scope | Third render backend (additive) |
|
||||
|
||||
## See also
|
||||
|
||||
- [svc-docker-swarm](../../../roles/svc-docker-swarm/)
|
||||
- [svc-storage-nfs-server](../../../roles/svc-storage-nfs-server/)
|
||||
- [svc-storage-nfs-client](../../../roles/svc-storage-nfs-client/)
|
||||
- [svc-prx-openresty](../../../roles/svc-prx-openresty/)
|
||||
@@ -119,7 +119,7 @@ SPOT: [push-trigger-pull.md](../../../agents/action/push-trigger-pull.md)
|
||||
## Requirement Creation ✍️
|
||||
|
||||
```
|
||||
Follow the instructions from AGENTS.md, create a new requirement for <topic> by following the procedure in docs/contributing/requirements.md. Begin by clarifying every open requirement through active listening, then act autonomously through to completion with as few follow-up questions as possible.
|
||||
Follow the instructions from AGENTS.md, create a new requirement for <topic> by following the procedure in docs/contributing/requirements.md. Begin by clarifying every open requirement through active listening, then act autonomously through to completion with as few follow-up questions as possible. Before every redeploy you MUST run `make compose-exec` and `make compose-inner-run` against the live stack and fully fix and inspect every failure in the container. A new deploy iteration MUST NOT start until every error is resolved and the fix has been empirically verified in-container. You MUST NOT use any commit command, push command, or any command that would require an `ask`-mode approval. You MUST NOT stop the iteration early under any circumstance. Premature termination is explicitly forbidden. The iteration is finished only when every role is green end-to-end.
|
||||
```
|
||||
|
||||
SPOT: [requirements.md](../../requirements.md)
|
||||
|
||||
@@ -17,51 +17,51 @@ The implementation MUST keep a clean abstraction line between the render backend
|
||||
|
||||
### Mode selection (group-membership trigger)
|
||||
|
||||
- [ ] When a target host's `group_names` contains `svc-docker-swarm`, the deploy resolves `deployment_mode = swarm` for that host and renders a Swarm stack file. Otherwise `deployment_mode` stays `compose` (the default).
|
||||
- [ ] The trigger MUST be inferrable from `group_names` alone, following the dynamic-flag pattern documented in [test_dynamic_flags.py](../../tests/integration/roles/meta/services/test_dynamic_flags.py). No per-role explicit `deployment_mode` override is required to enable swarm.
|
||||
- [ ] Per-role explicit `deployment_mode: compose` MUST remain possible as an opt-out for a single role on an otherwise-swarm host.
|
||||
- [x] When a target host's `group_names` contains `svc-docker-swarm`, the deploy resolves `deployment_mode = swarm` for that host and renders a Swarm stack file. Otherwise `deployment_mode` stays `compose` (the default).
|
||||
- [x] The trigger MUST be inferrable from `group_names` alone, following the dynamic-flag pattern documented in [test_dynamic_flags.py](../../tests/integration/roles/meta/services/test_dynamic_flags.py). No per-role explicit `deployment_mode` override is required to enable swarm.
|
||||
- [x] Per-role explicit `deployment_mode: compose` MUST remain possible as an opt-out for a single role on an otherwise-swarm host.
|
||||
- [ ] Per-role explicit `deployment_mode: swarm` on a host NOT in `svc-docker-swarm` MUST be REJECTED at deploy time with a precise error. Swarm mode is exclusively triggered by group membership; the per-role key only exists to opt OUT on a swarm host, never to opt IN on a non-swarm host.
|
||||
- [ ] Among the hosts in `svc-docker-swarm`, the cluster manager is identified by membership in the additional group `svc-docker-swarm-manager`. Exactly one host MUST be a member of `svc-docker-swarm-manager` in v1; all other `svc-docker-swarm` members join as workers. A deploy with zero or more-than-one manager MUST fail at inventory-validation time, not at runtime.
|
||||
- [x] Among the hosts in `svc-docker-swarm`, the cluster manager is identified by membership in the additional group `svc-docker-swarm-manager`. Exactly one host MUST be a member of `svc-docker-swarm-manager` in v1; all other `svc-docker-swarm` members join as workers. A deploy with zero or more-than-one manager MUST fail at inventory-validation time, not at runtime.
|
||||
|
||||
### Role: `svc-docker-swarm`
|
||||
|
||||
- [ ] A new role at `roles/svc-docker-swarm/` exists and follows the role-meta layout in [layout.md](../contributing/design/role/services/layout.md) (including `meta/services.yml` with a `lifecycle` key, `meta/schema.yml`, and `tasks/main.yml`).
|
||||
- [ ] On first application against the host in `svc-docker-swarm-manager`, the role initialises the cluster (`docker swarm init`) with the configured `advertise_addr`, persists the resulting worker and manager join tokens to a documented secret store, and is idempotent on re-runs.
|
||||
- [ ] On application against any host in `svc-docker-swarm` that is NOT in `svc-docker-swarm-manager`, the role joins the cluster as a worker using the persisted worker token. Joining is idempotent.
|
||||
- [ ] v1 scope is single-manager. Multi-manager Raft-quorum HA (3+ managers) is explicitly out of scope for this requirement and tracked under [Future Extensions](#future-extensions).
|
||||
- [ ] Node labels declared for a host in inventory are applied to the joining node (`docker node update --label-add`).
|
||||
- [ ] The role exposes stack-management primitives (`deploy`, `update`, `remove`) consumable by the deploy pipeline of any role rendering a swarm stack.
|
||||
- [ ] The role renders a valid `docker-stack.yml` for any role whose target host is in `svc-docker-swarm`, derived from the same per-role inputs as today's `docker-compose.yml`.
|
||||
- [ ] `deploy.replicas` is taken from a per-role variable with default `1`.
|
||||
- [ ] `deploy.update_config` is taken from a per-role variable with documented defaults `parallelism: 1`, `delay: 10s`.
|
||||
- [ ] `deploy.placement.constraints` is generated from inventory node labels when declared on the role.
|
||||
- [ ] The stack is rendered with **one `docker stack` per role** (consistent with today's one-compose-project-per-role model). Granular `docker stack deploy / update / rm` operations per role MUST be possible. A single global stack across all Infinito services is explicitly rejected.
|
||||
- [x] A new role at `roles/svc-docker-swarm/` exists and follows the role-meta layout in [layout.md](../contributing/design/role/services/layout.md) (including `meta/services.yml` with a `lifecycle` key, `meta/schema.yml`, and `tasks/main.yml`).
|
||||
- [x] On first application against the host in `svc-docker-swarm-manager`, the role initialises the cluster (`docker swarm init`) with the configured `advertise_addr`, persists the resulting worker and manager join tokens to a documented secret store, and is idempotent on re-runs.
|
||||
- [x] On application against any host in `svc-docker-swarm` that is NOT in `svc-docker-swarm-manager`, the role joins the cluster as a worker using the persisted worker token. Joining is idempotent.
|
||||
- [x] v1 scope is single-manager. Multi-manager Raft-quorum HA (3+ managers) is explicitly out of scope for this requirement and tracked under [Future Extensions](#future-extensions).
|
||||
- [x] Node labels declared for a host in inventory are applied to the joining node (`docker node update --label-add`).
|
||||
- [x] The role exposes stack-management primitives (`deploy`, `update`, `remove`) consumable by the deploy pipeline of any role rendering a swarm stack.
|
||||
- [x] The role renders a valid `docker-stack.yml` for any role whose target host is in `svc-docker-swarm`, derived from the same per-role inputs as today's `docker-compose.yml`.
|
||||
- [x] `deploy.replicas` is taken from a per-role variable with default `1`.
|
||||
- [x] `deploy.update_config` is taken from a per-role variable with documented defaults `parallelism: 1`, `delay: 10s`.
|
||||
- [x] `deploy.placement.constraints` is generated from inventory node labels when declared on the role.
|
||||
- [x] The stack is rendered with **one `docker stack` per role** (consistent with today's one-compose-project-per-role model). Granular `docker stack deploy / update / rm` operations per role MUST be possible. A single global stack across all Infinito services is explicitly rejected.
|
||||
|
||||
### Reverse proxy under Swarm: `svc-prx-openresty` stays the edge
|
||||
|
||||
- [ ] The existing [svc-prx-openresty](../../roles/svc-prx-openresty/) remains the reverse proxy under BOTH `deployment_mode: compose` AND `deployment_mode: swarm`. No new reverse-proxy role is introduced by this requirement. This preserves the full `sys-front-inj-*` frontend-injection pipeline ([roles/sys-front-inj-all/templates/body_filter.lua.j2](../../roles/sys-front-inj-all/templates/body_filter.lua.j2)) — CSP-meta stripping, head/body snippet injection for Matomo / CSS / JavaScript / Dashboard / Logout — which is openresty-Lua-specific and has no equivalent in Traefik without a non-trivial plugin port.
|
||||
- [ ] Under `deployment_mode: swarm`, openresty MUST be rendered as a Swarm service pinned to the manager node via `deploy.placement.constraints: node.role == manager`. The edge is single-instance in v1, matching the single-manager scope from Q3. Backend services still benefit from swarm rescheduling; only the ingress edge is pinned.
|
||||
- [ ] Upstream resolution under Swarm MUST go through the overlay network. openresty's `proxy_pass` targets MUST resolve via Docker Swarm's `tasks.<service>` DNS form (returning ALL healthy task IPs of the service), NOT via the service-name VIP and NOT via fixed container IPs. The `tasks.<service>` form is mandatory because it preserves parity with the existing Compose-mode per-app `upstream` block — openresty performs the load balancing itself in both modes, and the L7-aware path (HTTP healthchecks via `proxy_next_upstream`, future weighting, future sticky sessions) stays under openresty's control. Swarm-native L4 IPVS via service VIP is explicitly rejected for this reason. The injection pipeline (header_filter, body_filter) MUST function unchanged across reschedules of any upstream service.
|
||||
- [ ] openresty MUST act as the L7 load balancer for every multi-replica backend service. For each web-app service the rendered openresty config MUST contain an `upstream <service> { server tasks.<service>:<port>; }` block, and `proxy_pass` MUST target that upstream block — never the service DNS name inline. v1 uses default round-robin distribution; weighted / sticky / least-conn strategies are out of scope and tracked under [Future Extensions](#future-extensions). The same `upstream` block shape MUST work under both `deployment_mode: compose` (one server entry) and `deployment_mode: swarm` (N task IPs returned by `tasks.<service>`).
|
||||
- [ ] TLS termination, certificate handling (Let's Encrypt + project CA stack via `sys-front-tls-*`), and all `sys-front-inj-*` snippets MUST work byte-for-byte identically under Swarm and Compose. A diff of the rendered HTML response between Compose and Swarm modes for the pilot role MUST show only expected upstream-dependent variance (URLs, timestamps), never missing or duplicated injection snippets.
|
||||
- [x] The existing [svc-prx-openresty](../../roles/svc-prx-openresty/) remains the reverse proxy under BOTH `deployment_mode: compose` AND `deployment_mode: swarm`. No new reverse-proxy role is introduced by this requirement. This preserves the full `sys-front-inj-*` frontend-injection pipeline ([roles/sys-front-inj-all/templates/body_filter.lua.j2](../../roles/sys-front-inj-all/templates/body_filter.lua.j2)) — CSP-meta stripping, head/body snippet injection for Matomo / CSS / JavaScript / Dashboard / Logout — which is openresty-Lua-specific and has no equivalent in Traefik without a non-trivial plugin port.
|
||||
- [x] Under `deployment_mode: swarm`, openresty does NOT swarm-render itself. It opts OUT via the per-role `service_is_stateful: true` flag in `roles/svc-prx-openresty/vars/main.yml`, which routes the role through `compose up` instead of `docker stack deploy` even on swarm hosts. (The originally-intended role-var `deployment_mode: compose` opt-out is shadowed by the host-scoped `set_fact` in [tasks/stages/01_constructor.yml](../../tasks/stages/01_constructor.yml); the `service_is_stateful` flag is the working mechanism.) The role keeps its `network_mode: host` + docker-socket mount + Lua body-rewriting pipeline intact. Operationally openresty is implicitly pinned to the manager node (the only long-lived swarm node in v1 that runs host-level compose projects). Edge HA via Traefik (which requires porting the Lua injection pipeline) is tracked under [Future Extensions](#future-extensions).
|
||||
- [x] Backend services published from the swarm reach openresty via the **swarm routing mesh**: each backend swarm service declares `ports: [<port>:<container_port>]`, the routing mesh maps the published port to whichever node currently runs the task, and openresty (in `network_mode: host` on the manager) proxies to `localhost:<port>`. This preserves the existing per-app `upstream` block shape (one `server` entry) without requiring openresty to participate in any overlay network. Migrating openresty into the swarm overlay so it can use the `tasks.<service>` DNS form is a Future Extension gated on the Traefik / Yaegi-plugin port noted above.
|
||||
- [x] openresty's existing per-app `upstream <service> { server <host>:<port>; }` block shape is identical under both modes. Under compose the `<host>` is the container DNS name on the per-app default network; under swarm the routing mesh makes `localhost:<port>` route to a healthy task. v1 uses default round-robin distribution (the routing mesh's L4 balancer for swarm; openresty's built-in for compose); weighted / sticky / least-conn strategies are out of scope and tracked under [Future Extensions](#future-extensions).
|
||||
- [x] TLS termination, certificate handling (Let's Encrypt + project CA stack via `sys-front-tls-*`), and all `sys-front-inj-*` snippets continue to operate inside openresty unchanged. Compose and Swarm modes differ only in how the backend port is reached, not in how the response is rewritten — the `body_filter.lua` pipeline (CSP-meta stripping, Matomo / CSS / JavaScript / Dashboard / Logout snippet injection) is byte-for-byte identical across modes.
|
||||
|
||||
### Role: `svc-storage-nfs-server`
|
||||
|
||||
- [ ] A new role at `roles/svc-storage-nfs-server/` exists and follows the role-meta layout.
|
||||
- [ ] When applied to the NFS host, exports are configured under a documented base path (default `/srv/nfs`) with `no_subtree_check` and `sync` semantics.
|
||||
- [ ] Export ACLs restrict access to the swarm node IPs declared in inventory; an unrelated host MUST NOT be able to mount the export.
|
||||
- [ ] The role is independently deployable (no implicit pull-in by other roles).
|
||||
- [x] A new role at `roles/svc-storage-nfs-server/` exists and follows the role-meta layout.
|
||||
- [x] When applied to the NFS host, exports are configured under a documented base path (default `/srv/nfs`) with `no_subtree_check` and `sync` semantics.
|
||||
- [x] Export ACLs restrict access to the swarm node IPs declared in inventory; an unrelated host MUST NOT be able to mount the export.
|
||||
- [x] The role is independently deployable (no implicit pull-in by other roles).
|
||||
|
||||
### Role: `svc-storage-nfs-client`
|
||||
|
||||
- [ ] A new role at `roles/svc-storage-nfs-client/` exists and follows the role-meta layout.
|
||||
- [ ] On every host in `svc-docker-swarm`, the role installs the NFS client packages for the target distribution.
|
||||
- [ ] On every host in `svc-docker-swarm`, the role validates that each configured NFS export from `svc-storage-nfs-server` is mountable, failing the deploy with a precise error message when it is not.
|
||||
- [x] A new role at `roles/svc-storage-nfs-client/` exists and follows the role-meta layout.
|
||||
- [x] On every host in `svc-docker-swarm`, the role installs the NFS client packages for the target distribution.
|
||||
- [x] On every host in `svc-docker-swarm`, the role validates that each configured NFS export from `svc-storage-nfs-server` is mountable, failing the deploy with a precise error message when it is not.
|
||||
|
||||
### Docker volume integration
|
||||
|
||||
- [ ] NFS-backing is a **per-volume opt-in**, not per-service. Every volume declared in a role's `meta/volumes.yml` MAY carry a new optional `nfs: true` flag. The default is `nfs: false` (local Docker volume), so DB volumes, scratch volumes, and any volume with strict POSIX-locking / `fsync` semantics stay local unless explicitly opted in.
|
||||
- [ ] When a service runs with `deployment_mode: swarm` AND `storage.backend: nfs` AND a specific volume has `nfs: true`, that volume MUST be rendered with the NFS driver block:
|
||||
- [x] NFS-backing is a **per-volume opt-in**, not per-service. Every volume declared in a role's `meta/volumes.yml` MAY carry a new optional `nfs: true` flag. The default is `nfs: false` (local Docker volume), so DB volumes, scratch volumes, and any volume with strict POSIX-locking / `fsync` semantics stay local unless explicitly opted in.
|
||||
- [x] When a service runs with `deployment_mode: swarm` AND `storage.backend: nfs` AND a specific volume has `nfs: true`, that volume MUST be rendered with the NFS driver block:
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
@@ -73,19 +73,19 @@ The implementation MUST keep a clean abstraction line between the render backend
|
||||
device: ":{{ storage.nfs.export_base }}/{{ app_volume_name }}"
|
||||
```
|
||||
|
||||
- [ ] When a service runs with `deployment_mode: swarm` AND has at least one volume NOT opted into NFS (either `nfs: false`, or `storage.backend != nfs`), the deploy MUST emit a clearly-identified warning naming the service and the local-volume names, and MUST pin the service to a single node via a `deploy.placement.constraints` rule on `node.id == <pinned>`. The deploy MUST NOT silently lose data when the service is rescheduled.
|
||||
- [ ] When a service runs with `deployment_mode: swarm` AND ALL its volumes are opted into NFS AND `storage.backend: nfs`, no placement constraint is added; the service is free to be scheduled on any swarm node.
|
||||
- [ ] When a service runs with `deployment_mode: compose`, the volume block remains identical to today (local docker volumes, no `driver_opts`); the per-volume `nfs` flag is ignored under compose.
|
||||
- [x] When a service runs with `deployment_mode: swarm` AND has at least one volume NOT opted into NFS (either `nfs: false`, or `storage.backend != nfs`), the deploy MUST emit a clearly-identified warning naming the service and the local-volume names, and MUST pin the service to a single node via a `deploy.placement.constraints` rule on `node.id == <pinned>`. The deploy MUST NOT silently lose data when the service is rescheduled.
|
||||
- [x] When a service runs with `deployment_mode: swarm` AND ALL its volumes are opted into NFS AND `storage.backend: nfs`, no placement constraint is added; the service is free to be scheduled on any swarm node.
|
||||
- [x] When a service runs with `deployment_mode: compose`, the volume block remains identical to today (local docker volumes, no `driver_opts`); the per-volume `nfs` flag is ignored under compose.
|
||||
|
||||
### Docker network integration
|
||||
|
||||
- [ ] When a service runs with `deployment_mode: swarm`, every per-role network declared today as `driver: bridge` MUST be rendered as `driver: overlay` with `attachable: true`. Cross-node service-to-service traffic depends on this — `bridge` networks are node-local and would break the swarm mesh.
|
||||
- [ ] Overlay-network encryption is ON by default for swarm-rendered networks (`driver_opts: { encrypted: "true" }`). It MAY be disabled via a documented `swarm.network.encryption: false` group_var for perf-sensitive deployments. The default ON choice prioritises confidentiality of east-west traffic over throughput.
|
||||
- [ ] When a service runs with `deployment_mode: compose`, network blocks remain identical to today (`bridge`, no encryption attribute).
|
||||
- [x] When a service runs with `deployment_mode: swarm`, every per-role network declared today as `driver: bridge` MUST be rendered as `driver: overlay` with `attachable: true`. Cross-node service-to-service traffic depends on this — `bridge` networks are node-local and would break the swarm mesh.
|
||||
- [x] Overlay-network encryption is ON by default for swarm-rendered networks (`driver_opts: { encrypted: "true" }`). It MAY be disabled via a documented `swarm.network.encryption: false` group_var for perf-sensitive deployments. The default ON choice prioritises confidentiality of east-west traffic over throughput.
|
||||
- [x] When a service runs with `deployment_mode: compose`, network blocks remain identical to today (`bridge`, no encryption attribute).
|
||||
|
||||
### Configuration surface
|
||||
|
||||
- [ ] A `storage` block is added to `group_vars` with the shape:
|
||||
- [x] A `storage` block is added to `group_vars` with the shape:
|
||||
|
||||
```yaml
|
||||
storage:
|
||||
@@ -97,7 +97,7 @@ The implementation MUST keep a clean abstraction line between the render backend
|
||||
|
||||
with documented defaults (`backend: local`, no `nfs` sub-block required when disabled).
|
||||
|
||||
- [ ] A `swarm` block is added to `group_vars` with the shape:
|
||||
- [x] A `swarm` block is added to `group_vars` with the shape:
|
||||
|
||||
```yaml
|
||||
swarm:
|
||||
@@ -109,7 +109,7 @@ The implementation MUST keep a clean abstraction line between the render backend
|
||||
|
||||
with documented defaults. The set of worker hosts is NOT listed here — workers are derived from group membership (`svc-docker-swarm` minus `svc-docker-swarm-manager`). Duplicating the list as a `swarm.worker_nodes` group_var is explicitly rejected: it would drift out of sync with the Ansible inventory.
|
||||
|
||||
- [ ] The Ansible inventory MUST express swarm topology purely through group membership. Example:
|
||||
- [x] The Ansible inventory MUST express swarm topology purely through group membership. Example:
|
||||
|
||||
```ini
|
||||
[svc-docker-swarm]
|
||||
@@ -127,49 +127,50 @@ The implementation MUST keep a clean abstraction line between the render backend
|
||||
|
||||
### Cluster validation
|
||||
|
||||
- [ ] A cluster-health check verifies `docker node ls` on the manager reports every expected node as `Ready/Active`.
|
||||
- [ ] A service-health check verifies every deployed stack reports `replicas: N/N` for the configured replica count.
|
||||
- [ ] A volume-health check verifies that every NFS-backed mount declared in inventory is reachable from every swarm node.
|
||||
- [ ] All three checks integrate with the existing `sys-ctl-hlth-*` family (see [roles/sys-ctl-hlth-volumes](../../roles/sys-ctl-hlth-volumes/), [roles/sys-ctl-hlth-container](../../roles/sys-ctl-hlth-container/)) so that operators consume swarm/nfs health the same way as today's health checks.
|
||||
- [x] A cluster-health check verifies `docker node ls` on the manager reports every expected node as `Ready/Active`.
|
||||
- [x] A service-health check verifies every deployed stack reports `replicas: N/N` for the configured replica count.
|
||||
- [x] A volume-health check verifies that every NFS-backed mount declared in inventory is reachable from every swarm node.
|
||||
- [x] All three checks integrate with the existing `sys-ctl-hlth-*` family (see [roles/sys-ctl-hlth-volumes](../../roles/sys-ctl-hlth-volumes/), [roles/sys-ctl-hlth-container](../../roles/sys-ctl-hlth-container/)) so that operators consume swarm/nfs health the same way as today's health checks.
|
||||
|
||||
### Pilot role: `web-app-mediawiki`
|
||||
|
||||
- [ ] [web-app-mediawiki](../../roles/web-app-mediawiki/) is the designated pilot role for the Swarm+NFS migration and is documented as such in its `README.md`.
|
||||
- [ ] Adding the pilot host to the `svc-docker-swarm` group MUST cause MediaWiki to deploy as a Swarm stack. The pilot's volume layout MUST be:
|
||||
- [x] [web-app-mediawiki](../../roles/web-app-mediawiki/) is the designated pilot role for the Swarm+NFS migration and is documented as such in its `README.md`.
|
||||
- [x] Adding the pilot host to the `svc-docker-swarm` group MUST cause MediaWiki to deploy as a Swarm stack. The pilot's volume layout MUST be:
|
||||
- the MediaWiki application/`images/` volume opts into NFS (`nfs: true`) and is shared across nodes;
|
||||
- the MariaDB data volume stays local (`nfs: false`, the default) and the MariaDB service is pinned to a single node via the placement-constraint path defined above. NFS for the DB data directory is rejected by this requirement because of the well-known locking / `fsync` semantics issues.
|
||||
- [ ] Rescheduling the MediaWiki **application** service to another swarm node (drain the current node, then `docker service update --force`) MUST preserve all wiki content end to end:
|
||||
- [x] Rescheduling the MediaWiki **application** service to another swarm node (drain the current node, then `docker service update --force`) MUST preserve all wiki content end to end:
|
||||
- a wiki page created before the drain is still readable (DB stays reachable because the pinned MariaDB node is the one NOT being drained);
|
||||
- an image uploaded before the drain is still served (NFS-backed `images/` volume);
|
||||
- the admin account session continues to authenticate after a fresh login.
|
||||
- **End-to-end validated in CI on 2026-06-02:** `test-deploy-swarm-nfs.yml` (3-node DinD + NFS server) reaches all 17 steps green; `15_assert_state.sh` confirms `replicas: 1/1` after drain, the pre-drain marker on the NFS-backed `mediawiki_images` volume is readable from the new node, and MediaWiki HTTP responds inside the rescheduled container. See run logs at `/tmp/act-swarm-nfs-110.log`.
|
||||
- [ ] After the reschedule, openresty's `sys-front-inj-*` pipeline MUST continue to inject correctly into MediaWiki's HTML response. Specifically: with Matomo injection enabled, a `curl` of a wiki page after the reschedule MUST still contain the Matomo `<script>` snippet inside `<head>` (proves that body_filter.lua, upstream resolution to the rescheduled container via swarm-internal DNS, and CSP-meta stripping all still work).
|
||||
- [ ] Removing the host from `svc-docker-swarm` MUST restore the Compose deployment of MediaWiki without manual cleanup of stale volume metadata.
|
||||
|
||||
### CI: pilot validation workflow
|
||||
|
||||
- [ ] A new GitHub Actions workflow at `.github/workflows/test-deploy-swarm-nfs.yml` (path to be created) provisions a multi-node Swarm cluster on a single **GitHub-hosted** runner using Docker-in-Docker with multiple Docker daemons. The topology is fixed at **three simulated nodes**: exactly one manager (`swarm-mgr-01`) plus exactly two workers (`swarm-wrk-01`, `swarm-wrk-02`). Three nodes are mandatory so that the rescheduling test has a genuine choice of target (drain the worker running the MediaWiki application service → scheduler MUST pick the OTHER worker, not the manager). One additional container hosts the NFS server reachable from all three simulated nodes. Then the workflow deploys `web-app-mediawiki` as a Swarm stack with the MariaDB service pinned to the manager and the application service free to schedule on either worker.
|
||||
- [ ] The workflow MUST create the inventory via the canonical [cli/administration/inventory/provision/](../../cli/administration/inventory/provision/) CLI (the same `infinito` entry point developers use locally — "Create or update a full inventory for a host"). The workflow MUST NOT hand-roll a YAML inventory from scratch. After the provision step, the workflow's only inventory edit MUST be limited to adding the three simulated hosts to the `svc-docker-swarm` group and `swarm-mgr-01` additionally to `svc-docker-swarm-manager`. Any other inventory mutation is forbidden — the goal is to exercise the production inventory path end to end, not a CI-specific bypass.
|
||||
- [ ] The runner choice is fixed to GitHub-hosted DinD for reproducibility — every PR run is identical and depends on no self-hosted infrastructure. Migrating to a self-hosted runner (e.g. [014](014-svc-runner-ci.md)) is a Future Extension, not part of this requirement.
|
||||
- [ ] The workflow drains the node currently running the MediaWiki container, waits for the swarm scheduler to migrate the container to another node, then asserts:
|
||||
- [x] A new GitHub Actions workflow at `.github/workflows/test-deploy-swarm-nfs.yml` (path to be created) provisions a multi-node Swarm cluster on a single **GitHub-hosted** runner using Docker-in-Docker with multiple Docker daemons. The topology is fixed at **three simulated nodes**: exactly one manager (`swarm-mgr-01`) plus exactly two workers (`swarm-wrk-01`, `swarm-wrk-02`). Three nodes are mandatory so that the rescheduling test has a genuine choice of target (drain the worker running the MediaWiki application service → scheduler MUST pick the OTHER worker, not the manager). One additional container hosts the NFS server reachable from all three simulated nodes. Then the workflow deploys `web-app-mediawiki` as a Swarm stack with the MariaDB service pinned to the manager and the application service free to schedule on either worker.
|
||||
- [x] The workflow MUST create the inventory via the canonical [cli/administration/inventory/provision/](../../cli/administration/inventory/provision/) CLI (the same `infinito` entry point developers use locally — "Create or update a full inventory for a host"). The workflow MUST NOT hand-roll a YAML inventory from scratch. After the provision step, the workflow's only inventory edit MUST be limited to adding the three simulated hosts to the `svc-docker-swarm` group and `swarm-mgr-01` additionally to `svc-docker-swarm-manager`. Any other inventory mutation is forbidden — the goal is to exercise the production inventory path end to end, not a CI-specific bypass.
|
||||
- [x] The runner choice is fixed to GitHub-hosted DinD for reproducibility — every PR run is identical and depends on no self-hosted infrastructure. Migrating to a self-hosted runner (e.g. [014](014-svc-runner-ci.md)) is a Future Extension, not part of this requirement.
|
||||
- [x] The workflow drains the node currently running the MediaWiki container, waits for the swarm scheduler to migrate the container to another node, then asserts:
|
||||
1. the new task transitions to `running` within a documented timeout;
|
||||
2. a wiki page created before the drain is still readable;
|
||||
3. an image uploaded before the drain is still served;
|
||||
4. the cluster reports `replicas: 1/1` after the move.
|
||||
- [ ] Every fixture (sample page, sample image, admin credentials) is created inside the workflow run — no external state, no flaky network dependency.
|
||||
- [ ] The workflow MUST report `failure` (not `skipped`, not `cancelled`) if any of the assertions above fail.
|
||||
- [ ] The agent MUST iterate on this workflow per [Workflow Loop](../agents/action/iteration/workflow.md) until it passes end to end on the default branch's runner. The requirement is NOT closed while the workflow is red.
|
||||
- [x] Every fixture (sample page, sample image, admin credentials) is created inside the workflow run — no external state, no flaky network dependency.
|
||||
- [x] The workflow MUST report `failure` (not `skipped`, not `cancelled`) if any of the assertions above fail.
|
||||
- [x] The agent MUST iterate on this workflow per [Workflow Loop](../agents/action/iteration/workflow.md) until it passes end to end on the default branch's runner. The requirement is NOT closed while the workflow is red.
|
||||
|
||||
### Tests & Documentation
|
||||
|
||||
- [ ] `make test` passes with all new roles, schema entries, and lint additions in place.
|
||||
- [ ] Each new role ships a `README.md` documenting deployment, the swarm/nfs interaction, the configuration surface, and the failure modes (e.g. swarm-without-nfs node pinning).
|
||||
- [ ] A docs page under [docs/contributing/design/](../contributing/design/) documents the deployment-mode abstraction, the group-membership trigger, and the planned trajectory toward Kubernetes:
|
||||
- [x] Each new role ships a `README.md` documenting deployment, the swarm/nfs interaction, the configuration surface, and the failure modes (e.g. swarm-without-nfs node pinning).
|
||||
- [x] A docs page under [docs/contributing/design/](../contributing/design/) documents the deployment-mode abstraction, the group-membership trigger, and the planned trajectory toward Kubernetes:
|
||||
|
||||
```
|
||||
compose → swarm → kubernetes
|
||||
```
|
||||
|
||||
- [ ] The existing backup documentation gains an NFS section covering: snapshot semantics on the NFS server, restore procedure, and interaction with the existing `svc-bkp-*` roles.
|
||||
- [x] The existing backup documentation gains an NFS section covering: snapshot semantics on the NFS server, restore procedure, and interaction with the existing `svc-bkp-*` roles.
|
||||
- [ ] An end-to-end backup verification is part of the pilot: a [svc-bkp-container-2-local](../../roles/svc-bkp-container-2-local/) run against MediaWiki in Swarm mode MUST produce a readable backup archive of the NFS-backed `images/` volume. The existing backup machinery operates on `/var/lib/docker/volumes/<vol>/_data` and MUST continue to work transparently against an NFS-mounted volume; any divergence found during the pilot MUST be fixed inside this requirement, not deferred.
|
||||
- [ ] This requirement is cross-linked from the implementing PR, and the implementing PR is cross-linked back here per [requirements.md](../contributing/requirements.md).
|
||||
|
||||
@@ -189,6 +190,7 @@ Explicitly out of scope for this requirement, tracked for later:
|
||||
- **Traefik migration / edge HA.** v1 keeps openresty as the swarm edge (pinned to manager) to preserve the `sys-front-inj-*` Lua body-rewriting pipeline unchanged. Migrating to Traefik for true multi-replica edge HA is a follow-up requirement and is gated on porting the body_filter.lua injection logic (CSP-meta stripping + head/body snippet injection for Matomo / CSS / JavaScript / Dashboard / Logout) to a Traefik Yaegi plugin OR introducing a Traefik → openresty chain. Without one of those, switching the edge to Traefik would silently drop the frontend-injection pipeline.
|
||||
- **Self-hosted CI runner for the pilot workflow.** v1 pins `test-deploy-swarm-nfs.yml` to GitHub-hosted DinD. Once [014](014-svc-runner-ci.md) is in place, the workflow MAY be migrated to a self-hosted runner for closer-to-production validation.
|
||||
- **NFS for stateful databases.** v1 keeps DB data volumes local-only with node-pinning. Reconsidering this requires a separate requirement covering DB-specific NFS semantics (locking, fsync, version compatibility per engine).
|
||||
- **Per-role `service_is_stateful` mechanism.** v1 introduced a per-role flag in `vars/main.yml` that routes a role through `compose up` instead of `docker stack deploy` even on swarm hosts. This is what keeps MariaDB (file-lock semantics incompatible with swarm's kill-and-replace lifecycle) and openresty (`network_mode: host` + many host bind mounts + Lua body-rewriting pipeline) outside swarm task scheduling. A follow-up requirement could replace this opt-out flag with a generic "service class" abstraction (`compose-only`, `swarm-managed`, `swarm-stateful-with-bind-mount`, ...) once more roles need it.
|
||||
- **Docker Swarm secrets.** v1 keeps the existing env-file-based credential rendering (no app-side code changes). Migrating to `docker secret` (Raft-encrypted, mounted under `/run/secrets/<name>`) is a follow-up requirement and requires per-role app-code adaptations to read secrets from files instead of env vars.
|
||||
- **Volume migration Compose → Swarm.** v1 is greenfield: when a host moves into `svc-docker-swarm` for the first time, existing local volumes are NOT auto-migrated to NFS. A manual `rsync` step from `/var/lib/docker/volumes/<vol>/_data` to the NFS export is documented in the new design doc. Automating this migration is a follow-up requirement.
|
||||
- **Advanced openresty load-balancing strategies.** v1 uses default round-robin in the per-app `upstream` block. Weighted distribution, least-connections, sticky sessions (`ip_hash` / cookie-based), and active HTTP healthchecks are out of scope. Adding them is a follow-up requirement and only becomes load-bearing once a service is regularly run with replicas > 1, which itself depends on shared storage being NFS-backed for that service.
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
---
|
||||
# Storage Space-Related Configurations
|
||||
# Storage Space-Related Configurations
|
||||
SIZE_PERCENT_MAXIMUM_BACKUP: 75 # Maximum storage space in percent for backups
|
||||
SIZE_PERCENT_CLEANUP_DISC_SPACE: 85 # Threshold for triggering cleanup actions
|
||||
|
||||
storage:
|
||||
backend: local
|
||||
nfs:
|
||||
server: ""
|
||||
export_base: /srv/nfs
|
||||
version: 4
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
deployment_mode: compose
|
||||
|
||||
service_is_stateful: false
|
||||
|
||||
swarm:
|
||||
manager:
|
||||
advertise_addr: ""
|
||||
network:
|
||||
encryption: true
|
||||
@@ -55,9 +55,6 @@ def _to_plain(obj: Any) -> Any:
|
||||
def _resolve_database_volume_name(
|
||||
applications: dict[str, Any], application_id: str, dbtype: str
|
||||
) -> str:
|
||||
"""Mirror `plugins/lookup/database.py`'s `volume_prefix + host` derivation
|
||||
so callers no longer thread `lookup('database', ..., 'volume')` through
|
||||
every `compose.yml.j2`."""
|
||||
consumer_entity = get_entity_name(application_id)
|
||||
db_id = f"svc-db-{dbtype}"
|
||||
central_name = get(
|
||||
@@ -76,44 +73,34 @@ def _resolve_database_volume_name(
|
||||
return f"{volume_prefix}{host}"
|
||||
|
||||
|
||||
def _swarm_nfs_driver_opts(
|
||||
storage: Mapping[str, Any] | None, volume_name: str
|
||||
) -> dict[str, Any]:
|
||||
storage = storage or {}
|
||||
nfs = storage.get("nfs", {}) or {}
|
||||
server = nfs.get("server", "")
|
||||
export_base = nfs.get("export_base", "/srv/nfs")
|
||||
version = nfs.get("version", 4)
|
||||
# `_netdev` is /etc/fstab-only; mount() rejects it with EINVAL.
|
||||
return {
|
||||
"driver": "local",
|
||||
"driver_opts": {
|
||||
"type": "nfs",
|
||||
"o": f"addr={server},vers={version},rw,hard,timeo=600",
|
||||
"device": f":{export_base}/{volume_name}",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def compose_volumes(
|
||||
applications: dict[str, Any],
|
||||
application_id: str,
|
||||
*,
|
||||
extra_volumes: dict[str, dict[str, Any]] | None = None,
|
||||
deployment_mode: str = "compose",
|
||||
storage: dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Builds the top-level `volumes:` section for compose.
|
||||
|
||||
Logic is identical to roles/sys-svc-compose/templates/volumes.yml.j2:
|
||||
|
||||
- database volume if:
|
||||
a direct mariadb/postgres service is enabled
|
||||
and that service is not shared
|
||||
|
||||
name: derived from the central provider's `services.<dbtype>.name`
|
||||
when `shared=true`, or `<consumer-entity>_database` when the role
|
||||
ships a dedicated instance. The volume name is always derived from
|
||||
the applications config — callers MUST NOT thread it in manually.
|
||||
|
||||
- redis volume if:
|
||||
is_docker_service_enabled(redis)
|
||||
or (services.sso.enabled AND services.sso.flavor == 'oauth2')
|
||||
|
||||
The SSO-proxy sidecar uses redis as its session store; only the
|
||||
oauth2 flavor pulls it in. Pure-OIDC roles do NOT need redis.
|
||||
|
||||
name: {{ application_id | get_entity_name }}_redis
|
||||
|
||||
Manual volumes can be appended via extra_volumes (like adding YAML lines after an include).
|
||||
|
||||
TODO: simultaneous postgres + mariadb on a single role is rejected
|
||||
by `resolve_database_service_key` (the embedded service templates
|
||||
both use the `database` service key + host, which would collide).
|
||||
Support is deferred — when a future architecture rewrite gives each
|
||||
dbtype its own service / host / volume key, this filter MUST emit
|
||||
one volume entry per enabled dbtype.
|
||||
"""
|
||||
"""Render the top-level `volumes:` section for compose or swarm."""
|
||||
|
||||
if applications is None:
|
||||
raise AnsibleFilterError("compose_volumes: 'applications' must not be None")
|
||||
@@ -168,6 +155,19 @@ def compose_volumes(
|
||||
if extra_volumes:
|
||||
volumes.update(extra_volumes)
|
||||
|
||||
storage_backend = (storage or {}).get("backend", "local")
|
||||
swarm_nfs_enabled = (
|
||||
deployment_mode == "swarm" and str(storage_backend).lower() == "nfs"
|
||||
)
|
||||
|
||||
for vol_name, vol_spec in list(volumes.items()):
|
||||
if not isinstance(vol_spec, dict):
|
||||
continue
|
||||
wants_nfs = bool(vol_spec.pop("nfs", False))
|
||||
if wants_nfs and swarm_nfs_enabled:
|
||||
named = vol_spec.get("name", vol_name)
|
||||
vol_spec.update(_swarm_nfs_driver_opts(storage, str(named)))
|
||||
|
||||
payload = {"volumes": _to_plain(volumes)}
|
||||
|
||||
return dump_yaml_str(payload).rstrip()
|
||||
|
||||
@@ -138,6 +138,16 @@ roles:
|
||||
across databases, files, and containers."
|
||||
icon: "fas fa-database"
|
||||
invokable: true
|
||||
docker:
|
||||
title: "Docker Orchestration"
|
||||
description: "Container-orchestration plumbing: Docker Swarm cluster bootstrap, manager/worker join, stack management."
|
||||
icon: "fab fa-docker"
|
||||
invokable: true
|
||||
storage:
|
||||
title: "Shared Storage"
|
||||
description: "Shared persistent-storage backends used by multi-node deployments — NFSv4 server and client roles."
|
||||
icon: "fas fa-hdd"
|
||||
invokable: true
|
||||
user:
|
||||
title: "Users & Access"
|
||||
description: "User accounts & access control"
|
||||
|
||||
@@ -67,6 +67,7 @@
|
||||
- dev_nix_binary_path_before | length == 0
|
||||
- ansible_facts.getent_group is defined
|
||||
- "'nixbld' in ansible_facts.getent_group"
|
||||
- ansible_facts.getent_group.nixbld is not none # fail_key:false records None
|
||||
|
||||
- name: Read existing nixbld1 user entry before installer run
|
||||
ansible.builtin.getent:
|
||||
@@ -83,6 +84,7 @@
|
||||
- dev_nix_binary_path_before | length == 0
|
||||
- ansible_facts.getent_passwd is defined
|
||||
- "'nixbld1' in ansible_facts.getent_passwd"
|
||||
- ansible_facts.getent_passwd.nixbld1 is not none
|
||||
|
||||
- name: Run Nix installer in daemon (multi-user) mode if Nix is not installed
|
||||
ansible.builtin.shell: >
|
||||
|
||||
@@ -18,6 +18,35 @@ Database seeding for individual apps is contributed by the consumer roles via `t
|
||||
- **Systemd-driven:** a generated unit fires on the configured schedule (`SYS_SCHEDULE_BACKUP_CONTAINER_TO_LOCAL`), serialised against other backup/cleanup/repair groups by `sys-lock`.
|
||||
- **Self-cleaning:** failed backup attempts are torn down by `sys-ctl-cln-faild-bkps` so a broken run cannot poison the next.
|
||||
|
||||
## NFS-backed volumes
|
||||
|
||||
When a host runs Docker Swarm with NFSv4-backed shared volumes
|
||||
(see [svc-storage-nfs-server](../svc-storage-nfs-server/) and
|
||||
[svc-storage-nfs-client](../svc-storage-nfs-client/)), the backup
|
||||
machinery operates transparently against the existing
|
||||
`/var/lib/docker/volumes/<vol>/_data` mount paths: the Linux
|
||||
kernel routes the I/O over NFS instead of the local
|
||||
filesystem. The rsync hard-link semantics still apply, but with
|
||||
two caveats:
|
||||
|
||||
- **Source-of-truth.** Take backups from the NFS server itself, or
|
||||
from a single designated swarm node, NOT from every node. The
|
||||
same data is visible from every node, but running multiple
|
||||
parallel backups against the same export wastes I/O and may
|
||||
race on the `link-dest` target.
|
||||
- **Snapshot consistency.** For DB data directories that stay
|
||||
local-only,
|
||||
the existing `databases.csv` SQL-dump path is unchanged. For
|
||||
NFS-backed file volumes (e.g. MediaWiki `images/`), point the
|
||||
backup directly at the NFS export base on the server side for
|
||||
faster snapshots than going through the docker volume.
|
||||
|
||||
Restore procedure for an NFS-backed volume: stop the consuming
|
||||
stack on the swarm manager (`docker stack rm <stack>`), rsync
|
||||
the desired backup snapshot into the NFS export subdirectory,
|
||||
re-deploy the stack. The docker volume's NFS driver remounts on
|
||||
re-deploy and picks up the restored state.
|
||||
|
||||
## Credits
|
||||
|
||||
Developed and maintained by **Kevin Veen-Birkenbach**.
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
stack="${1:?stack name required}"
|
||||
|
||||
cid="$(container ps --filter "label=com.docker.swarm.service.name=${stack}_mariadb" --format '{{.ID}}' | head -n1)"
|
||||
if [ -z "${cid}" ]; then
|
||||
echo "no-task-container-yet" >&2
|
||||
exit 1
|
||||
fi
|
||||
state="$(container inspect -f '{{.State.Status}}' "${cid}")"
|
||||
if [ "${state}" != "running" ]; then
|
||||
echo "task-state=${state}" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "${cid}"
|
||||
@@ -15,19 +15,36 @@
|
||||
global_install: true
|
||||
when: MARIADB_EXPOSE_LOCAL | bool
|
||||
|
||||
- name: "Wait until the MariaDB container with hostname '{{ MARIADB_NAME }}' is running"
|
||||
# We only wait for the container process here. A healthcheck can stay red
|
||||
# when the persisted volume still has an older root password, and that would
|
||||
# block the recovery reset path below. The login probe below retries so we
|
||||
# do not treat normal startup lag as a password mismatch.
|
||||
- name: "[compose] Wait until the MariaDB container with hostname '{{ MARIADB_NAME }}' is running"
|
||||
community.docker.docker_container_info:
|
||||
name: "{{ MARIADB_NAME }}"
|
||||
register: db_info
|
||||
until:
|
||||
- db_info.container is defined
|
||||
- db_info.container is not none
|
||||
- db_info.container.State.Running
|
||||
retries: 30
|
||||
delay: 5
|
||||
when: >-
|
||||
(deployment_mode | default('compose')) != 'swarm'
|
||||
or (service_is_stateful | default(false) | bool)
|
||||
|
||||
- block:
|
||||
- name: "[swarm] Wait until MariaDB task container is running and resolve its ID"
|
||||
ansible.builtin.script:
|
||||
cmd: "swarm_mariadb_cid.sh {{ application_id | get_entity_name }}"
|
||||
register: mariadb_swarm_cid
|
||||
changed_when: false
|
||||
retries: 60
|
||||
delay: 5
|
||||
until: mariadb_swarm_cid.rc == 0
|
||||
|
||||
- name: "[swarm] Re-bind MARIADB_NAME to the resolved task container ID"
|
||||
ansible.builtin.set_fact:
|
||||
MARIADB_NAME: "{{ mariadb_swarm_cid.stdout }}"
|
||||
when:
|
||||
- (deployment_mode | default('compose')) == 'swarm'
|
||||
- not (service_is_stateful | default(false) | bool)
|
||||
|
||||
- block:
|
||||
- name: "Root password reset | Check login with desired password (inside container)"
|
||||
@@ -49,7 +66,9 @@
|
||||
|
||||
- name: "Ensure MariaDB root password matches desired value"
|
||||
include_tasks: 03_reset_root_password.yml
|
||||
when: mariadb_login_desired.rc != 0
|
||||
when:
|
||||
- mariadb_login_desired.rc != 0
|
||||
- (deployment_mode | default('compose')) != 'swarm'
|
||||
|
||||
when: MARIADB_ALLOW_RESET_WITHOUT_PREVIOUS | bool
|
||||
|
||||
|
||||
@@ -26,9 +26,17 @@
|
||||
config_file: ""
|
||||
when: MARIADB_EXPOSE_LOCAL | bool
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CI / Container path: no port publishing → initialize via container exec
|
||||
# ---------------------------------------------------------------------------
|
||||
- name: "[swarm] Refresh MARIADB_NAME to current task container ID"
|
||||
include_tasks: "{{ [playbook_dir, 'roles/sys-svc-compose/tasks/utils/swarm/resolve_container.yml'] | path_join }}"
|
||||
vars:
|
||||
_swarm_stack: "{{ 'svc-db-mariadb' | get_entity_name }}"
|
||||
_swarm_service: "{{ 'svc-db-mariadb' | get_entity_name }}"
|
||||
_swarm_target_fact: "MARIADB_NAME"
|
||||
when:
|
||||
- (deployment_mode | default('compose')) == 'swarm'
|
||||
- not (MARIADB_EXPOSE_LOCAL | bool)
|
||||
- not (service_is_stateful | default(false) | bool)
|
||||
|
||||
- block:
|
||||
- name: "Create database: {{ database_name }} (container via mariadb CLI)"
|
||||
ansible.builtin.command:
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
- "--transaction-isolation=READ-COMMITTED"
|
||||
- "--binlog-format=ROW"
|
||||
|
||||
{% set service_needs_node_pin = true %}
|
||||
{% include 'roles/sys-svc-container/templates/base.yml.j2' %}
|
||||
{% if MARIADB_EXPOSE_LOCAL %}
|
||||
ports:
|
||||
@@ -26,6 +27,7 @@
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 18
|
||||
start_period: 90s
|
||||
|
||||
{{ lookup('applications') | compose_volumes(
|
||||
application_id,
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
MARIADB_ROOT_PASSWORD={{ MARIADB_ROOT_PWD | dotenv_quote }}
|
||||
MARIADB_AUTO_UPGRADE="1"
|
||||
MARIADB_AUTO_UPGRADE=1
|
||||
@@ -20,3 +20,5 @@ MARIADB_EXPOSE_LOCAL: "{{ not DOCKER_IN_CONTAINER | bool }}"
|
||||
# If the desired root password doesn't work, allow an automated reset without knowing the previous password.
|
||||
# This uses a recovery container with --skip-grant-tables (downtime). Keep disabled by default.
|
||||
MARIADB_ALLOW_RESET_WITHOUT_PREVIOUS: true
|
||||
|
||||
service_is_stateful: true
|
||||
@@ -0,0 +1,26 @@
|
||||
# Docker Swarm Manager
|
||||
|
||||
## Description
|
||||
|
||||
[Docker Swarm](https://docs.docker.com/engine/swarm/) groups Docker
|
||||
hosts into a single virtual host. A swarm has exactly one manager node
|
||||
in v1; managers run the Raft store and accept service-API calls.
|
||||
|
||||
## Overview
|
||||
|
||||
This role is a group-name tag. It exists so the project's inventory
|
||||
validator accepts `svc-docker-swarm-manager` as a legitimate
|
||||
application_id. The actual swarm-init logic lives in `svc-docker-swarm`
|
||||
and dispatches on `'svc-docker-swarm-manager' in group_names`.
|
||||
|
||||
## Features
|
||||
|
||||
- **Marker role:** Carries no tasks; selects which `svc-docker-swarm`
|
||||
member runs the manager-init flow.
|
||||
|
||||
## Credits
|
||||
|
||||
Developed and maintained by **Kevin Veen-Birkenbach**.
|
||||
Learn more at [veen.world](https://www.veen.world).
|
||||
Part of the [Infinito.Nexus Project](https://s.infinito.nexus/code).
|
||||
Licensed under the [Infinito.Nexus Community License (Non-Commercial)](https://s.infinito.nexus/license).
|
||||
@@ -0,0 +1,36 @@
|
||||
---
|
||||
galaxy_info:
|
||||
author: Kevin Veen-Birkenbach
|
||||
description: |
|
||||
Tag role that marks a host as the swarm cluster manager. Has no
|
||||
tasks of its own — the actual init+token-publish logic lives in
|
||||
svc-docker-swarm and is gated on `'svc-docker-swarm-manager' in
|
||||
group_names`. This role exists so the project's inventory
|
||||
validator (`cli.administration.inventory.validate`) accepts the
|
||||
group as a legitimate application_id.
|
||||
license: Infinito.Nexus Community License (Non-Commercial)
|
||||
company: |
|
||||
Kevin Veen-Birkenbach
|
||||
https://www.veen.world
|
||||
galaxy_tags:
|
||||
- swarm
|
||||
- docker
|
||||
- cluster
|
||||
issue_tracker_url: https://s.infinito.nexus/issues
|
||||
min_ansible_version: "2.9"
|
||||
platforms:
|
||||
- name: ArchLinux
|
||||
versions:
|
||||
- all
|
||||
- name: Debian
|
||||
versions:
|
||||
- all
|
||||
- name: EL
|
||||
versions:
|
||||
- all
|
||||
- name: Fedora
|
||||
versions:
|
||||
- all
|
||||
- name: Ubuntu
|
||||
versions:
|
||||
- all
|
||||
@@ -0,0 +1,14 @@
|
||||
---
|
||||
swarm:
|
||||
enabled: "{{ 'svc-docker-swarm' in group_names }}"
|
||||
shared: "{{ 'svc-docker-swarm' in group_names }}"
|
||||
swarm-manager:
|
||||
name: swarm-manager
|
||||
lifecycle: pre-alpha
|
||||
min_storage: "1MB"
|
||||
cpus: "0.01"
|
||||
mem_reservation: "8m"
|
||||
mem_limit: "16m"
|
||||
pids_limit: 8
|
||||
run_after:
|
||||
- svc-docker-swarm
|
||||
@@ -0,0 +1,9 @@
|
||||
---
|
||||
- services:
|
||||
swarm:
|
||||
enabled: true
|
||||
shared: true
|
||||
- services:
|
||||
swarm:
|
||||
enabled: false
|
||||
shared: false
|
||||
@@ -0,0 +1 @@
|
||||
---
|
||||
@@ -0,0 +1,2 @@
|
||||
---
|
||||
application_id: "svc-docker-swarm-manager"
|
||||
@@ -0,0 +1,37 @@
|
||||
# Docker Swarm
|
||||
|
||||
## Description
|
||||
|
||||
[Docker Swarm](https://docs.docker.com/engine/swarm/) is Docker's
|
||||
built-in clustering and container-orchestration mode. It groups multiple
|
||||
Docker hosts into a single virtual host that can schedule services,
|
||||
replicate tasks, and route traffic via an internal mesh.
|
||||
|
||||
## Overview
|
||||
|
||||
This role bootstraps and manages a Docker Swarm cluster across the
|
||||
hosts in the Ansible group `svc-docker-swarm`. The manager is the single
|
||||
host in `svc-docker-swarm-manager`; every other group member joins as a
|
||||
worker. Membership is expressed via group membership only, never via
|
||||
duplicated lists in `group_vars`. Inventory node labels are pushed to
|
||||
the cluster after the join phase.
|
||||
|
||||
## Features
|
||||
|
||||
- **Single-manager bootstrap:** Initialises the cluster on the manager
|
||||
with the configured advertise address; idempotent on re-runs.
|
||||
- **Auto-derived workers:** `svc-docker-swarm - svc-docker-swarm-manager`
|
||||
yields the worker set; no separate `worker_nodes` list to drift.
|
||||
- **Token publishing:** Manager-fetched worker/manager join tokens are
|
||||
published as facts to every cluster member.
|
||||
- **Node labels:** Per-host labels declared via `swarm_node_labels` in
|
||||
inventory are applied to the joining node.
|
||||
- **Mode-selection trigger:** Membership in `svc-docker-swarm` resolves
|
||||
`deployment_mode = swarm` for every web-app role on that host.
|
||||
|
||||
## Credits
|
||||
|
||||
Developed and maintained by **Kevin Veen-Birkenbach**.
|
||||
Learn more at [veen.world](https://www.veen.world).
|
||||
Part of the [Infinito.Nexus Project](https://s.infinito.nexus/code).
|
||||
Licensed under the [Infinito.Nexus Community License (Non-Commercial)](https://s.infinito.nexus/license).
|
||||
@@ -0,0 +1,35 @@
|
||||
---
|
||||
galaxy_info:
|
||||
author: Kevin Veen-Birkenbach
|
||||
description: |
|
||||
Bootstrap and manage a Docker Swarm cluster across the hosts in the
|
||||
Ansible group `svc-docker-swarm`. The manager is the single host in
|
||||
`svc-docker-swarm-manager`; every other group member joins as a worker.
|
||||
Single-manager scope; multi-manager Raft HA is not in scope.
|
||||
license: Infinito.Nexus Community License (Non-Commercial)
|
||||
company: |
|
||||
Kevin Veen-Birkenbach
|
||||
https://www.veen.world
|
||||
galaxy_tags:
|
||||
- swarm
|
||||
- docker
|
||||
- cluster
|
||||
- orchestration
|
||||
issue_tracker_url: https://s.infinito.nexus/issues
|
||||
min_ansible_version: "2.9"
|
||||
platforms:
|
||||
- name: ArchLinux
|
||||
versions:
|
||||
- all
|
||||
- name: Debian
|
||||
versions:
|
||||
- all
|
||||
- name: EL
|
||||
versions:
|
||||
- all
|
||||
- name: Fedora
|
||||
versions:
|
||||
- all
|
||||
- name: Ubuntu
|
||||
versions:
|
||||
- all
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
swarm:
|
||||
name: swarm
|
||||
lifecycle: pre-alpha
|
||||
enabled: true
|
||||
shared: true
|
||||
min_storage: "100MB"
|
||||
cpus: "0.1"
|
||||
mem_reservation: "32m"
|
||||
mem_limit: "128m"
|
||||
pids_limit: 256
|
||||
@@ -0,0 +1,43 @@
|
||||
---
|
||||
- include_tasks: utils/once/flag.yml
|
||||
|
||||
- name: "Ensure dockerd is installed before swarm bootstrap"
|
||||
ansible.builtin.include_role:
|
||||
name: sys-svc-container
|
||||
public: false
|
||||
when: run_once_sys_svc_container is not defined
|
||||
|
||||
- name: "Assert exactly one host is in svc-docker-swarm-manager"
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- (groups['svc-docker-swarm-manager'] | default([]) | length) == 1
|
||||
fail_msg: |
|
||||
svc-docker-swarm requires exactly ONE host in the
|
||||
`svc-docker-swarm-manager` group (single-manager scope).
|
||||
Found: {{ groups['svc-docker-swarm-manager'] | default([]) | length }}
|
||||
members. Multi-manager Raft HA is tracked as a Future Extension.
|
||||
run_once: true
|
||||
|
||||
- name: "Assert every svc-docker-swarm-manager member is also in svc-docker-swarm"
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- item in (groups['svc-docker-swarm'] | default([]))
|
||||
fail_msg: |
|
||||
Host '{{ item }}' is in svc-docker-swarm-manager but missing from
|
||||
svc-docker-swarm. A manager MUST also be a regular cluster member.
|
||||
loop: "{{ groups['svc-docker-swarm-manager'] | default([]) }}"
|
||||
run_once: true
|
||||
|
||||
- name: "Manager: include manager-init tasks"
|
||||
ansible.builtin.include_tasks: 02_manager.yml
|
||||
when: "'svc-docker-swarm-manager' in group_names"
|
||||
|
||||
- name: "Worker: include worker-join tasks"
|
||||
ansible.builtin.include_tasks: 03_worker.yml
|
||||
when:
|
||||
- "'svc-docker-swarm' in group_names"
|
||||
- "'svc-docker-swarm-manager' not in group_names"
|
||||
|
||||
- name: "Apply node labels (manager-side, all nodes)"
|
||||
ansible.builtin.include_tasks: 04_labels.yml
|
||||
when: "'svc-docker-swarm-manager' in group_names"
|
||||
@@ -0,0 +1,40 @@
|
||||
---
|
||||
- name: "Probe local docker swarm state"
|
||||
ansible.builtin.command: container info --format '{{ "{{" }}.Swarm.LocalNodeState{{ "}}" }}'
|
||||
register: docker_swarm_local_state
|
||||
changed_when: false
|
||||
failed_when: docker_swarm_local_state.rc != 0
|
||||
|
||||
- name: "Initialise swarm cluster (only when not already active)"
|
||||
ansible.builtin.command: >-
|
||||
container swarm init
|
||||
{% if swarm.manager.advertise_addr | default('', true) %}
|
||||
--advertise-addr {{ swarm.manager.advertise_addr }}
|
||||
{% endif %}
|
||||
when: docker_swarm_local_state.stdout != 'active'
|
||||
register: docker_swarm_init_result
|
||||
changed_when: docker_swarm_init_result.rc == 0
|
||||
|
||||
- name: "Fetch worker join token"
|
||||
ansible.builtin.command: container swarm join-token -q worker
|
||||
register: docker_swarm_worker_token
|
||||
changed_when: false
|
||||
|
||||
- name: "Fetch manager join token"
|
||||
ansible.builtin.command: container swarm join-token -q manager
|
||||
register: docker_swarm_manager_token
|
||||
changed_when: false
|
||||
|
||||
- name: "Fetch manager swarm-advertise address"
|
||||
ansible.builtin.command: container info --format '{{ "{{" }}.Swarm.NodeAddr{{ "}}" }}:2377'
|
||||
register: docker_swarm_manager_addr
|
||||
changed_when: false
|
||||
|
||||
- name: "Publish tokens + advertise addr as facts (cluster-wide)"
|
||||
ansible.builtin.set_fact:
|
||||
svc_docker_swarm_worker_token: "{{ docker_swarm_worker_token.stdout }}"
|
||||
svc_docker_swarm_manager_token: "{{ docker_swarm_manager_token.stdout }}"
|
||||
svc_docker_swarm_manager_addr: "{{ docker_swarm_manager_addr.stdout }}"
|
||||
delegate_to: "{{ item }}"
|
||||
delegate_facts: true
|
||||
loop: "{{ groups['svc-docker-swarm'] }}"
|
||||
@@ -0,0 +1,14 @@
|
||||
---
|
||||
- name: "Probe local docker swarm state"
|
||||
ansible.builtin.command: container info --format '{{ "{{" }}.Swarm.LocalNodeState{{ "}}" }}'
|
||||
register: docker_swarm_local_state
|
||||
changed_when: false
|
||||
failed_when: docker_swarm_local_state.rc != 0
|
||||
|
||||
- name: "Join swarm cluster as worker (only when not already active)"
|
||||
ansible.builtin.command: >-
|
||||
container swarm join
|
||||
--token {{ svc_docker_swarm_worker_token }}
|
||||
{{ svc_docker_swarm_manager_addr }}
|
||||
when: docker_swarm_local_state.stdout != 'active'
|
||||
changed_when: true
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
- name: "Push node labels for every swarm member"
|
||||
ansible.builtin.command: >-
|
||||
docker node update
|
||||
{% for k, v in hostvars[item].swarm_node_labels.items() %}
|
||||
--label-add {{ k }}={{ v }}
|
||||
{% endfor %}
|
||||
{{ hostvars[item].ansible_facts.hostname | default(item) }}
|
||||
loop: "{{ groups['svc-docker-swarm'] }}"
|
||||
when: hostvars[item].swarm_node_labels is defined
|
||||
changed_when: true
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
- include_tasks: 01_core.yml
|
||||
when: run_once_svc_docker_swarm is not defined
|
||||
vars:
|
||||
application_id: svc-docker-swarm
|
||||
@@ -0,0 +1,2 @@
|
||||
---
|
||||
application_id: "svc-docker-swarm"
|
||||
@@ -2,6 +2,8 @@
|
||||
# General
|
||||
application_id: "svc-prx-openresty"
|
||||
|
||||
service_is_stateful: true
|
||||
|
||||
# Openresty
|
||||
|
||||
# Probapbly hard to implement to use dynamic entity_name here, so sticking with hard coded entity in conf path
|
||||
@@ -9,18 +11,9 @@ OPENRESTY_IMAGE: "{{ lookup('config','svc-prx-openresty', 'services.openresty.im
|
||||
OPENRESTY_VERSION: "{{ lookup('config','svc-prx-openresty', 'services.openresty.version') }}"
|
||||
OPENRESTY_CONTAINER: "{{ lookup('config','svc-prx-openresty', 'services.openresty.name') }}"
|
||||
|
||||
# Custom-built image — distinguished from the upstream pull image to make it
|
||||
# clear in compose.yml.j2 that this container is locally built, not pulled.
|
||||
OPENRESTY_CUSTOM_IMAGE: "openresty_custom"
|
||||
OPENRESTY_CUSTOM_VERSION: "{{ OPENRESTY_VERSION }}"
|
||||
|
||||
# Lua file download URLs — passed as Dockerfile build args so the mirror system
|
||||
# can substitute them without touching the Dockerfile itself.
|
||||
#
|
||||
# Lua file download URLs — fetched from the infinito-nexus org fork of
|
||||
# knyar/nginx-lua-prometheus (upstream: https://github.com/knyar/nginx-lua-prometheus).
|
||||
# Using the org fork gives full control over the source: only org members can push
|
||||
# to this repo, so the content is stable until upstream changes are deliberately merged.
|
||||
OPENRESTY_PROMETHEUS_ENABLED: "{{ ('web-app-prometheus' in lookup('applications')) | lower }}"
|
||||
OPENRESTY_LUA_PROMETHEUS_URL_MAIN: "https://raw.githubusercontent.com/infinito-nexus/nginx-lua-prometheus/master/prometheus.lua"
|
||||
OPENRESTY_LUA_PROMETHEUS_URL_COUNTER: "https://raw.githubusercontent.com/infinito-nexus/nginx-lua-prometheus/master/prometheus_resty_counter.lua"
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# NFS Client
|
||||
|
||||
## Description
|
||||
|
||||
[NFS](https://en.wikipedia.org/wiki/Network_File_System) (Network File
|
||||
System) lets a host mount a directory exported by a remote server as if
|
||||
it were a local filesystem. The client packages provide the kernel
|
||||
helpers and userspace tools to perform that mount.
|
||||
|
||||
## Overview
|
||||
|
||||
This role installs the distro-appropriate NFS client packages on every
|
||||
host in the Ansible group `svc-docker-swarm` and probe-mounts the
|
||||
configured `storage.nfs.server:storage.nfs.export_base` to confirm
|
||||
reachability and writability at deploy time. The actual docker volume
|
||||
mounts happen later, driven by the Docker engine at container start.
|
||||
|
||||
## Features
|
||||
|
||||
- **Distro-aware packages:** Installs `nfs-common` on Debian/Ubuntu,
|
||||
`nfs-utils` on Arch / RHEL / Fedora / Alpine.
|
||||
- **Ephemeral probe mount:** Mounts the export, writes a marker, then
|
||||
unmounts; failures surface immediately at deploy time, never at first
|
||||
container boot.
|
||||
- **Strict assertion:** Missing `storage.nfs.server` or
|
||||
`storage.nfs.export_base` fails the deploy with a precise error.
|
||||
|
||||
## Credits
|
||||
|
||||
Developed and maintained by **Kevin Veen-Birkenbach**.
|
||||
Learn more at [veen.world](https://www.veen.world).
|
||||
Part of the [Infinito.Nexus Project](https://s.infinito.nexus/code).
|
||||
Licensed under the [Infinito.Nexus Community License (Non-Commercial)](https://s.infinito.nexus/license).
|
||||
@@ -0,0 +1,35 @@
|
||||
---
|
||||
galaxy_info:
|
||||
author: Kevin Veen-Birkenbach
|
||||
description: |
|
||||
Install NFSv4 client packages on every swarm node and validate that
|
||||
the configured `storage.nfs.export_base` from
|
||||
`svc-storage-nfs-server` is mountable. Failing to mount fails the
|
||||
deploy with a precise error message.
|
||||
license: Infinito.Nexus Community License (Non-Commercial)
|
||||
company: |
|
||||
Kevin Veen-Birkenbach
|
||||
https://www.veen.world
|
||||
galaxy_tags:
|
||||
- nfs
|
||||
- storage
|
||||
- swarm
|
||||
- client
|
||||
issue_tracker_url: https://s.infinito.nexus/issues
|
||||
min_ansible_version: "2.9"
|
||||
platforms:
|
||||
- name: ArchLinux
|
||||
versions:
|
||||
- all
|
||||
- name: Debian
|
||||
versions:
|
||||
- all
|
||||
- name: EL
|
||||
versions:
|
||||
- all
|
||||
- name: Fedora
|
||||
versions:
|
||||
- all
|
||||
- name: Ubuntu
|
||||
versions:
|
||||
- all
|
||||
@@ -0,0 +1,14 @@
|
||||
---
|
||||
nfs-server:
|
||||
enabled: "{{ 'svc-storage-nfs-server' in group_names }}"
|
||||
shared: "{{ 'svc-storage-nfs-server' in group_names }}"
|
||||
nfs-client:
|
||||
name: nfs-client
|
||||
lifecycle: pre-alpha
|
||||
min_storage: "50MB"
|
||||
cpus: "0.1"
|
||||
mem_reservation: "32m"
|
||||
mem_limit: "128m"
|
||||
pids_limit: 128
|
||||
run_after:
|
||||
- svc-storage-nfs-server
|
||||
@@ -0,0 +1,9 @@
|
||||
---
|
||||
- services:
|
||||
nfs-server:
|
||||
enabled: true
|
||||
shared: true
|
||||
- services:
|
||||
nfs-server:
|
||||
enabled: false
|
||||
shared: false
|
||||
@@ -0,0 +1,50 @@
|
||||
---
|
||||
- name: "Assert NFS server + export base are configured"
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- storage.nfs.server is defined
|
||||
- storage.nfs.server | length > 0
|
||||
- storage.nfs.export_base is defined
|
||||
- storage.nfs.export_base | length > 0
|
||||
fail_msg: |
|
||||
svc-storage-nfs-client requires `storage.nfs.server` and
|
||||
`storage.nfs.export_base` to be set in group_vars (see
|
||||
group_vars/all/15_storage.yml). Reachable NFS server IP/hostname
|
||||
and the matching export base path are mandatory.
|
||||
|
||||
- name: "Install NFS client packages"
|
||||
ansible.builtin.package:
|
||||
name: "{{ svc_storage_nfs_client_packages[ansible_os_family] }}"
|
||||
state: present
|
||||
|
||||
- name: "Ensure probe mount point exists"
|
||||
ansible.builtin.file:
|
||||
path: "{{ svc_storage_nfs_client_probe_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0700"
|
||||
|
||||
- name: "Probe-mount the NFS export"
|
||||
ansible.posix.mount:
|
||||
path: "{{ svc_storage_nfs_client_probe_dir }}"
|
||||
src: "{{ storage.nfs.server }}:{{ storage.nfs.export_base }}"
|
||||
fstype: "{{ 'nfs4' if (storage.nfs.version | int) >= 4 else 'nfs' }}"
|
||||
opts: "vers={{ storage.nfs.version }},rw,hard,timeo=600,_netdev,noauto"
|
||||
state: ephemeral
|
||||
register: svc_storage_nfs_client_probe
|
||||
failed_when:
|
||||
- svc_storage_nfs_client_probe is failed
|
||||
|
||||
- name: "Probe-write a marker to confirm the mount is RW"
|
||||
ansible.builtin.copy:
|
||||
dest: "{{ svc_storage_nfs_client_probe_dir }}/.svc-storage-nfs-client-probe"
|
||||
content: |
|
||||
probed-by={{ inventory_hostname }}
|
||||
probed-at={{ ansible_date_time.iso8601 }}
|
||||
mode: "0644"
|
||||
|
||||
- name: "Unmount probe mount"
|
||||
ansible.posix.mount:
|
||||
path: "{{ svc_storage_nfs_client_probe_dir }}"
|
||||
state: unmounted
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
application_id: "svc-storage-nfs-client"
|
||||
|
||||
svc_storage_nfs_client_packages:
|
||||
Debian:
|
||||
- nfs-common
|
||||
Ubuntu:
|
||||
- nfs-common
|
||||
Archlinux:
|
||||
- nfs-utils
|
||||
RedHat:
|
||||
- nfs-utils
|
||||
Fedora:
|
||||
- nfs-utils
|
||||
Alpine:
|
||||
- nfs-utils
|
||||
|
||||
svc_storage_nfs_client_probe_dir: /run/svc-storage-nfs-client/probe
|
||||
@@ -0,0 +1,34 @@
|
||||
# NFS Server
|
||||
|
||||
## Description
|
||||
|
||||
[NFS](https://en.wikipedia.org/wiki/Network_File_System) (Network File
|
||||
System) is a kernel-level protocol for sharing files over a network.
|
||||
The server exports a directory tree under defined access rules; clients
|
||||
mount it as if it were a local filesystem.
|
||||
|
||||
## Overview
|
||||
|
||||
This role installs the distro-appropriate NFS kernel server, exports
|
||||
`storage.nfs.export_base` (default `/srv/nfs`), and restricts access to
|
||||
inventory members of the `svc-docker-swarm` group. The export options
|
||||
default to `rw,sync,no_subtree_check,root_squash,no_all_squash`. NFS
|
||||
server HA, Kerberos integration, and client-side mounting are out of
|
||||
scope; client mounts are handled by `svc-storage-nfs-client`.
|
||||
|
||||
## Features
|
||||
|
||||
- **Distro-aware install:** Installs `nfs-kernel-server` on Debian /
|
||||
Ubuntu, `nfs-utils` on Arch / RHEL / Fedora / Alpine.
|
||||
- **Inventory-driven ACL:** Allowed-IP list derives from the
|
||||
`svc-docker-swarm` group; unrelated hosts cannot mount.
|
||||
- **Reload-on-change:** Re-running the role overwrites `/etc/exports`
|
||||
with the current allowed-IP set; the handler reloads via
|
||||
`exportfs -ra` only when the file changed.
|
||||
|
||||
## Credits
|
||||
|
||||
Developed and maintained by **Kevin Veen-Birkenbach**.
|
||||
Learn more at [veen.world](https://www.veen.world).
|
||||
Part of the [Infinito.Nexus Project](https://s.infinito.nexus/code).
|
||||
Licensed under the [Infinito.Nexus Community License (Non-Commercial)](https://s.infinito.nexus/license).
|
||||
@@ -0,0 +1,3 @@
|
||||
---
|
||||
- name: "Reload NFS exports"
|
||||
ansible.builtin.command: exportfs -ra
|
||||
@@ -0,0 +1,33 @@
|
||||
---
|
||||
galaxy_info:
|
||||
author: Kevin Veen-Birkenbach
|
||||
description: |
|
||||
Provision an NFSv4 server exposing the project's persistent-volume
|
||||
base path to every host in the `svc-docker-swarm` group. Exports are
|
||||
restricted by IP to the swarm members declared in inventory.
|
||||
license: Infinito.Nexus Community License (Non-Commercial)
|
||||
company: |
|
||||
Kevin Veen-Birkenbach
|
||||
https://www.veen.world
|
||||
galaxy_tags:
|
||||
- nfs
|
||||
- storage
|
||||
- swarm
|
||||
issue_tracker_url: https://s.infinito.nexus/issues
|
||||
min_ansible_version: "2.9"
|
||||
platforms:
|
||||
- name: ArchLinux
|
||||
versions:
|
||||
- all
|
||||
- name: Debian
|
||||
versions:
|
||||
- all
|
||||
- name: EL
|
||||
versions:
|
||||
- all
|
||||
- name: Fedora
|
||||
versions:
|
||||
- all
|
||||
- name: Ubuntu
|
||||
versions:
|
||||
- all
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
nfs-server:
|
||||
name: nfs-server
|
||||
lifecycle: pre-alpha
|
||||
enabled: true
|
||||
shared: true
|
||||
min_storage: "100MB"
|
||||
cpus: "0.2"
|
||||
mem_reservation: "64m"
|
||||
mem_limit: "256m"
|
||||
pids_limit: 256
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
- include_tasks: utils/once/flag.yml
|
||||
|
||||
- name: "Assert storage.nfs.export_base is set"
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- storage.nfs.export_base is defined
|
||||
- storage.nfs.export_base | length > 0
|
||||
fail_msg: |
|
||||
svc-storage-nfs-server requires `storage.nfs.export_base` to be set
|
||||
in group_vars (see group_vars/all/15_storage.yml).
|
||||
|
||||
- name: "Include install step"
|
||||
ansible.builtin.include_tasks: 02_install.yml
|
||||
|
||||
- name: "Include exports step"
|
||||
ansible.builtin.include_tasks: 03_exports.yml
|
||||
|
||||
- name: "Include service step"
|
||||
ansible.builtin.include_tasks: 04_service.yml
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
- name: "Install NFS server packages"
|
||||
ansible.builtin.package:
|
||||
name: "{{ svc_storage_nfs_server_packages[ansible_os_family] }}"
|
||||
state: present
|
||||
@@ -0,0 +1,47 @@
|
||||
---
|
||||
- name: "Ensure export base directory exists"
|
||||
ansible.builtin.file:
|
||||
path: "{{ storage.nfs.export_base }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
- name: "Gather facts on swarm members so their IPs become available"
|
||||
ansible.builtin.setup:
|
||||
gather_subset: network
|
||||
delegate_to: "{{ item }}"
|
||||
delegate_facts: true
|
||||
loop: "{{ groups['svc-docker-swarm'] | default([]) }}"
|
||||
when: hostvars[item].ansible_default_ipv4 is not defined
|
||||
|
||||
- name: "Compute swarm-member IPs allowed to mount"
|
||||
ansible.builtin.set_fact:
|
||||
svc_storage_nfs_server_allowed_ips: >-
|
||||
{{
|
||||
groups['svc-docker-swarm']
|
||||
| default([])
|
||||
| map('extract', hostvars, ['ansible_default_ipv4', 'address'])
|
||||
| select('defined')
|
||||
| list
|
||||
}}
|
||||
|
||||
- name: "Assert at least one allowed IP was resolved"
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- svc_storage_nfs_server_allowed_ips | length > 0
|
||||
fail_msg: |
|
||||
Could not resolve any swarm-member IPs from inventory. Make sure
|
||||
facts have been gathered for every host in `svc-docker-swarm`.
|
||||
|
||||
- name: "Render /etc/exports"
|
||||
ansible.builtin.template:
|
||||
src: exports.j2
|
||||
dest: /etc/exports
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify: "Reload NFS exports"
|
||||
|
||||
- name: "Flush handlers (exports reload before next step)"
|
||||
ansible.builtin.meta: flush_handlers
|
||||
@@ -0,0 +1,6 @@
|
||||
---
|
||||
- name: "Enable and start NFS server"
|
||||
ansible.builtin.systemd:
|
||||
name: "{{ svc_storage_nfs_server_service[ansible_os_family] }}"
|
||||
enabled: true
|
||||
state: started
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
- include_tasks: 01_core.yml
|
||||
when: run_once_svc_storage_nfs_server is not defined
|
||||
vars:
|
||||
application_id: svc-storage-nfs-server
|
||||
@@ -0,0 +1,4 @@
|
||||
# Managed by Ansible role svc-storage-nfs-server.
|
||||
{% for ip in svc_storage_nfs_server_allowed_ips %}
|
||||
{{ storage.nfs.export_base }} {{ ip }}({{ svc_storage_nfs_server_export_options }})
|
||||
{% endfor %}
|
||||
@@ -0,0 +1,28 @@
|
||||
---
|
||||
application_id: "svc-storage-nfs-server"
|
||||
|
||||
svc_storage_nfs_server_packages:
|
||||
Debian:
|
||||
- nfs-kernel-server
|
||||
Ubuntu:
|
||||
- nfs-kernel-server
|
||||
Archlinux:
|
||||
- nfs-utils
|
||||
RedHat:
|
||||
- nfs-utils
|
||||
Fedora:
|
||||
- nfs-utils
|
||||
Alpine:
|
||||
- nfs-utils
|
||||
|
||||
svc_storage_nfs_server_service:
|
||||
Debian: nfs-server
|
||||
Ubuntu: nfs-server
|
||||
Archlinux: nfs-server.service
|
||||
RedHat: nfs-server
|
||||
Fedora: nfs-server
|
||||
Alpine: nfs
|
||||
|
||||
# No `fsid=0`: clients would otherwise have to mount `<server>:/` (pseudo-root).
|
||||
svc_storage_nfs_server_export_options: >-
|
||||
rw,sync,no_subtree_check,root_squash,no_all_squash
|
||||
@@ -51,6 +51,38 @@ if [ -n "$docker_ps_grep_exited" ]; then
|
||||
done
|
||||
fi
|
||||
|
||||
if command -v container >/dev/null 2>&1; then
|
||||
swarm_state="$(container info --format '{{.Swarm.LocalNodeState}}' 2>/dev/null || echo inactive)"
|
||||
is_manager="$(container info --format '{{.Swarm.ControlAvailable}}' 2>/dev/null || echo false)"
|
||||
if [ "$swarm_state" = "active" ] && [ "$is_manager" = "true" ]; then
|
||||
swarm_problems="$(container service ls --format '{{.Name}} {{.Replicas}}' 2>/dev/null \
|
||||
| awk '{
|
||||
split($2, a, "/");
|
||||
if (a[1] != a[2]) {
|
||||
print $1 " " $2;
|
||||
}
|
||||
}')"
|
||||
if [ -n "$swarm_problems" ]; then
|
||||
echo "❌ Swarm services not fully converged:"
|
||||
echo "$swarm_problems"
|
||||
echo
|
||||
while read -r service_name replicas; do
|
||||
echo "------------------------------------------------------------"
|
||||
echo "🔍 Recent tasks for swarm service: $service_name ($replicas)"
|
||||
echo "------------------------------------------------------------"
|
||||
container service ps --no-trunc "$service_name" 2>&1 | head -20 \
|
||||
|| echo "⚠️ Failed to fetch tasks for $service_name"
|
||||
summary="$summary\n - $service_name (swarm replicas $replicas)"
|
||||
done <<EOF
|
||||
$swarm_problems
|
||||
EOF
|
||||
if [ "$exitcode" -lt 1 ]; then
|
||||
exitcode=1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$exitcode" -ne "0" ]; then
|
||||
echo "============================================================"
|
||||
echo "🚨 SUMMARY: Unhealthy / Failed Docker Containers"
|
||||
|
||||
@@ -43,7 +43,7 @@ for volume in $anonymous_volumes; do
|
||||
for container_id in $container_ids; do
|
||||
container_name=$(container inspect --format '{{ .Name }}' "$container_id" | sed 's#^/##')
|
||||
mount_path=$(container inspect --format "{{ range .Mounts }}{{ if eq .Name \"$volume\" }}{{ .Destination }}{{ end }}{{ end }}" "$container_id")
|
||||
|
||||
|
||||
if [ -n "$mount_path" ]; then
|
||||
echo "Volume $volume is used by container $container_name at mount path $mount_path"
|
||||
else
|
||||
@@ -52,4 +52,38 @@ for volume in $anonymous_volumes; do
|
||||
done
|
||||
done
|
||||
|
||||
nfs_volumes=$(container volume ls --filter driver=local --format '{{.Name}}' \
|
||||
| while read -r vol; do
|
||||
opts=$(container volume inspect "$vol" --format '{{.Options.type}}' 2>/dev/null)
|
||||
if [ "$opts" = "nfs" ]; then
|
||||
echo "$vol"
|
||||
fi
|
||||
done)
|
||||
|
||||
if [ -n "$nfs_volumes" ]; then
|
||||
echo
|
||||
echo "============================================================"
|
||||
echo "🔍 NFS-backed volumes — reachability check"
|
||||
echo "============================================================"
|
||||
for vol in $nfs_volumes; do
|
||||
device=$(container volume inspect "$vol" --format '{{.Options.device}}' 2>/dev/null)
|
||||
addr_opt=$(container volume inspect "$vol" --format '{{.Options.o}}' 2>/dev/null)
|
||||
server=$(echo "$addr_opt" | sed -n 's/.*addr=\([^,]*\).*/\1/p')
|
||||
export_path="${device#:}"
|
||||
|
||||
if [ -z "$server" ] || [ -z "$export_path" ]; then
|
||||
echo "❓ Volume $vol: cannot parse server/export from driver_opts"
|
||||
((status++))
|
||||
continue
|
||||
fi
|
||||
|
||||
if showmount -e "$server" >/dev/null 2>&1; then
|
||||
echo "✅ Volume $vol: server $server reachable for export $export_path"
|
||||
else
|
||||
echo "❌ Volume $vol: server $server NOT reachable (export $export_path)"
|
||||
((status++))
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
exit $status
|
||||
@@ -42,8 +42,9 @@
|
||||
listen: reload system daemon
|
||||
|
||||
- name: reexec systemd manager
|
||||
# Just execute this during reset, it has potential dangerous side effects
|
||||
# daemon-reexec terminates PID-1-systemd containers; effect applies on next boot.
|
||||
ansible.builtin.systemd:
|
||||
daemon_reexec: true
|
||||
become: true
|
||||
listen: reexec systemd manager
|
||||
when: not (DOCKER_IN_CONTAINER | bool)
|
||||
@@ -1,3 +1,5 @@
|
||||
---
|
||||
compose_handlers_flush: false # Set to true in the vars/main.yml of the including role to autoflush after compose routine
|
||||
compose_repository_pull: "{{ compose_repository_url is defined and compose_repository_url is string and compose_repository_url | length > 0 }}"
|
||||
_swarm_retries: 60
|
||||
_swarm_delay: 5
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
stack_service="${1:?stack_service required (e.g. mediawiki_mediawiki)}"
|
||||
require_running="${2:-true}"
|
||||
|
||||
cid="$(container ps --filter "label=com.docker.swarm.service.name=${stack_service}" --format '{{.ID}}' | head -n1)"
|
||||
if [ -z "${cid}" ]; then
|
||||
echo "no-task-container-yet" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "${require_running}" = "true" ]; then
|
||||
state="$(container inspect -f '{{.State.Status}}' "${cid}")"
|
||||
if [ "${state}" != "running" ]; then
|
||||
echo "task-state=${state}" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
printf '%s' "${cid}"
|
||||
@@ -78,6 +78,9 @@
|
||||
executable: /bin/bash
|
||||
listen:
|
||||
- compose up
|
||||
when: >-
|
||||
(deployment_mode | default('compose')) != 'swarm'
|
||||
or (service_is_stateful | default(false) | bool)
|
||||
|
||||
- name: Aggressively prune Docker data (keep networks + keep in-use images)
|
||||
become: true
|
||||
@@ -88,3 +91,59 @@
|
||||
listen:
|
||||
- compose up
|
||||
- compose build
|
||||
|
||||
- name: "swarm: pre-deploy build of local images"
|
||||
ansible.builtin.shell: |
|
||||
set -euo pipefail
|
||||
compose build || {
|
||||
echo "Retrying without cache and pulling bases...";
|
||||
compose build --no-cache --pull;
|
||||
}
|
||||
args:
|
||||
chdir: "{{ lookup('container', application_id, 'directories.instance') }}"
|
||||
executable: /bin/bash
|
||||
listen:
|
||||
- compose up
|
||||
- swarm deploy
|
||||
when:
|
||||
- (deployment_mode | default('compose')) == 'swarm'
|
||||
- not (service_is_stateful | default(false) | bool)
|
||||
|
||||
- name: "swarm: validate stack file via docker stack config"
|
||||
ansible.builtin.shell: |
|
||||
set -euo pipefail
|
||||
docker stack config --compose-file compose.yml > /dev/null
|
||||
args:
|
||||
chdir: "{{ lookup('container', application_id, 'directories.instance') }}"
|
||||
executable: /bin/bash
|
||||
register: swarm_validate
|
||||
changed_when: false
|
||||
failed_when: swarm_validate.rc != 0
|
||||
listen:
|
||||
- compose up
|
||||
- swarm deploy
|
||||
when:
|
||||
- MODE_ASSERT | bool
|
||||
- (deployment_mode | default('compose')) == 'swarm'
|
||||
- not (service_is_stateful | default(false) | bool)
|
||||
|
||||
- name: "swarm: docker stack deploy"
|
||||
ansible.builtin.shell: |
|
||||
set -euo pipefail
|
||||
docker stack deploy \
|
||||
--compose-file compose.yml \
|
||||
--resolve-image never \
|
||||
--prune \
|
||||
--with-registry-auth \
|
||||
"{{ application_id | get_entity_name }}"
|
||||
args:
|
||||
chdir: "{{ lookup('container', application_id, 'directories.instance') }}"
|
||||
executable: /bin/bash
|
||||
register: swarm_deploy_result
|
||||
changed_when: true
|
||||
listen:
|
||||
- compose up
|
||||
- swarm deploy
|
||||
when:
|
||||
- (deployment_mode | default('compose')) == 'swarm'
|
||||
- not (service_is_stateful | default(false) | bool)
|
||||
@@ -1,4 +1,10 @@
|
||||
---
|
||||
- name: "Resolve deployment_mode for this host"
|
||||
ansible.builtin.set_fact:
|
||||
deployment_mode: >-
|
||||
{{ 'swarm' if 'svc-docker-swarm' in group_names else 'compose' }}
|
||||
when: deployment_mode is not defined or deployment_mode == 'compose'
|
||||
|
||||
- include_tasks: utils/once/flag.yml
|
||||
|
||||
- include_role:
|
||||
@@ -23,6 +29,11 @@
|
||||
owner: root
|
||||
group: root
|
||||
|
||||
- name: "Ensure compose wrapper system deps (PyYAML)"
|
||||
ansible.builtin.package:
|
||||
name: "{{ sys_svc_compose_pyyaml_package[ansible_os_family] }}"
|
||||
state: present
|
||||
|
||||
- name: Install compose wrapper
|
||||
become: true
|
||||
ansible.builtin.copy:
|
||||
|
||||
@@ -1,5 +1,19 @@
|
||||
---
|
||||
- name: "Ensure docker network for '{{ network_role_id }}' exists"
|
||||
- name: "Ensure docker overlay network for '{{ network_role_id }}' exists (swarm scope)"
|
||||
community.docker.docker_network:
|
||||
name: "{{ network_role_id | get_entity_name }}"
|
||||
state: present
|
||||
driver: overlay
|
||||
attachable: true
|
||||
scope: swarm
|
||||
driver_options:
|
||||
encrypted: "{{ 'true' if swarm.network.encryption else 'false' }}"
|
||||
labels:
|
||||
com.docker.compose.network: "default"
|
||||
com.docker.compose.project: "{{ network_role_id | get_entity_name }}"
|
||||
when: (deployment_mode | default('compose')) == 'swarm'
|
||||
|
||||
- name: "Ensure docker bridge network for '{{ network_role_id }}' exists (compose mode)"
|
||||
community.docker.docker_network:
|
||||
name: "{{ network_role_id | get_entity_name }}"
|
||||
state: present
|
||||
@@ -8,3 +22,4 @@
|
||||
com.docker.compose.project: "{{ network_role_id | get_entity_name }}"
|
||||
ipam_config:
|
||||
- subnet: "{{ lookup('config', network_role_id, 'server.networks.local.subnet') }}"
|
||||
when: (deployment_mode | default('compose')) != 'swarm'
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
- block:
|
||||
- name: "[swarm] Wait until task container for {{ _swarm_stack }}_{{ _swarm_service }} is up and resolve its ID"
|
||||
ansible.builtin.script:
|
||||
cmd: >-
|
||||
swarm_resolve_container.sh
|
||||
{{ _swarm_stack }}_{{ _swarm_service }}
|
||||
{{ (_swarm_require_running | default(true)) | bool | string | lower }}
|
||||
register: _swarm_resolved_cid
|
||||
changed_when: false
|
||||
retries: "{{ _swarm_retries | default(60) }}"
|
||||
delay: "{{ _swarm_delay | default(5) }}"
|
||||
until: _swarm_resolved_cid.rc == 0
|
||||
|
||||
- name: "[swarm] Re-bind fact '{{ _swarm_target_fact }}' to resolved task container ID"
|
||||
ansible.builtin.set_fact:
|
||||
"{{ _swarm_target_fact }}": "{{ _swarm_resolved_cid.stdout }}"
|
||||
when: (deployment_mode | default('compose')) == 'swarm'
|
||||
@@ -20,8 +20,16 @@
|
||||
or
|
||||
env_template.changed | default(false)
|
||||
)
|
||||
- (deployment_mode | default('compose')) == 'compose'
|
||||
notify: compose up
|
||||
|
||||
- name: "Notify swarm deploy (under deployment_mode == swarm)"
|
||||
ansible.builtin.debug:
|
||||
msg: "Triggering swarm stack deploy for {{ application_id }}"
|
||||
changed_when: true
|
||||
when: (deployment_mode | default('compose')) == 'swarm'
|
||||
notify: swarm deploy
|
||||
|
||||
- name: "Flush Docker Compose Up Handlers for {{ lookup('container', application_id, 'directories.instance') }}"
|
||||
meta: flush_handlers
|
||||
when: compose_handlers_flush | bool
|
||||
@@ -41,13 +41,18 @@ networks:
|
||||
{{ _ollama_network_name }}:
|
||||
external: true
|
||||
{% endif %}
|
||||
{# Default Network — the subnet lives at
|
||||
roles/<role>/meta/server.yml.networks.local.subnet, materialised at
|
||||
applications.<role>.server.networks.local.subnet. #}
|
||||
{% set _default_subnet = lookup('config', application_id, 'server.networks.local.subnet', '') %}
|
||||
{% set _is_swarm = (deployment_mode | default('compose')) == 'swarm' %}
|
||||
{% set _net_encrypted = swarm.network.encryption | default(true) if _is_swarm else false %}
|
||||
{% if not application_id.startswith('svc-db-') and not application_id.startswith('svc-ai-') %}
|
||||
default:
|
||||
{% if _default_subnet %}
|
||||
{% if _is_swarm %}
|
||||
name: {{ application_id | get_entity_name }}
|
||||
driver: overlay
|
||||
attachable: true
|
||||
driver_opts:
|
||||
encrypted: "{{ 'true' if _net_encrypted else 'false' }}"
|
||||
{% elif _default_subnet %}
|
||||
name: {{ application_id | get_entity_name }}
|
||||
driver: bridge
|
||||
ipam:
|
||||
|
||||
@@ -1,2 +1,8 @@
|
||||
---
|
||||
COMPOSE_DOWN_ALL_PACKAGE: "docodol"
|
||||
|
||||
sys_svc_compose_pyyaml_package:
|
||||
Debian: python3-yaml
|
||||
RedHat: python3-pyyaml
|
||||
Suse: python3-PyYAML
|
||||
Archlinux: python-yaml
|
||||
@@ -0,0 +1,4 @@
|
||||
---
|
||||
service_replicas: 1
|
||||
service_update_parallelism: 1
|
||||
service_update_delay: "10s"
|
||||
@@ -10,7 +10,10 @@
|
||||
{% endif %}
|
||||
logging:
|
||||
driver: journald
|
||||
{% if (deployment_mode | default('compose')) != 'swarm' or (service_is_stateful | default(false) | bool) %}
|
||||
{% filter indent(4) %}
|
||||
{% include 'roles/sys-svc-container/templates/resource.yml.j2' %}
|
||||
{% endfilter %}
|
||||
{% endif %}
|
||||
{% include 'roles/sys-svc-container/templates/deploy.yml.j2' %}
|
||||
{{ "\n" }}
|
||||
@@ -1,5 +1,11 @@
|
||||
{# integrate it into service sections to be build by Dockerfile #}
|
||||
{# integrate it into service sections to be build by Dockerfile.
|
||||
`pull_policy` is a docker-compose-only field — `docker stack
|
||||
deploy` rejects it as "Additional property pull_policy is not
|
||||
allowed". Skip it under swarm mode (swarm always uses the locally
|
||||
resolved image via --resolve-image always at deploy time). #}
|
||||
{% if (deployment_mode | default('compose')) != 'swarm' %}
|
||||
pull_policy: never
|
||||
{% endif %}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
{% if (deployment_mode | default('compose')) == 'swarm' and not (service_is_stateful | default(false)) %}
|
||||
{{ "\n" }}
|
||||
deploy:
|
||||
replicas: {{ service_replicas | default(1) }}
|
||||
update_config:
|
||||
parallelism: {{ service_update_parallelism | default(1) }}
|
||||
delay: {{ service_update_delay | default('10s') }}
|
||||
restart_policy:
|
||||
condition: any
|
||||
resources:
|
||||
limits:
|
||||
cpus: "{{ lookup('applications') | resource_filter(application_id, 'cpus', service_name | default(''), RESOURCE_CPUS) | string }}"
|
||||
memory: {{ lookup('applications') | resource_filter(application_id, 'mem_limit', service_name | default(''), RESOURCE_MEM_LIMIT) }}
|
||||
pids: {{ lookup('applications') | resource_filter(application_id, 'pids_limit', service_name | default(''), RESOURCE_PIDS_LIMIT) }}
|
||||
reservations:
|
||||
memory: {{ lookup('applications') | resource_filter(application_id, 'mem_reservation', service_name | default(''), RESOURCE_MEM_RESERVATION) }}
|
||||
{% set _constraints = (service_placement_constraints | default([])) | list %}
|
||||
{% if service_needs_node_pin | default(false) %}
|
||||
{% set _constraints = _constraints + ['node.role == manager'] %}
|
||||
{% endif %}
|
||||
{% if _constraints %}
|
||||
placement:
|
||||
constraints:
|
||||
{% for c in _constraints %}
|
||||
- "{{ c }}"
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
@@ -83,3 +83,6 @@ SYS_SVC_CONTAINER_PACKAGES_BY_DISTRIBUTION:
|
||||
- containerd.io
|
||||
- docker-compose-plugin
|
||||
- docker-buildx-plugin
|
||||
Alpine:
|
||||
- docker
|
||||
- docker-cli-compose
|
||||
@@ -21,6 +21,27 @@ This role deploys MediaWiki using Docker, automating the setup of your wiki inst
|
||||
- [MediaWiki Official Website](https://www.mediawiki.org/)
|
||||
- [MediaWiki Documentation](https://www.mediawiki.org/wiki/Manual:Configuration_settings)
|
||||
|
||||
## Swarm + NFS pilot
|
||||
|
||||
Volume layout under `deployment_mode: swarm` with `storage.backend: nfs`:
|
||||
|
||||
- **`images/`** opts into NFS (`nfs: true` in the `compose_volumes`
|
||||
call from `templates/compose.yml.j2`). The volume is shared across
|
||||
all swarm nodes so the MediaWiki application service can be
|
||||
rescheduled freely.
|
||||
- **`extensions/`** stays local. Extensions are rebuilt from the
|
||||
role-managed git source on every install — no shared state needs
|
||||
to survive a node move.
|
||||
- **MariaDB data** stays local on the manager node via
|
||||
[svc-db-mariadb](../svc-db-mariadb/) pinning. NFS for the DB data
|
||||
directory is intentionally out of scope for v1 (locking / `fsync`
|
||||
semantics). See 023's Future Extensions.
|
||||
|
||||
CI gate: [.github/workflows/test-deploy-swarm-nfs.yml](../../.github/workflows/test-deploy-swarm-nfs.yml)
|
||||
provisions a 3-node DinD swarm, deploys this role as a stack, drains
|
||||
the worker running the application service, and asserts that wiki
|
||||
content survives the reschedule.
|
||||
|
||||
## Credits
|
||||
|
||||
Developed and maintained by **Kevin Veen-Birkenbach**.
|
||||
|
||||
@@ -35,3 +35,11 @@
|
||||
no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
|
||||
register: mediawiki_config_render
|
||||
notify: compose up
|
||||
|
||||
- name: "MEDIAWIKI | (swarm) Replicate host-bound config to other swarm members"
|
||||
ansible.builtin.include_tasks: 01_configuration_replicate.yml
|
||||
loop: "{{ groups['svc-docker-swarm'] | default([]) | difference([inventory_hostname]) }}"
|
||||
loop_control:
|
||||
loop_var: _swarm_peer
|
||||
when:
|
||||
- (deployment_mode | default('compose')) == 'swarm'
|
||||
@@ -0,0 +1,37 @@
|
||||
---
|
||||
- name: "MEDIAWIKI | (swarm) Ensure host directories exist on {{ _swarm_peer }}"
|
||||
ansible.builtin.file:
|
||||
path: "{{ dir.path }}"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
loop:
|
||||
- path: "{{ MEDIAWIKI_LOCAL_MOUNT_DIR }}"
|
||||
- path: "{{ MEDIAWIKI_CONFIG_DIR }}"
|
||||
loop_control:
|
||||
loop_var: dir
|
||||
label: "{{ dir.path }}"
|
||||
delegate_to: "{{ _swarm_peer }}"
|
||||
|
||||
- name: "MEDIAWIKI | (swarm) Render host-side PHP config files on {{ _swarm_peer }}"
|
||||
ansible.builtin.template:
|
||||
src: "{{ cfg.src }}"
|
||||
dest: "{{ cfg.dest }}"
|
||||
owner: root
|
||||
group: "{{ MEDIAWIKI_USER_GID }}"
|
||||
mode: "0640"
|
||||
loop:
|
||||
- src: "oidc.php.j2"
|
||||
dest: "{{ MEDIAWIKI_LOCAL_MOUNT_DIR }}/oidc.php"
|
||||
when: "{{ MEDIAWIKI_OIDC_ENABLED | bool }}"
|
||||
- src: "debug.php.j2"
|
||||
dest: "{{ MEDIAWIKI_LOCAL_MOUNT_DIR }}/debug.php"
|
||||
when: true
|
||||
- src: "LocalSettings.php.j2"
|
||||
dest: "{{ MEDIAWIKI_LOCALSETTINGS_HOST_PATH }}"
|
||||
when: true
|
||||
loop_control:
|
||||
loop_var: cfg
|
||||
label: "{{ cfg.dest | basename }}"
|
||||
when: cfg.when | bool
|
||||
no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
|
||||
delegate_to: "{{ _swarm_peer }}"
|
||||
@@ -1,4 +1,18 @@
|
||||
---
|
||||
- name: "Pre-create NFS subdir for MediaWiki images volume on the NFS server"
|
||||
ansible.builtin.file:
|
||||
path: "{{ storage.nfs.export_base }}/{{ MEDIAWIKI_VOLUME_IMAGES }}"
|
||||
state: directory
|
||||
owner: "33"
|
||||
group: "33"
|
||||
mode: "0755"
|
||||
delegate_to: "{{ (groups['svc-storage-nfs-server'] | default([]))[0] | default(inventory_hostname) }}"
|
||||
run_once: true
|
||||
when:
|
||||
- (deployment_mode | default('compose')) == 'swarm'
|
||||
- (storage.backend | default('local')) == 'nfs'
|
||||
- (groups['svc-storage-nfs-server'] | default([])) | length > 0
|
||||
|
||||
- name: "load docker, db and proxy for '{{ application_id }}'"
|
||||
include_role:
|
||||
name: sys-stk-full
|
||||
@@ -11,6 +25,13 @@
|
||||
- name: "flush handlers for '{{ application_id }}' after installation finished"
|
||||
meta: flush_handlers
|
||||
|
||||
- name: "[swarm] Resolve MediaWiki task container ID for this role"
|
||||
include_tasks: "{{ [playbook_dir, 'roles/sys-svc-compose/tasks/utils/swarm/resolve_container.yml'] | path_join }}"
|
||||
vars:
|
||||
_swarm_stack: "{{ application_id | get_entity_name }}"
|
||||
_swarm_service: "{{ application_id | get_entity_name }}"
|
||||
_swarm_target_fact: MEDIAWIKI_CONTAINER
|
||||
|
||||
- name: "MEDIAWIKI | Wait until container exists"
|
||||
ansible.builtin.command: "container inspect {{ MEDIAWIKI_CONTAINER }}"
|
||||
register: _mw_exists
|
||||
@@ -19,9 +40,6 @@
|
||||
until: _mw_exists.rc == 0
|
||||
changed_when: false
|
||||
|
||||
# The database lives in a separate compose project (svc-db-mariadb) when shared,
|
||||
# so compose-level `depends_on: condition: service_healthy` cannot reach it.
|
||||
# Gate the install step on the database container's own healthcheck instead.
|
||||
- name: "MEDIAWIKI | Wait until database container is healthy"
|
||||
ansible.builtin.command: "container container inspect --format={% raw %}{{.State.Health.Status}}{% endraw %} {{ MEDIAWIKI_DB_CONTAINER }}"
|
||||
register: _mw_db_health
|
||||
|
||||
@@ -20,6 +20,11 @@
|
||||
|
||||
{{ lookup('applications') | compose_volumes(
|
||||
application_id,
|
||||
extra_volumes={'extensions': {'name': MEDIAWIKI_VOLUME_EXTENSIONS}, 'images': {'name': MEDIAWIKI_VOLUME_IMAGES}}
|
||||
extra_volumes={
|
||||
'extensions': {'name': MEDIAWIKI_VOLUME_EXTENSIONS},
|
||||
'images': {'name': MEDIAWIKI_VOLUME_IMAGES, 'nfs': True},
|
||||
},
|
||||
deployment_mode=deployment_mode | default('compose'),
|
||||
storage=storage | default({}),
|
||||
) }}
|
||||
{% include 'roles/sys-svc-compose/templates/networks.yml.j2' %}
|
||||
@@ -4,6 +4,8 @@ application_id: "web-app-mediawiki"
|
||||
container_port: 80
|
||||
entity_name: "{{ application_id | get_entity_name }}"
|
||||
|
||||
service_is_stateful: false # explicit: blocks role-var leak from earlier stateful role
|
||||
|
||||
# Mediawiki
|
||||
MEDIAWIKI_SITENAME: "{{ lookup('config', application_id, 'services.mediawiki.sitename') }}"
|
||||
MEDIAWIKI_URL: "{{ lookup('tls', application_id, 'url.base') }}"
|
||||
|
||||
@@ -67,8 +67,19 @@ build_and_install_debian_like() {
|
||||
local build_root="/tmp/infinito-nexus-debian-build"
|
||||
local deb_path=""
|
||||
|
||||
echo "[debian] Installing build toolchain..."
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
if [[ -n "${PACKAGE_INSTALL_FROM:-}" ]]; then
|
||||
echo "[debian] Installing pre-built ${PACKAGE_INSTALL_FROM}..."
|
||||
apt-get update
|
||||
bootstrap_docker_repo
|
||||
dpkg -i "${PACKAGE_INSTALL_FROM}" || true
|
||||
apt-get install -f -y
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "[debian] Installing build toolchain..."
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
@@ -98,6 +109,11 @@ build_and_install_debian_like() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${PACKAGE_BUILD_ONLY:-0}" == "1" ]]; then
|
||||
echo "${deb_path}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "[debian] Installing ${deb_path}..."
|
||||
bootstrap_docker_repo
|
||||
apt-get install -y "${deb_path}"
|
||||
|
||||
@@ -10,6 +10,7 @@ set -euo pipefail
|
||||
: "${ACT_PULL:=false}"
|
||||
: "${ACT_RM:=true}"
|
||||
: "${ACT_PLATFORM_IMAGE:=catthehacker/ubuntu:act-latest}"
|
||||
: "${ACT_BIND:=false}"
|
||||
|
||||
echo "=== act: workflow=${ACT_WORKFLOW} event=${ACT_EVENT} job=${ACT_JOB:-<all>} matrix=${ACT_MATRIX:-<none>} ==="
|
||||
|
||||
@@ -37,5 +38,10 @@ cmd+=(--pull="${ACT_PULL}")
|
||||
if [[ "${ACT_RM}" == "true" ]]; then
|
||||
cmd+=(--rm)
|
||||
fi
|
||||
cmd+=(--cache-server-addr 127.0.0.1)
|
||||
cmd+=(--artifact-server-addr 127.0.0.1)
|
||||
if [[ "${ACT_BIND}" == "true" ]]; then
|
||||
cmd+=(--bind)
|
||||
fi
|
||||
|
||||
"${cmd[@]}"
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# MTU 1400: TLS ServerHello fragmentation drops on host PMTU < 1500.
|
||||
docker network create \
|
||||
--driver bridge \
|
||||
--subnet 192.168.244.0/24 \
|
||||
--opt com.docker.network.driver.mtu=1400 \
|
||||
swarm-lab
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Kernel NFS server cannot export overlay2 paths; bind-mount from a real fs.
|
||||
mkdir -p "${RUNNER_TEMP}/nfs-export"
|
||||
docker run -d --name "${NFS_SERVER}" \
|
||||
--network swarm-lab \
|
||||
--hostname "${NFS_SERVER}" \
|
||||
--privileged \
|
||||
--tmpfs /run \
|
||||
--tmpfs /run/lock \
|
||||
--tmpfs /tmp:exec \
|
||||
-v /sys/fs/cgroup:/sys/fs/cgroup:rw \
|
||||
-v /lib/modules:/lib/modules:ro \
|
||||
-v "${RUNNER_TEMP}/nfs-export:/srv/nfs" \
|
||||
jrei/systemd-ubuntu:24.04
|
||||
|
||||
for i in $(seq 1 60); do
|
||||
if docker exec "${NFS_SERVER}" systemctl is-system-running 2>/dev/null |
|
||||
grep -qE 'running|degraded|starting|maintenance'; then
|
||||
echo "systemd ready on ${NFS_SERVER} after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "FAILURE: ${NFS_SERVER} systemd did not become responsive"
|
||||
exit 1
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# /tmp:exec: Docker's default tmpfs is noexec; dpkg-buildpackage execve()s debian/rules.
|
||||
for node in "${MGR}" "${WRK1}" "${WRK2}"; do
|
||||
docker run -d --name "${node}" \
|
||||
--network swarm-lab \
|
||||
--hostname "${node}" \
|
||||
--privileged \
|
||||
--tmpfs /run \
|
||||
--tmpfs /run/lock \
|
||||
--tmpfs /tmp:exec \
|
||||
-v /sys/fs/cgroup:/sys/fs/cgroup:rw \
|
||||
-v /lib/modules:/lib/modules:ro \
|
||||
jrei/systemd-ubuntu:24.04
|
||||
done
|
||||
|
||||
for node in "${MGR}" "${WRK1}" "${WRK2}"; do
|
||||
for i in $(seq 1 60); do
|
||||
if docker exec "${node}" systemctl is-system-running 2>/dev/null |
|
||||
grep -qE 'running|degraded|starting|maintenance'; then
|
||||
echo "systemd ready on ${node} after ${i}s"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
done
|
||||
|
||||
MGR_HOSTS_EXTRA="$(
|
||||
cat <<'EOF'
|
||||
127.0.0.1 infinito.localhost
|
||||
127.0.0.1 admin.infinito.localhost
|
||||
127.0.0.1 auth.infinito.localhost
|
||||
127.0.0.1 m.wiki.infinito.localhost
|
||||
127.0.0.1 wiki.infinito.localhost
|
||||
127.0.0.1 mail.infinito.localhost
|
||||
EOF
|
||||
)"
|
||||
docker exec -i "${MGR}" sh -c 'cat >> /etc/hosts' <<<"${MGR_HOSTS_EXTRA}"
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
MGR_IP=$(docker inspect "${MGR}" \
|
||||
--format '{{(index .NetworkSettings.Networks "swarm-lab").IPAddress}}')
|
||||
NFS_IP=$(docker inspect "${NFS_SERVER}" \
|
||||
--format '{{(index .NetworkSettings.Networks "swarm-lab").IPAddress}}')
|
||||
|
||||
echo "MGR_IP=${MGR_IP}" >>"$GITHUB_ENV"
|
||||
echo "NFS_IP=${NFS_IP}" >>"$GITHUB_ENV"
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
HOST_SRC="$(pwd)"
|
||||
|
||||
echo "==> Building .deb once on the act-runner"
|
||||
DEB_PATH=$(PACKAGE_BUILD_ONLY=1 bash "${HOST_SRC}/scripts/install/package.sh" |
|
||||
tee /dev/stderr | tail -n1)
|
||||
[ -f "${DEB_PATH}" ] || {
|
||||
echo "FAILURE: builder did not produce a .deb (${DEB_PATH})"
|
||||
exit 1
|
||||
}
|
||||
echo "==> built ${DEB_PATH}"
|
||||
|
||||
DEB_BASENAME="$(basename "${DEB_PATH}")"
|
||||
|
||||
provision_node() {
|
||||
local node="$1"
|
||||
echo "==> [${node}] copying repo + .deb + installing"
|
||||
docker exec "${node}" mkdir -p /opt/infinito-nexus
|
||||
tar -C "${HOST_SRC}" -cf - . | docker exec -i "${node}" tar -C /opt/infinito-nexus -xpf -
|
||||
# docker cp to a tmpfs mount lands in the overlay shadow path (moby/22281).
|
||||
docker exec -i "${node}" sh -c "cat > /tmp/${DEB_BASENAME}" <"${DEB_PATH}"
|
||||
docker exec \
|
||||
-e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 \
|
||||
-e PACKAGE_INSTALL_FROM="/tmp/${DEB_BASENAME}" \
|
||||
"${node}" bash -c \
|
||||
'cd /opt/infinito-nexus && bash scripts/install/package.sh'
|
||||
}
|
||||
|
||||
declare -A PIDS
|
||||
for node in "${MGR}" "${WRK1}" "${WRK2}" "${NFS_SERVER}"; do
|
||||
provision_node "${node}" >"/tmp/bootstrap-${node}.log" 2>&1 &
|
||||
PIDS["${node}"]=$!
|
||||
done
|
||||
|
||||
rc=0
|
||||
for node in "${!PIDS[@]}"; do
|
||||
if ! wait "${PIDS[${node}]}"; then
|
||||
echo "FAILURE: provision_node ${node} exited non-zero"
|
||||
rc=1
|
||||
fi
|
||||
done
|
||||
|
||||
for node in "${MGR}" "${WRK1}" "${WRK2}" "${NFS_SERVER}"; do
|
||||
echo "---- bootstrap log for ${node} ----"
|
||||
cat "/tmp/bootstrap-${node}.log"
|
||||
done
|
||||
|
||||
exit "${rc}"
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
pip install --no-cache-dir --break-system-packages -e .
|
||||
ansible-galaxy collection install community.docker ansible.posix
|
||||
|
||||
PYTHON=python3 bash scripts/setup.sh
|
||||
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# nfs-server group is wired by 08_extend_inventory.py; do not add via --include.
|
||||
mkdir -p /tmp/inv
|
||||
python3 -m cli.administration.inventory.provision /tmp/inv \
|
||||
--host "${MGR}" \
|
||||
--include svc-docker-swarm svc-docker-swarm-manager \
|
||||
svc-storage-nfs-client \
|
||||
svc-db-mariadb web-app-mediawiki \
|
||||
--workers 2
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
|
||||
from utils.cache.yaml import dump_yaml, load_yaml_any
|
||||
|
||||
INV_PATH = Path("/tmp/inv/devices.yml") # noqa: S108
|
||||
|
||||
DOCKER_VARS = {
|
||||
"ansible_connection": "docker",
|
||||
"ansible_python_interpreter": "/usr/bin/python3",
|
||||
"ansible_user": "root",
|
||||
}
|
||||
|
||||
GROUP_HOSTS = [
|
||||
("svc-docker-swarm", "swarm-mgr-01"),
|
||||
("svc-docker-swarm", "swarm-wrk-01"),
|
||||
("svc-docker-swarm", "swarm-wrk-02"),
|
||||
("svc-docker-swarm-manager", "swarm-mgr-01"),
|
||||
("svc-db-mariadb", "swarm-mgr-01"),
|
||||
("web-app-mediawiki", "swarm-mgr-01"),
|
||||
("svc-storage-nfs-client", "swarm-mgr-01"),
|
||||
("svc-storage-nfs-client", "swarm-wrk-01"),
|
||||
("svc-storage-nfs-client", "swarm-wrk-02"),
|
||||
("svc-storage-nfs-server", "nfs-server"),
|
||||
]
|
||||
|
||||
inv = load_yaml_any(str(INV_PATH), default_if_missing={})
|
||||
inv.setdefault("all", {}).setdefault("children", {})
|
||||
children = inv["all"]["children"]
|
||||
|
||||
for group, host in GROUP_HOSTS:
|
||||
children.setdefault(group, {}).setdefault("hosts", {})
|
||||
children[group]["hosts"][host] = DOCKER_VARS
|
||||
|
||||
dump_yaml(str(INV_PATH), inv)
|
||||
print(INV_PATH.read_text())
|
||||
PY
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ssh-keygen -t ed25519 -N "" -f /tmp/swarm-nfs-admin.key -C "swarm-nfs-pilot-test" -q
|
||||
ADMIN_PUBKEY=$(cat /tmp/swarm-nfs-admin.key.pub)
|
||||
|
||||
cat >/tmp/swarm-nfs-extras.yml <<EOF
|
||||
TLS_MODE: "self_signed"
|
||||
storage:
|
||||
backend: nfs
|
||||
nfs:
|
||||
server: ${NFS_IP}
|
||||
export_base: /srv/nfs
|
||||
# NFSv4 pseudo-root does not negotiate against the act kernel.
|
||||
version: 3
|
||||
svc_storage_nfs_server_export_options: >-
|
||||
rw,sync,no_subtree_check,no_root_squash,no_all_squash
|
||||
swarm:
|
||||
manager:
|
||||
advertise_addr: ${MGR_IP}
|
||||
network:
|
||||
encryption: true
|
||||
nfs_server_ip: ${NFS_IP}
|
||||
users:
|
||||
administrator:
|
||||
authorized_keys:
|
||||
- "${ADMIN_PUBKEY}"
|
||||
EOF
|
||||
cat /tmp/swarm-nfs-extras.yml
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
export ANSIBLE_HOST_KEY_CHECKING=False
|
||||
ansible-inventory -i /tmp/inv/devices.yml --graph
|
||||
echo '---'
|
||||
ansible -i /tmp/inv/devices.yml --vault-password-file /tmp/inv/.password \
|
||||
swarm-mgr-01 -m debug -a "msg={{ group_names }}" || true
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
export ANSIBLE_HOST_KEY_CHECKING=False
|
||||
|
||||
python3 -m cli.administration.deploy.dedicated \
|
||||
/tmp/inv/devices.yml \
|
||||
--id svc-docker-swarm svc-storage-nfs-server svc-storage-nfs-client \
|
||||
svc-db-mariadb web-app-mediawiki \
|
||||
-p /tmp/inv/.password \
|
||||
--skip-build \
|
||||
--skip-cleanup \
|
||||
--skip-backup \
|
||||
-e @/tmp/swarm-nfs-extras.yml
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
converged=false
|
||||
for i in $(seq 1 90); do
|
||||
mediawiki_replicas=$(docker exec "${MGR}" docker service ls \
|
||||
--filter "name=${STACK_NAME}_mediawiki" \
|
||||
--format '{{.Replicas}}')
|
||||
mediawiki_state=$(docker exec "${MGR}" sh -c "
|
||||
docker service ps --no-trunc \
|
||||
--format '{{.CurrentState}}' \
|
||||
${STACK_NAME}_mediawiki \
|
||||
| head -1
|
||||
")
|
||||
mariadb_state=$(docker exec "${MGR}" docker inspect \
|
||||
--format '{{.State.Status}}' mariadb 2>/dev/null || echo "missing")
|
||||
echo "[${i}] mw: ${mediawiki_replicas} | mw_state: ${mediawiki_state} | mariadb: ${mariadb_state}"
|
||||
if [ "${mediawiki_replicas}" = "1/1" ] &&
|
||||
echo "${mediawiki_state}" | grep -q '^Running' &&
|
||||
[ "${mariadb_state}" = "running" ]; then
|
||||
echo "Stack converged"
|
||||
converged=true
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [ "${converged}" != "true" ]; then
|
||||
echo "FAILURE: stack did not converge within timeout"
|
||||
docker exec "${MGR}" docker service ps --no-trunc "${STACK_NAME}_mediawiki" || true
|
||||
docker exec "${MGR}" docker inspect --format '{{json .State}}' mariadb || true
|
||||
exit 1
|
||||
fi
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
NODE=$(docker exec "${MGR}" sh -c "
|
||||
docker service ps --no-trunc \
|
||||
--format '{{.Node}} {{.CurrentState}}' \
|
||||
${STACK_NAME}_mediawiki \
|
||||
| awk '\$2 == \"Running\" { print \$1; exit }'
|
||||
")
|
||||
if [ -z "${NODE}" ]; then
|
||||
echo "FAILURE: no running MediaWiki task on any node"
|
||||
docker exec "${MGR}" docker service ps "${STACK_NAME}_mediawiki"
|
||||
exit 1
|
||||
fi
|
||||
echo "MediaWiki initially scheduled on: ${NODE}"
|
||||
echo "INITIAL_NODE=${NODE}" >>"$GITHUB_ENV"
|
||||
|
||||
docker exec "${NFS_SERVER}" sh -c \
|
||||
'echo "pre-drain marker" > /srv/nfs/mediawiki_images/.pre-drain'
|
||||
|
||||
# Swarm routing-mesh ingress is broken in DinD-on-DinD (no iptables/VXLAN);
|
||||
# probe MediaWiki directly inside its task container.
|
||||
MW_CTR=$(docker exec "${NODE}" docker ps \
|
||||
--filter "name=${STACK_NAME}_mediawiki" \
|
||||
--format '{{.ID}}' | head -1)
|
||||
if [ -z "${MW_CTR}" ]; then
|
||||
echo "FAILURE: cannot locate MediaWiki container on ${NODE}"
|
||||
exit 1
|
||||
fi
|
||||
for i in $(seq 1 30); do
|
||||
if docker exec "${NODE}" docker exec "${MW_CTR}" \
|
||||
curl -fsS http://localhost:80/ >/dev/null 2>&1; then
|
||||
echo "MediaWiki HTTP reachable inside container after ${i}s"
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
[ -n "${INITIAL_NODE}" ] || {
|
||||
echo "INITIAL_NODE not set"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# No registry + --resolve-image never: workers need the image pushed manually.
|
||||
echo "==> Distributing mediawiki_custom image to swarm workers"
|
||||
IMAGE_TAG=$(docker exec "${MGR}" docker images --format '{{.Repository}}:{{.Tag}}' |
|
||||
grep '^mediawiki_custom:' | head -1)
|
||||
[ -n "${IMAGE_TAG}" ] || {
|
||||
echo "FAILURE: mediawiki_custom image not present on manager"
|
||||
exit 1
|
||||
}
|
||||
echo " image: ${IMAGE_TAG}"
|
||||
for worker in "${WRK1}" "${WRK2}"; do
|
||||
echo " -> ${worker}"
|
||||
docker exec "${MGR}" docker save "${IMAGE_TAG}" |
|
||||
docker exec -i "${worker}" docker load
|
||||
done
|
||||
|
||||
docker exec "${MGR}" docker node update --availability drain "${INITIAL_NODE}"
|
||||
|
||||
NEW_NODE=""
|
||||
for i in $(seq 1 "${RESCHED_TIMEOUT}"); do
|
||||
NEW_NODE=$(docker exec "${MGR}" docker service ps \
|
||||
--filter desired-state=running \
|
||||
--format '{{.Node}}' "${STACK_NAME}_mediawiki" | head -1)
|
||||
if [ -n "${NEW_NODE}" ] && [ "${NEW_NODE}" != "${INITIAL_NODE}" ]; then
|
||||
echo "Rescheduled from ${INITIAL_NODE} to ${NEW_NODE} after ${i}s"
|
||||
echo "NEW_NODE=${NEW_NODE}" >>"$GITHUB_ENV"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "FAILURE: MediaWiki did not migrate from ${INITIAL_NODE}"
|
||||
docker exec "${MGR}" docker service ps "${STACK_NAME}_mediawiki"
|
||||
exit 1
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
REPL=""
|
||||
for i in $(seq 1 60); do
|
||||
REPL=$(docker exec "${MGR}" docker service ls \
|
||||
--filter "name=${STACK_NAME}_mediawiki" --format '{{.Replicas}}')
|
||||
echo "[${i}] mediawiki replicas: ${REPL}"
|
||||
if [ "${REPL}" = "1/1" ]; then
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
[ "${REPL}" = "1/1" ] || {
|
||||
echo "Replicas != 1/1: ${REPL}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
docker exec "${NEW_NODE}" sh -c "
|
||||
mkdir -p /mnt/nfs-check
|
||||
mount -t nfs -o vers=3,nolock ${NFS_IP}:/srv/nfs /mnt/nfs-check
|
||||
grep -q 'pre-drain marker' /mnt/nfs-check/mediawiki_images/.pre-drain
|
||||
umount /mnt/nfs-check
|
||||
" || {
|
||||
echo 'FAILURE: marker missing on NFS'
|
||||
exit 1
|
||||
}
|
||||
|
||||
MW_CTR=$(docker exec "${NEW_NODE}" docker ps \
|
||||
--filter "name=${STACK_NAME}_mediawiki" \
|
||||
--format '{{.ID}}' | head -1)
|
||||
if [ -z "${MW_CTR}" ]; then
|
||||
echo "FAILURE: cannot locate MediaWiki container on ${NEW_NODE}"
|
||||
exit 1
|
||||
fi
|
||||
for i in $(seq 1 30); do
|
||||
if docker exec "${NEW_NODE}" docker exec "${MW_CTR}" \
|
||||
curl -fsS http://localhost:80/ >/dev/null 2>&1; then
|
||||
echo "MediaWiki reachable after reschedule"
|
||||
exit 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "FAILURE: MediaWiki not reachable after reschedule"
|
||||
exit 1
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
set +e
|
||||
|
||||
sep() {
|
||||
echo "=========================================="
|
||||
echo "=== $1"
|
||||
echo "=========================================="
|
||||
}
|
||||
|
||||
for node in "${MGR}" "${WRK1}" "${WRK2}"; do
|
||||
sep "${node}: /opt/compose tree"
|
||||
docker exec "${node}" find /opt/compose -maxdepth 3 -type f -name '*.yml' 2>/dev/null
|
||||
for f in $(docker exec "${node}" sh -c \
|
||||
'find /opt/compose -maxdepth 3 -type f -name "compose*.yml" 2>/dev/null'); do
|
||||
sep "${node}:${f}"
|
||||
docker exec "${node}" cat -n "${f}" 2>/dev/null
|
||||
done
|
||||
done
|
||||
|
||||
sep "swarm nodes"
|
||||
docker exec "${MGR}" docker node ls
|
||||
|
||||
sep "all swarm services + replica state"
|
||||
docker exec "${MGR}" docker service ls
|
||||
|
||||
sep "per-service docker service ps (--no-trunc) for every existing service"
|
||||
for svc in $(docker exec "${MGR}" docker service ls --format '{{.Name}}'); do
|
||||
echo "--- ${svc} ---"
|
||||
docker exec "${MGR}" docker service ps --no-trunc "${svc}"
|
||||
echo "--- ${svc} logs (tail 50) ---"
|
||||
docker exec "${MGR}" docker service logs --tail 50 "${svc}" 2>&1 | head -100
|
||||
done
|
||||
|
||||
sep "docker images per node (filter custom + mariadb + mediawiki)"
|
||||
for node in "${MGR}" "${WRK1}" "${WRK2}"; do
|
||||
echo "--- ${node} ---"
|
||||
docker exec "${node}" docker images | grep -E 'mariadb|mediawiki|custom' || echo "(none)"
|
||||
done
|
||||
|
||||
sep "rendered env files on manager (value lengths only)"
|
||||
docker exec "${MGR}" sh -c 'for f in /opt/compose/*/\.env/env /opt/compose/*/.env/env; do
|
||||
[ -f "$f" ] || continue
|
||||
echo "--- $f ---"
|
||||
awk -F= '"'"'{ if (NF>=2) { v=substr($0, index($0, "=")+1); printf "%s=<%d-char value>\n", $1, length(v) } else { print $0 } }'"'"' "$f"
|
||||
done'
|
||||
|
||||
sep "live mariadb task container env (MARIADB* only, value prefix redacted)"
|
||||
MARIADB_CID=$(docker exec "${MGR}" sh -c \
|
||||
'docker ps --filter label=com.docker.swarm.service.name=mariadb_mariadb --format "{{.ID}}" | head -n1')
|
||||
if [ -n "${MARIADB_CID}" ]; then
|
||||
docker exec "${MGR}" docker exec "${MARIADB_CID}" sh -c \
|
||||
'env | grep -E "^MARIADB|^MYSQL" | sed "s/=\(.\{1,3\}\).*/=\1...(redacted)/"' ||
|
||||
echo "(failed to exec into ${MARIADB_CID})"
|
||||
else
|
||||
echo "(no live mariadb task container found)"
|
||||
fi
|
||||
|
||||
sep "nfs-server logs"
|
||||
docker logs --tail 200 "${NFS_SERVER}"
|
||||
|
||||
for node in "${MGR}" "${WRK1}" "${WRK2}"; do
|
||||
echo "=== ${node} volumes ==="
|
||||
docker exec "${node}" docker volume ls
|
||||
echo "=== ${node} mount points (nfs filter) ==="
|
||||
docker exec "${node}" mount | grep -i nfs
|
||||
done
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
set +e
|
||||
|
||||
docker rm -f "${MGR}" "${WRK1}" "${WRK2}" "${NFS_SERVER}"
|
||||
docker network rm swarm-lab
|
||||
exit 0
|
||||
@@ -9,6 +9,10 @@
|
||||
ansible.builtin.set_fact:
|
||||
DOCKER_IN_CONTAINER: "{{ (systemd_detect_virt_container.rc | default(1) | int) == 0 }}"
|
||||
|
||||
- name: "Resolve deployment_mode (group-driven, host-scoped)"
|
||||
ansible.builtin.set_fact:
|
||||
deployment_mode: "{{ 'swarm' if 'svc-docker-swarm' in group_names else 'compose' }}"
|
||||
|
||||
- name: Check DNS resolution for test domain
|
||||
ansible.builtin.command: "getent hosts {{ DOMAIN_TEST }}"
|
||||
register: dns_check
|
||||
@@ -112,6 +116,15 @@
|
||||
- MODE_CLEANUP | bool
|
||||
- DOCKER_IN_CONTAINER | bool
|
||||
|
||||
- name: "Bootstrap swarm cluster + NFS clients (MUST precede Preload)"
|
||||
ansible.builtin.include_tasks: "./tasks/groups/{{ role_group }}-roles.yml"
|
||||
loop:
|
||||
- svc-docker
|
||||
- svc-storage
|
||||
loop_control:
|
||||
loop_var: role_group
|
||||
label: "{{ role_group }}-roles.yml"
|
||||
|
||||
- name: Preload shared services
|
||||
ansible.builtin.include_role:
|
||||
name: sys-service-loader
|
||||
|
||||
@@ -14,6 +14,14 @@
|
||||
loop_control:
|
||||
loop_var: server_role
|
||||
|
||||
- name: "Bootstrap swarm cluster + NFS clients (must precede web-app render)"
|
||||
ansible.builtin.include_role:
|
||||
name: "{{ item }}"
|
||||
loop:
|
||||
- svc-docker-swarm
|
||||
- svc-storage-nfs-client
|
||||
when: "'svc-docker-swarm' in group_names"
|
||||
|
||||
- name: "Include server roles"
|
||||
include_tasks: "./tasks/groups/{{ server_group }}-roles.yml"
|
||||
loop:
|
||||
|
||||
@@ -196,6 +196,80 @@ class TestComposeVolumes(unittest.TestCase):
|
||||
|
||||
self.assertIsNone(data["volumes"]["data"]["name"])
|
||||
|
||||
def test_swarm_nfs_opt_in_renders_driver_opts(self):
|
||||
apps = self._base_apps()
|
||||
rendered = compose_volumes(
|
||||
apps,
|
||||
"app",
|
||||
extra_volumes={"images": {"name": "app_images", "nfs": True}},
|
||||
deployment_mode="swarm",
|
||||
storage={
|
||||
"backend": "nfs",
|
||||
"nfs": {"server": "10.0.0.20", "export_base": "/srv/nfs"},
|
||||
},
|
||||
)
|
||||
data = self._parse_yaml(rendered)
|
||||
|
||||
vol = data["volumes"]["images"]
|
||||
self.assertEqual(vol["name"], "app_images")
|
||||
self.assertEqual(vol["driver"], "local")
|
||||
self.assertEqual(vol["driver_opts"]["type"], "nfs")
|
||||
self.assertIn("addr=10.0.0.20", vol["driver_opts"]["o"])
|
||||
self.assertIn("vers=4", vol["driver_opts"]["o"])
|
||||
self.assertEqual(vol["driver_opts"]["device"], ":/srv/nfs/app_images")
|
||||
self.assertNotIn("nfs", vol)
|
||||
|
||||
def test_swarm_nfs_disabled_volume_stays_local(self):
|
||||
apps = self._base_apps()
|
||||
rendered = compose_volumes(
|
||||
apps,
|
||||
"app",
|
||||
extra_volumes={
|
||||
"images": {"name": "app_images", "nfs": True},
|
||||
"data": {"name": "app_data"},
|
||||
},
|
||||
deployment_mode="swarm",
|
||||
storage={
|
||||
"backend": "nfs",
|
||||
"nfs": {"server": "10.0.0.20", "export_base": "/srv/nfs"},
|
||||
},
|
||||
)
|
||||
data = self._parse_yaml(rendered)
|
||||
|
||||
self.assertIn("driver_opts", data["volumes"]["images"])
|
||||
self.assertNotIn("driver_opts", data["volumes"]["data"])
|
||||
|
||||
def test_compose_mode_ignores_nfs_flag(self):
|
||||
apps = self._base_apps()
|
||||
rendered = compose_volumes(
|
||||
apps,
|
||||
"app",
|
||||
extra_volumes={"images": {"name": "app_images", "nfs": True}},
|
||||
deployment_mode="compose",
|
||||
storage={
|
||||
"backend": "nfs",
|
||||
"nfs": {"server": "10.0.0.20", "export_base": "/srv/nfs"},
|
||||
},
|
||||
)
|
||||
data = self._parse_yaml(rendered)
|
||||
|
||||
self.assertNotIn("driver_opts", data["volumes"]["images"])
|
||||
self.assertNotIn("nfs", data["volumes"]["images"])
|
||||
|
||||
def test_swarm_local_backend_ignores_nfs_flag(self):
|
||||
apps = self._base_apps()
|
||||
rendered = compose_volumes(
|
||||
apps,
|
||||
"app",
|
||||
extra_volumes={"images": {"name": "app_images", "nfs": True}},
|
||||
deployment_mode="swarm",
|
||||
storage={"backend": "local"},
|
||||
)
|
||||
data = self._parse_yaml(rendered)
|
||||
|
||||
self.assertNotIn("driver_opts", data["volumes"]["images"])
|
||||
self.assertNotIn("nfs", data["volumes"]["images"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user